diff --git a/lambdas/shared/Makefile b/lambdas/shared/Makefile index 0ecd1882c5..dbb8b04c9b 100644 --- a/lambdas/shared/Makefile +++ b/lambdas/shared/Makefile @@ -1,5 +1,10 @@ +TEST_ENV := @PYTHONPATH=src:tests:src/common + test: - @PYTHONPATH=src:../ python -m unittest discover -s tests -p "test_*.py" -v + $(TEST_ENV) python -m unittest discover -s tests -p "test_*.py" -v + +testv: + $(TEST_ENV) python -m unittest discover -s tests/test_common/validator -p "test_*.py" -v test-list: @PYTHONPATH=src:tests python -m unittest discover -s tests -p "test_*.py" --verbose | grep test_ diff --git a/lambdas/shared/poetry.lock b/lambdas/shared/poetry.lock index 0c0539dadf..d564634790 100644 --- a/lambdas/shared/poetry.lock +++ b/lambdas/shared/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "async-timeout" @@ -818,6 +818,35 @@ urllib3 = ">=1.25.10,<3.0" [package.extras] tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-PyYAML", "types-requests"] +[[package]] +name = "ruff" +version = "0.14.0" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "ruff-0.14.0-py3-none-linux_armv6l.whl", hash = "sha256:58e15bffa7054299becf4bab8a1187062c6f8cafbe9f6e39e0d5aface455d6b3"}, + {file = "ruff-0.14.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:838d1b065f4df676b7c9957992f2304e41ead7a50a568185efd404297d5701e8"}, + {file = "ruff-0.14.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:703799d059ba50f745605b04638fa7e9682cc3da084b2092feee63500ff3d9b8"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ba9a8925e90f861502f7d974cc60e18ca29c72bb0ee8bfeabb6ade35a3abde7"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e41f785498bd200ffc276eb9e1570c019c1d907b07cfb081092c8ad51975bbe7"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30a58c087aef4584c193aebf2700f0fbcfc1e77b89c7385e3139956fa90434e2"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f8d07350bc7af0a5ce8812b7d5c1a7293cf02476752f23fdfc500d24b79b783c"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eec3bbbf3a7d5482b5c1f42d5fc972774d71d107d447919fca620b0be3e3b75e"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16b68e183a0e28e5c176d51004aaa40559e8f90065a10a559176713fcf435206"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb732d17db2e945cfcbbc52af0143eda1da36ca8ae25083dd4f66f1542fdf82e"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:c958f66ab884b7873e72df38dcabee03d556a8f2ee1b8538ee1c2bbd619883dd"}, + {file = "ruff-0.14.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7eb0499a2e01f6e0c285afc5bac43ab380cbfc17cd43a2e1dd10ec97d6f2c42d"}, + {file = "ruff-0.14.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4c63b2d99fafa05efca0ab198fd48fa6030d57e4423df3f18e03aa62518c565f"}, + {file = "ruff-0.14.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:668fce701b7a222f3f5327f86909db2bbe99c30877c8001ff934c5413812ac02"}, + {file = "ruff-0.14.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a86bf575e05cb68dcb34e4c7dfe1064d44d3f0c04bbc0491949092192b515296"}, + {file = "ruff-0.14.0-py3-none-win32.whl", hash = "sha256:7450a243d7125d1c032cb4b93d9625dea46c8c42b4f06c6b709baac168e10543"}, + {file = "ruff-0.14.0-py3-none-win_amd64.whl", hash = "sha256:ea95da28cd874c4d9c922b39381cbd69cb7e7b49c21b8152b014bd4f52acddc2"}, + {file = "ruff-0.14.0-py3-none-win_arm64.whl", hash = "sha256:f42c9495f5c13ff841b1da4cb3c2a42075409592825dada7c5885c2c844ac730"}, + {file = "ruff-0.14.0.tar.gz", hash = "sha256:62ec8969b7510f77945df916de15da55311fade8d6050995ff7f680afe582c57"}, +] + [[package]] name = "s3transfer" version = "0.14.0" @@ -911,4 +940,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = "~3.11" -content-hash = "bb81e1d561bff2f0ba6b13575d1617f37456d36e5b2140fbf70382102a984aff" +content-hash = "819920e5e6f30601ceb20849edf39d63ab5612815209cad3770026db70cebe90" diff --git a/lambdas/shared/pyproject.toml b/lambdas/shared/pyproject.toml index f3f3c8b9f4..bdfa301777 100644 --- a/lambdas/shared/pyproject.toml +++ b/lambdas/shared/pyproject.toml @@ -29,7 +29,24 @@ pyjwt = "^2.10.1" [tool.poetry.group.dev.dependencies] coverage = "^7.10.7" +ruff = "^0.14.0" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.ruff] +line-length = 120 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "B", "C4", "I", "UP"] # Common sensible defaults +ignore = ["E501"] # Ignore line-too-long if needed + +[tool.ruff.lint.isort] +force-single-line = true # Optional: force single-line imports + +[tool.ruff.format] +quote-style = "double" # Or "single", depending on your code style +indent-style = "space" +line-ending = "lf" diff --git a/lambdas/shared/src/common/aws_dynamodb.py b/lambdas/shared/src/common/aws_dynamodb.py index 67eb1eaf0d..2c4ea698c0 100644 --- a/lambdas/shared/src/common/aws_dynamodb.py +++ b/lambdas/shared/src/common/aws_dynamodb.py @@ -1,4 +1,5 @@ -from common.clients import dynamodb_resource, logger +from common.clients import dynamodb_resource +from common.clients import logger def get_dynamodb_table(table_name): diff --git a/lambdas/shared/src/common/aws_lambda_event.py b/lambdas/shared/src/common/aws_lambda_event.py index b73c898c6c..413c0a5883 100644 --- a/lambdas/shared/src/common/aws_lambda_event.py +++ b/lambdas/shared/src/common/aws_lambda_event.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Dict +from typing import Any class AwsEventType(Enum): @@ -10,7 +10,7 @@ class AwsEventType(Enum): class AwsLambdaEvent: - def __init__(self, event: Dict[str, Any]): + def __init__(self, event: dict[str, Any]): self.event_source = None self.event_type = AwsEventType.UNKNOWN self.event_source = event.get("eventSource") diff --git a/lambdas/shared/src/common/cache.py b/lambdas/shared/src/common/cache.py index f2d24e5e4a..1d34bedef6 100644 --- a/lambdas/shared/src/common/cache.py +++ b/lambdas/shared/src/common/cache.py @@ -1,5 +1,4 @@ import json -from typing import Optional class Cache: @@ -19,7 +18,7 @@ def put(self, key: str, value: dict): self.cache[key] = value self._overwrite() - def get(self, key: str) -> Optional[dict]: + def get(self, key: str) -> dict | None: return self.cache.get(key, None) def delete(self, key: str): diff --git a/lambdas/shared/src/common/log_decorator.py b/lambdas/shared/src/common/log_decorator.py index 63c81610bf..cabbdf5750 100644 --- a/lambdas/shared/src/common/log_decorator.py +++ b/lambdas/shared/src/common/log_decorator.py @@ -9,7 +9,8 @@ from datetime import datetime from functools import wraps -from common.clients import firehose_client, logger +from common.clients import firehose_client +from common.clients import logger def send_log_to_firehose(stream_name, log_data: dict) -> None: diff --git a/lambdas/shared/src/common/s3_reader.py b/lambdas/shared/src/common/s3_reader.py index 6b0505b911..7eb6347565 100644 --- a/lambdas/shared/src/common/s3_reader.py +++ b/lambdas/shared/src/common/s3_reader.py @@ -1,4 +1,5 @@ -from common.clients import logger, s3_client +from common.clients import logger +from common.clients import s3_client class S3Reader: diff --git a/lambdas/shared/src/common/validator/__init__.py b/lambdas/shared/src/common/validator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/src/common/validator/enums/__init__.py b/lambdas/shared/src/common/validator/enums/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/src/common/validator/enums/error_levels.py b/lambdas/shared/src/common/validator/enums/error_levels.py new file mode 100644 index 0000000000..51a31ffabf --- /dev/null +++ b/lambdas/shared/src/common/validator/enums/error_levels.py @@ -0,0 +1,11 @@ +# all error Levels +CRITICAL_ERROR = 0 +WARNING = 1 +NOTIFICATION = 2 + + +MESSAGES = { + CRITICAL_ERROR: "Critical Validation Error [%s]: %s", + WARNING: "Non-Critical Validation Error [%s]: %s", + NOTIFICATION: "Quality Issue Found [%s]: %s", +} diff --git a/lambdas/shared/src/common/validator/enums/exception_messages.py b/lambdas/shared/src/common/validator/enums/exception_messages.py new file mode 100644 index 0000000000..386e025e0e --- /dev/null +++ b/lambdas/shared/src/common/validator/enums/exception_messages.py @@ -0,0 +1,30 @@ +# all exceptions and messgaes +UNEXPECTED_EXCEPTION = 0 +VALUE_CHECK_FAILED = 1 +HEADER_CHECK_FAILED = 2 +RECORD_LENGTH_CHECK_FAILED = 3 +VALUE_PREDICATE_FALSE = 4 +RECORD_CHECK_FAILED = 5 +RECORD_PREDICATE_FALSE = 6 +UNIQUE_CHECK_FAILED = 7 +ASSERT_CHECK_FAILED = 8 +FINALLY_ASSERT_CHECK_FAILED = 9 +PARSING_ERROR = 10 +PARENT_FAILED = 11 +KEY_CHECK_FAILED = 12 + +MESSAGES = { + UNEXPECTED_EXCEPTION: "Unexpected exception [%s]: %s", + VALUE_CHECK_FAILED: "Value check failed.", + HEADER_CHECK_FAILED: "Header check failed.", + RECORD_LENGTH_CHECK_FAILED: "Record length check failed.", + RECORD_CHECK_FAILED: "Record check failed.", + VALUE_PREDICATE_FALSE: "Value predicate returned false.", + RECORD_PREDICATE_FALSE: "Record predicate returned false.", + UNIQUE_CHECK_FAILED: "Unique check failed.", + ASSERT_CHECK_FAILED: "Assertion check failed.", + FINALLY_ASSERT_CHECK_FAILED: "Final assertion check failed.", + PARSING_ERROR: "Failed to parse data correctly.", + PARENT_FAILED: "The parent expression failed to validate", + KEY_CHECK_FAILED: "Value could not be found in the Key list", +} diff --git a/lambdas/shared/src/common/validator/expression_checker.py b/lambdas/shared/src/common/validator/expression_checker.py new file mode 100644 index 0000000000..13152a8b3e --- /dev/null +++ b/lambdas/shared/src/common/validator/expression_checker.py @@ -0,0 +1,736 @@ +# Root and base type expression checker functions +import datetime +import re +import uuid +from enum import Enum + +import common.validator.enums.exception_messages as ExceptionMessages +from common.validator.lookup.key_data import KeyData +from common.validator.lookup.lookup_data import LookUpData +from common.validator.record_error import ErrorReport +from common.validator.record_error import RecordError + + +class ExpressionType(Enum): + DATETIME = "DATETIME" + DATE = "DATE" + UUID = "UUID" + INT = "INT" + FLOAT = "FLOAT" + REGEX = "REGEX" + EQUAL = "EQUAL" + NOTEQUAL = "NOTEQUAL" + IN = "IN" + NRANGE = "NRANGE" + INARRAY = "INARRAY" + UPPER = "UPPER" + LOWER = "LOWER" + LENGTH = "LENGTH" + STARTSWITH = "STARTSWITH" + ENDSWITH = "ENDSWITH" + EMPTY = "EMPTY" + NOTEMPTY = "NOTEMPTY" + POSITIVE = "POSITIVE" + GENDER = "GENDER" + NHSNUMBER = "NHSNUMBER" + MAXOBJECTS = "MAXOBJECTS" + POSTCODE = "POSTCODE" + ONLYIF = "ONLYIF" + LOOKUP = "LOOKUP" + KEYCHECK = "KEYCHECK" + + +class ExpressionChecker: + def __init__(self, data_parser, summarise, report_unexpected_exception): + self.data_parser = data_parser # FHIR data parser for additional functions + self.data_look_up = LookUpData() # used for generic look up + self.key_data = KeyData() # used for key check on data we know (Snomed / ODS etc) + self.summarise = summarise + self.report_unexpected_exception = report_unexpected_exception + + def validate_expression(self, expression_type: str, rule, field_name, field_value, row) -> ErrorReport: + match expression_type: + case "DATETIME": + return self._validate_datetime(rule, field_name, field_value, row) + case "DATE": + return self._validate_datetime(rule, field_name, field_value, row) + case "UUID": + return self._validate_uuid(rule, field_name, field_value, row) + case "INT": + return self._validate_integer(rule, field_name, field_value, row) + case "FLOAT": + return self._validate_float(rule, field_name, field_value, row) + case "REGEX": + return self._validate_regex(rule, field_name, field_value, row) + case "EQUAL": + return self._validate_equal(rule, field_name, field_value, row) + case "NOTEQUAL": + return self._validate_not_equal(rule, field_name, field_value, row) + case "IN": + return self._validate_in(rule, field_name, field_value, row) + case "NRANGE": + return self._validate_n_range(rule, field_name, field_value, row) + case "INARRAY": + return self._validate_in_array(rule, field_name, field_value, row) + case "UPPER": + return self._validate_upper(rule, field_name, field_value, row) + case "LOWER": + return self._validate_lower(rule, field_name, field_value, row) + case "LENGTH": + return self._validate_length(rule, field_name, field_value, row) + case "STARTSWITH": + return self._validate_starts_with(rule, field_name, field_value, row) + case "ENDSWITH": + return self._validate_ends_with(rule, field_name, field_value, row) + case "EMPTY": + return self._validate_empty(rule, field_name, field_value, row) + case "NOTEMPTY": + return self._validate_not_empty(rule, field_name, field_value, row) + case "POSITIVE": + return self._validate_positive(rule, field_name, field_value, row) + case "POSTCODE": + return self._validate_post_code(rule, field_name, field_value, row) + case "GENDER": + return self._validate_gender(rule, field_name, field_value, row) + case "NHSNUMBER": + return self._validate_nhs_number(rule, field_name, field_value, row) + case "MAXOBJECTS": + return self._validate_max_objects(rule, field_name, field_value, row) + case "ONLYIF": + return self._validate_only_if(rule, field_name, field_value, row) + case "LOOKUP": + return self._validate_against_lookup(rule, field_name, field_value, row) + case "KEYCHECK": + return self._validate_against_key(rule, field_name, field_value, row) + case _: + return "Schema expression not found! Check your expression type : " + expression_type + + # iso8086 date time validate + def _validate_datetime(self, rule, field_name, field_value, row) -> ErrorReport: + try: + datetime.date.fromisoformat(field_value) + # TODO - rule is not used - could be date only, date time, past, future etc + if rule: + pass + + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # UUID validate + def _validate_uuid(self, expressionRule, field_name, field_value, row) -> ErrorReport: + try: + uuid.UUID(str(field_value)) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Integer Validate + def _validate_integer(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + int(field_value) + if expression_rule: + # TODO - code is incomplete here. It appears there should be a check + # against expression_rule but it's not implemented. eg max, min, equal etc + # eg "1" means value must be 1 + # "1:10" means value must be between 1 to 10 + # "1,10" means value must be either 1 or 10 + # ":10" means value must be less than or equal to 10 + # "1:" means value must be greater than or equal to 1 + # ">10" means value must be greater than 10 + # "<10" means value must be less than 10 + + check_value = int(expression_rule) + if field_value != check_value: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value integer check failed", + "Value does not equal expected value, Expected- " + expression_rule + " found- " + field_value, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Float Validate + def _validate_float(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + float(field_value) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Length Validate + def _validate_length(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + str_len = len(field_value) + check_length = int(expression_rule) + if str_len > check_length: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, "Value length check failed", "Value is longer than expected" + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Regex Validate + def _validate_regex(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + result = re.search(expression_rule, field_value) + if not result: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "String REGEX check failed", + "Value does not meet regex rules", + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Equal Validate + def _validate_equal(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + if field_value != expression_rule: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value equals check failed", + "Value does not equal expected value, Expected- " + expression_rule + " found- " + field_value, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Not Equal Validate + def _validate_not_equal(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + if field_value == expression_rule: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value not equals check failed", + "Value equals expected value when it should not, Expected- " + + expression_rule + + " found- " + + field_value, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # In Validate + def _validate_in(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + if expression_rule.lower() in field_value.lower(): + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Data not in Value failed", + "Check Data not found in Value, List- " + expression_rule + " Data- " + field_value, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # NRange Validate + def _validate_n_range(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + value = float(field_value) + rule = expression_rule.split(",") + range1 = float(rule[0]) + range2 = float(rule[1]) + + if range1 <= value >= range2: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value range check failed", + "Value is not within the number range, data- " + field_value, + ) + return None + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # InArray Validate + def _validate_in_array(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + rule_list = expression_rule.split(",") + + if field_value not in rule_list: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value not in array check failed", + "Check Value not found in data array", + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Upper Validate + def _validate_upper(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + result = field_value.isupper() + + if not result: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value not uppercase", + "Check Value not found to be uppercase, value- " + field_value, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Lower Validate + def _validate_lower(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + result = field_value.islower() + + if not result: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value not lowercase", + "Check Value not found to be lowercase, data- " + field_value, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Starts With Validate + def _validate_starts_with(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + result = field_value.startswith(expression_rule) + if not result: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value starts with failure", + "Value does not start as expected, Expected- " + expression_rule + " found- " + field_value, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Ends With Validate + def _validate_ends_with(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + result = field_value.endswith(expression_rule) + if not result: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value ends with failure", + "Value does not end as expected, Expected- " + expression_rule + " found- " + field_value, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Empty Validate + def _validate_empty(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + if field_value: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value is empty failure", + "Value has data, not as expected, data- " + field_value, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Not Empty Validate + def _validate_not_empty(self, expression_rule, field_name, field_value, row) -> ErrorReport: + try: + if not field_value: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, "Value not empty failure", "Value is empty, not as expected" + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, field_name, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) + + # Positive Validate + def _validate_positive(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + value = float(fieldValue) + if value < 0: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value is not positive failure", + "Value is not positive as expected, data- " + fieldValue, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) + + # NHSNumber Validate + def _validate_nhs_number(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + regexRule = "^6[0-9]{10}$" + result = re.search(regexRule, fieldValue) + if not result: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "NHS Number check failed", + "NHS Number does not meet regex rules, data- " + fieldValue, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) + + # Gender Validate + def _validate_gender(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + ruleList = ["0", "1", "2", "9"] + + if fieldValue not in ruleList: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Gender check failed", + "Gender value not found in array, data- " + fieldValue, + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) + + # PostCode Validate + def _validate_post_code(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + regexRule = "^([Gg][Ii][Rr] 0[Aa]{2})|((([A-Za-z][0-9]{1,2})|(([A-Za-z][A-Ha-hJ-Yj-y]" + "[0-9]{1,2})|(([AZa-z][0-9][A-Za-z])|([A-Za-z][A-Ha-hJ-Yj-y][0-9]?[A-Za-z])))) [0-9][A-Za-z]{2})$" + result = re.search(regexRule, fieldValue) + if not result: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, "Postcode check failed", "Postcode does not meet regex rules" + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) + + # Max Objects Validate + def _validate_max_objects(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + value = len(fieldValue) + if value > int(expressionRule): + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Max Objects failure", + "Number of objects is greater than expected", + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) + + # Default to Validate + def _validate_only_if(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + conversionList = expressionRule.split("|") + location = conversionList[0] + valueCheck = conversionList[1] + dataValue = self.dataParser.getKeyValue(location) + + if dataValue[0] != valueCheck: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Validate Only If failure", + "Value was not found at that position", + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) + + # Check with Lookup + def _validate_against_lookup(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + result = self.dataLookUp.findLookUp(fieldValue) + if not result: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value lookup failure", + "Value was not found in Lookup List, Expected- " + fieldValue + " found- nothing", + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) + + # Check with Key Lookup + def _validate_against_key(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + result = self.KeyData.findKey(expressionRule, fieldValue) + if not result: + raise RecordError( + ExceptionMessages.KEY_CHECK_FAILED, + "Key lookup failure", + "Value was not found in Key List, Expected- " + fieldValue + " found- nothing", + ) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.KEY_CHECK_FAILED + message = ( + e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.KEY_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) diff --git a/lambdas/shared/src/common/validator/lookup/__init__.py b/lambdas/shared/src/common/validator/lookup/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/src/common/validator/lookup/key_data.py b/lambdas/shared/src/common/validator/lookup/key_data.py new file mode 100644 index 0000000000..deb3071aeb --- /dev/null +++ b/lambdas/shared/src/common/validator/lookup/key_data.py @@ -0,0 +1,126 @@ +# --------------------------------------------------------------------------------------------------------- +# main conversion lookup + + +class KeyData: + # data settings + def __init__(self): + self.procedure = ["956951000000104"] + + self.organisation = ["RJ1", "RJC02"] + + self.site = [ + "368208006", + "279549004", + "74262004", + "368209003", + "723979003", + "61396006", + "723980000", + "11207009", + "420254004", + ] + + self.route = [ + "54471007", + "372449004", + "372450004", + "372451000", + "372452007", + "404820008", + "18246711000001107", + "372453002", + "372454008", + "127490009", + "372457001", + "9191401000001100", + "18682911000001103", + "10334211000001103", + "718329006", + "18679011000001101", + "34777511000001106", + "372458006", + "58100008", + "12130007", + "372459003", + "418821007", + "372460008", + "372461007", + "420719007", + "19537211000001108", + "372463005", + "372464004", + "372465003", + "448077001", + "38233211000001106", + "372466002", + "372467006", + "78421000", + "255559005", + "372468001", + "417255000", + "38239002", + "372469009", + "372470005", + "418586008", + "72607000", + "447122006", + "62226000", + "47625008", + "420287000", + "372471009", + "418401004", + "21856811000001103", + "127491008", + "9907001000001103", + "46713006", + "127492001", + "418730005", + "54485002", + "26643006", + "372473007", + "10547007", + "225691000001105", + "9191501000001101", + "372474001", + "39338211000001108", + "3323001000001107", + "372475000", + "11478901000001102", + "419762003", + "39337511000001107", + "37161004", + "11564311000001109", + "418321004", + "3594011000001102", + "372476004", + "34206005", + "37839007", + "419874009", + "11564211000001101", + "33770711000001104", + "6064005", + "45890007", + "11479001000001107", + "404815008", + "90028008", + "16857009", + ] + + # Look up the term for the code + def findKey(self, key_source, field_value): + try: + match key_source: + case "Procedure": + return field_value in self.Procedure + case "Organisation": + return field_value in self.Organisation + case "Site": + return field_value in self.Site + case "Route": + return field_value in self.Route + case _: + return False + except Exception: + return False + return False diff --git a/lambdas/shared/src/common/validator/lookup/lookup_data.py b/lambdas/shared/src/common/validator/lookup/lookup_data.py new file mode 100644 index 0000000000..42eadbb528 --- /dev/null +++ b/lambdas/shared/src/common/validator/lookup/lookup_data.py @@ -0,0 +1,109 @@ +# --------------------------------------------------------------------------------------------------------- +# main conversion lookup + + +class LookUpData: + # data settings + def __init__(self): + self.all_data = { + "368208006": "Left upper arm structure", + "279549004": "Nasal cavity structure", + "74262004": "Oral cavity structure", + "368209003": "Right upper arm structure", + "723979003": "Structure of left buttock", + "61396006": "Structure of left thigh", + "723980000": "Structure of right buttock", + "11207009": "Structure of right thigh", + "420254004": "Body cavity", + "54471007": "Buccal", + "372449004": "Dental", + "372450004": "Endocervical", + "372451000": "Endosinusial", + "372452007": "Endotracheopulmonary", + "404820008": "Epidural", + "18246711000001107": "Epilesional", + "372453002": "Extraamniotic", + "372454008": "Gastroenteral", + "127490009": "Gastrostomy route", + "372457001": "Gingival", + "9191401000001100": "Haemodiafiltration", + "18682911000001103": "Haemodialysis", + "10334211000001103": "Haemofiltration", + "718329006": "Infiltration", + "18679011000001101": "Inhalation", + "34777511000001106": "Intestinal use", + "372458006": "Intraamniotic", + "58100008": "Intraarterial", + "12130007": "Intraarticular", + "372459003": "Intrabursal", + "418821007": "Intracameral use", + "372460008": "Intracardiac", + "372461007": "Intracavernous", + "420719007": "Intracerebroventricular", + "19537211000001108": "Intracervical route", + "372463005": "Intracoronary", + "372464004": "Intradermal", + "372465003": "Intradiscal", + "448077001": "Intraepidermal", + "38233211000001106": "Intraglandular", + "372466002": "Intralesional", + "372467006": "Intralymphatic", + "78421000": "Intramuscular", + "255559005": "Intramuscular", + "372468001": "Intraocular", + "417255000": "Intraosseous", + "38239002": "Intraperitoneal", + "372469009": "Intrapleural", + "372470005": "Intrasternal", + "418586008": "Intratendinous route", + "72607000": "Intrathecal", + "447122006": "Intratumoral", + "62226000": "Intrauterine", + "47625008": "Intravenous", + "420287000": "Intraventricular cardiac", + "372471009": "Intravesical", + "418401004": "Intravitreal", + "21856811000001103": "Iontophoresis", + "127491008": "Jejunostomy route", + "9907001000001103": "Line lock", + "46713006": "Nasal", + "127492001": "Nasogastric route", + "418730005": "Nasojejunal route", + "54485002": "Ophthalmic route", + "26643006": "Oral", + "372473007": "Oromucosal", + "10547007": "Otic", + "225691000001105": "PEG tube route", + "9191501000001101": "Percutaneous", + "372474001": "Periarticular", + "39338211000001108": "Peribulbar ocular", + "3323001000001107": "Pericardial route", + "372475000": "Perineural", + "11478901000001102": "Periosseous", + "419762003": "Peritendinous route", + "39337511000001107": "Peritumoral", + "37161004": "Rectal", + "11564311000001109": "Regional perfusion", + "418321004": "Retrobulbar route", + "3594011000001102": "Route of administration not applicable", + "372476004": "Subconjunctival", + "34206005": "Subcutaneous", + "37839007": "Sublingual", + "419874009": "Submucosal", + "11564211000001101": "Submucosal rectal", + "33770711000001104": "Subretinal", + "6064005": "Topical", + "45890007": "Transdermal", + "11479001000001107": "Translingual", + "404815008": "Transmucosal", + "90028008": "Urethral", + "16857009": "Vaginal", + } + + # Look up the term for the code + def find_lookup(self, field_value): + try: + lookup_value = self.all_data[field_value] + except Exception: + lookup_value = "" + return lookup_value diff --git a/lambdas/shared/src/common/validator/parsers/__init__.py b/lambdas/shared/src/common/validator/parsers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/src/common/validator/parsers/csv_line_parser.py b/lambdas/shared/src/common/validator/parsers/csv_line_parser.py new file mode 100644 index 0000000000..bb07210254 --- /dev/null +++ b/lambdas/shared/src/common/validator/parsers/csv_line_parser.py @@ -0,0 +1,21 @@ +# CSV Row importer and data access +import csv + + +class CSVLineParser: + # parser variables + def __init__(self): + self.csv_file_data = {} + + # parse the CSV into a Dictionary + def parse_csv_line(self, csv_row, csv_header): + # create a key value mapping + keys = list(csv.reader([csv_header]))[0] + values = list(csv.reader([csv_row]))[0] + self.csv_file_data = dict(map(lambda i, j: (i, j), keys, values)) + + # retrieve a column of data to work with + def get_key_value(self, field_name): + # creating empty lists, convert to list + data = [self.csv_file_data[field_name]] + return data diff --git a/lambdas/shared/src/common/validator/parsers/csv_parser.py b/lambdas/shared/src/common/validator/parsers/csv_parser.py new file mode 100644 index 0000000000..9116c8b6b6 --- /dev/null +++ b/lambdas/shared/src/common/validator/parsers/csv_parser.py @@ -0,0 +1,28 @@ +# CSV importer and data access +import csv + + +class CSVParser: + """File Management""" + + # parser variables + def __init__(self): + self.csv_file_data = {} + + # parse the CSV into a Dictionary + def parse_csv_file(self, csv_filename): + input_file = csv.DictReader(open(csv_filename)) + input_file = csv.DictReader(open(csv_filename), delimiter="|") + self.csv_file_data = {elem: [] for elem in input_file.fieldnames} + keys = self.csv_file_data.keys() + for row in input_file: + for key in keys: + self.csv_file_data[key].append(row[key]) + + # --------------------------------------------- + # Scan and retrieve values + # retrieve a column of data to work with + def get_key_value(self, field_name): + # creating empty lists + data = self.csv_file_data[field_name] + return data diff --git a/lambdas/shared/src/common/validator/parsers/fhir_parser.py b/lambdas/shared/src/common/validator/parsers/fhir_parser.py new file mode 100644 index 0000000000..1eb32bb212 --- /dev/null +++ b/lambdas/shared/src/common/validator/parsers/fhir_parser.py @@ -0,0 +1,102 @@ +# FHIR JSON importer and data access +import json + + +class FHIRParser: + # parser variables + def __init__(self): + self.fhir_resource = {} + + # ------------------------------------------- + # File Management + # used for files + def parse_fhir_file(self, fhir_file_name): + with open(fhir_file_name) as json_file: + self.fhir_resource = json.load(json_file) + + # used for JSON FHIR Resource data + def parse_fhir_data(self, fhir_data): + self.fhir_resource = fhir_data + + # ------------------------------------------------ + # Scan and Identify + # scan for a key name or a value + def _scan_values_for_match(self, parent, match_value): + try: + for key in parent: + if parent[key] == match_value: + return True + return False + except Exception: + return False + + # locate an index for an item in a list + def _locate_list_id(self, parent, locator): + field_list = locator.split(":") + node_id = 0 + index = 0 + try: + while index < len(parent): + for key in parent[index]: + if (parent[index][key] == field_list[1]) or (key == field_list[1]): + node_id = index + break + else: + if self._scan_values_for_match(parent[index][key], field_list[1]): + node_id = index + break + index += 1 + except Exception: + return "" + return parent[node_id] + + # identify a node in the FHIR data + def _get_node(self, parent, child): + # check for indices + try: + result = parent[child] + except Exception: + try: + child = int(child) + result = parent[child] + except Exception: + result = "" + return result + + # locate a value for a key + def _scan_for_value(self, fhir_fields): + field_list = fhir_fields.split("|") + # get root field before we iterate + rootfield = self.fhir_resource[field_list[0]] + del field_list[0] + try: + for field in field_list: + if field.startswith("#"): + rootfield = self._locate_list_id(rootfield, field) # check here for default index?? + else: + rootfield = self._get_node(rootfield, field) + except Exception: + rootfield = "" + return rootfield + + # get the value list for a key + def get_key_value(self, field_name): + value = [] + try: + response_value = self._scan_for_value(field_name) + except Exception: + response_value = "" + + value.append(response_value) + return value + + # get the value list for a key + def get_key_single_value(self, field_name): + value = "" + try: + response_value = self._scan_for_value(field_name) + except Exception: + response_value = "" + + value = response_value + return value diff --git a/lambdas/shared/src/common/validator/parsers/schema_parser.py b/lambdas/shared/src/common/validator/parsers/schema_parser.py new file mode 100644 index 0000000000..3294f3fbac --- /dev/null +++ b/lambdas/shared/src/common/validator/parsers/schema_parser.py @@ -0,0 +1,25 @@ +# Schema Parser +# Moved from file loading to JSON string better for elasticache + + +class SchemaParser: + def __init__(self): + # parser variables + self.schema_file = {} + self.expressions = {} + + def parse_schema(self, schema_file): # changed to accept JSON better for cache + self.schema_file = schema_file + self.expressions = self.schema_file["expressions"] + + def expression_count(self): + count = 0 + count = sum([1 for d in self.expressions if "expression" in d]) + return count + + def get_expressions(self): + return self.expressions + + def get_expression(self, expression_number): + expression = self.expressions[expression_number] + return expression diff --git a/lambdas/shared/src/common/validator/record_error.py b/lambdas/shared/src/common/validator/record_error.py new file mode 100644 index 0000000000..f9bd0442ed --- /dev/null +++ b/lambdas/shared/src/common/validator/record_error.py @@ -0,0 +1,44 @@ +class ErrorReport: + def __init__( + self, + code: int = None, + message: str = None, + row: int = None, + field: str = None, + details: str = None, + summarise: bool = False, + error_level: int = None, + ): + self.code = code + self.message = message + self.row = row + self.field = field + self.details = details + self.summarise = summarise + # these are set when the error is added to the report + self.error_group = None + self.name = None + self.id = None + self.error_level = error_level + + # function to return the object as a dictionary + def to_dict(self): + ret = {"code": self.code, "message": self.message} + if not self.summarise: + ret.update({"row": self.row, "field": self.field, "details": self.details}) + return ret + + +# record exception capture +class RecordError(Exception): + def __init__(self, code=None, message=None, details=None): + super().__init__(message) + self.code = code + self.message = message + self.details = details + + def __str__(self): + return repr((self.code, self.message, self.details)) + + def __repr__(self): + return repr((self.code, self.message, self.details)) diff --git a/lambdas/shared/src/common/validator/reporter/__init__.py b/lambdas/shared/src/common/validator/reporter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/src/common/validator/reporter/dq_reporter.py b/lambdas/shared/src/common/validator/reporter/dq_reporter.py new file mode 100644 index 0000000000..9874c8814f --- /dev/null +++ b/lambdas/shared/src/common/validator/reporter/dq_reporter.py @@ -0,0 +1,59 @@ +import datetime +import json + +from dateutil import parser + +import common.validator.enums.error_levels as ErrorLevels +from common.validator.record_error import ErrorReport + + +class DQReporter: + def __init__(self): + # parser variables + self.error_report = { + "eventId": "", + "validationDate": "", + "validated": "true", + "results": { + "totalErrors": 0, + "completeness": {"errors": 0, "fields": []}, + "consistency": {"errors": 0, "fields": []}, + "validity": {"errors": 0, "fields": []}, + "timeliness_processed": 0, + }, + } + + # create the date difference for the report in minutes + def diff_dates(self, date1, date2): + diff_seconds = abs(date2 - date1).total_seconds() + diff_minutes = diff_seconds / 60 + return diff_minutes + + def generate_error_report(self, event_id, occurrence, error_records: list[ErrorReport]): + occurrence_date = occurrence + occurrence_date = parser.parse(occurrence_date, ignoretz=True) + validation_date = datetime.datetime.now(tz=None) + + time_taken = self.diff_dates(occurrence_date, validation_date) + + self.error_report["validationDate"] = validation_date.isoformat() + self.error_report["eventId"] = event_id + self.error_report["results"]["timeliness_processed"] = time_taken + + for errorRecord in error_records: + self.update_report(errorRecord) + + json_error_report = json.dumps(self.error_report) + return json_error_report + + def update_report(self, error_data: ErrorReport): + error_group = error_data.error_group + if error_data.error_level == ErrorLevels.CRITICAL_ERROR: + self.error_report["validated"] = "false" + total_errors = self.error_report["results"]["totalErrors"] + results_error_count = self.error_report["results"][error_group]["errors"] + results_error_count += 1 + total_errors += 1 + self.error_report["results"][error_group]["fields"].append(error_data.name) + self.error_report["results"][error_group]["errors"] = results_error_count + self.error_report["results"]["totalErrors"] = total_errors diff --git a/lambdas/shared/src/common/validator/validator.py b/lambdas/shared/src/common/validator/validator.py new file mode 100644 index 0000000000..70fc7e996c --- /dev/null +++ b/lambdas/shared/src/common/validator/validator.py @@ -0,0 +1,232 @@ +# Main validation engine + +from enum import Enum + +import common.validator.enums.error_levels as ErrorLevels +import common.validator.enums.exception_messages as ExceptionMessages +from common.validator.expression_checker import ExpressionChecker +from common.validator.parsers.csv_line_parser import CSVLineParser +from common.validator.parsers.csv_parser import CSVParser +from common.validator.parsers.fhir_parser import FHIRParser +from common.validator.parsers.schema_parser import SchemaParser +from common.validator.record_error import ErrorReport +from common.validator.reporter.dq_reporter import DQReporter + + +class DataType(Enum): + FHIR = "FHIR" + FHIRJSON = "FHIRJSON" + CSV = "CSV" + CSVROW = "CSVROW" + + +class Validator: + def __init__(self, schema_file="", data_type: DataType = None, filepath=""): + self.filepath = filepath + self.json_data = {} + self.schema_file = schema_file + self.csv_row = "" + self.csv_header = "" + self.data_type = data_type + self.data_parser = "" + self.error_records: list[ErrorReport] = [] + + def _get_csv_line_parser(self, csv_row, csv_header): + csv_parser = CSVLineParser() + csv_parser.parse_csv_line(csv_row, csv_header) + return csv_parser + + def _get_csv_parser(self, filepath): + csv_parser = CSVParser() + csv_parser.parse_csv_file(filepath) + return csv_parser + + def _get_fhir_parser(self, filepath): + fhir_parser = FHIRParser() + fhir_parser.parse_fhir_file(filepath) + return fhir_parser + + def _get_fhir_json_parser(self, fhir_data): + fhir_parser = FHIRParser() + fhir_parser.parse_fhir_data(fhir_data) + return fhir_parser + + def _get_schema_parser(self, schemafile): + schema_parser = SchemaParser() + schema_parser.parse_schema(schemafile) + return schema_parser + + def _add_error_record( + self, error_record: ErrorReport, expression_error_group, expression_name, expression_id, error_level + ): + if error_record is not None: + error_record.error_group = expression_error_group + error_record.name = expression_name + error_record.id = expression_id + error_record.error_level = error_level + self.error_records.append(error_record) + + # Function to help identify a parent failure in the error list + def _check_error_record_for_fail(self, expression_id): + for error_record in self.error_records: + if error_record.id == expression_id: + return True + return False + + # validate a single expression against the data file + def _validate_expression( + self, expression_validator: ExpressionChecker, expression, inc_header_in_row_count + ) -> ErrorReport | int: + row = 1 + if inc_header_in_row_count: + row = 2 + + if self.isCSV: + expression_fieldname = expression["fieldNameCSV"] + else: + expression_fieldname = expression["fieldNameFHIR"] + + expression_id = expression["expressionId"] + error_level = expression["errorLevel"] + expression_name = expression["expression"]["expressionName"] + expression_type = expression["expression"]["expressionType"] + expression_rule = expression["expression"]["expressionRule"] + expression_error_group = expression["errorGroup"] + + # Check to see if the expression has a parent, if so did the parent validate + if "parentExpression" in expression: + parent_expression = expression["parentExpression"] + if self._check_error_record_for_fail(parent_expression): + error_record = { + "code": ExceptionMessages.PARENT_FAILED, + "message": ExceptionMessages.MESSAGES[ExceptionMessages.PARENT_FAILED] + + ", Parent ID: " + + parent_expression, + } + self._add_error_record( + error_record, expression_error_group, expression_name, expression_id, error_level + ) + return error_record + + try: + expression_values = self.data_parser.get_key_value(expression_fieldname) + except Exception as e: + message = f"Data get values Unexpected exception [{e.__class__.__name__}]: {e}" + error_report = ErrorReport(code=ExceptionMessages.PARSING_ERROR, message=message) + # original code had self.CriticalErrorLevel. Replaced with error_level + self._add_error_record(error_report, expression_error_group, expression_name, expression_id, error_level) + return error_report + + for value in expression_values: + error_record: ErrorReport | None = None + try: + error_record = expression_validator.validate_expression( + expression_type, expression_rule, expression_fieldname, value, row + ) + if error_record is not None: + self._add_error_record( + error_record, expression_error_group, expression_name, expression_id, error_level + ) + except Exception: + print(f"Exception validating expression {expression_id} on row {row}: {error_record}") + row += 1 + return row + + def validate_fhir( + self, filepath, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True + ) -> list[ErrorReport]: + self.data_type = DataType.FHIR + self.filepath = filepath + return self.run_validation(summarise, report_unexpected_exception, inc_header_in_row_count) + + def validate_csv( + self, filepath, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True + ) -> list[ErrorReport]: + self.data_type = DataType.CSV + self.filepath = filepath + return self.run_validation(summarise, report_unexpected_exception, inc_header_in_row_count) + + def validate_csv_row( + self, csv_row, csv_header, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True + ) -> list[ErrorReport]: + self.data_type = DataType.CSVROW + self.csv_row = csv_row + self.csv_header = csv_header + return self.run_validation(summarise, report_unexpected_exception, inc_header_in_row_count) + + def validate_fhir_json( + self, json_data, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True + ) -> list[ErrorReport]: + self.data_type = DataType.FHIRJSON + self.json_data = json_data + return self.validate_fhir_json(json_data, summarise, report_unexpected_exception, inc_header_in_row_count) + + # run the validation against the data + def run_validation( + self, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True + ) -> list[ErrorReport]: + try: + self.error_records.clear() + + match self.data_type: # 'FHIR', 'FHIRJSON', 'CSV', 'CSVROW' + case DataType.FHIR: + self.data_parser = self._get_fhir_parser(self.filepath) + self.isCSV = False + case DataType.FHIRJSON: + self.data_parser = self._get_fhir_json_parser(self.json_data) + self.isCSV = False + case DataType.CSV: + self.data_parser = self._get_csv_parser(self.filepath) + self.isCSV = True + case DataType.CSVROW: + self.data_parser = self._get_csv_line_parser(self.csv_row, self.csv_header) + self.isCSV = True + + except Exception as e: + if report_unexpected_exception: + message = f"Data Parser Unexpected exception [{e.__class__.__name__}]: {e}" + return [ErrorReport(code=0, message=message)] + + try: + schemaParser = self._get_schema_parser(self.schema_file) + except Exception as e: + if report_unexpected_exception: + message = f"Schema Parser Unexpected exception [{e.__class__.__name__}]: {e}" + return [ErrorReport(code=0, message=message)] + + try: + expression_validator = ExpressionChecker(self.data_parser, summarise, report_unexpected_exception) + except Exception as e: + if report_unexpected_exception: + message = f"Expression Checker Unexpected exception [{e.__class__.__name__}]: {e}" + return [ErrorReport(code=0, message=message)] + + # get list of expressions + try: + expressions = schemaParser.get_expressions() + except Exception as e: + if report_unexpected_exception: + message = f"Expression Getter Unexpected exception [{e.__class__.__name__}]: {e}" + return [ErrorReport(code=0, message=message)] + + for expression in expressions: + self._validate_expression(expression_validator, expression, inc_header_in_row_count) + + return self.error_records + + # ------------------------------------------------------------------------- + # Report Generation + # Build the error Report + def build_error_report(self, eventId): + OccurrenceDateTime = self.data_parser.get_key_single_value("occurrenceDateTime") + dq_reporter = DQReporter() + dq_report = dq_reporter.generate_error_report(eventId, OccurrenceDateTime, self.error_records) + + return dq_report + + # Check all errors to see if we have a critical error that would fail the validation + def has_validation_failed(self): + for error_record in self.error_records: + if error_record.error_level == ErrorLevels.CRITICAL_ERROR: + return True + return False diff --git a/lambdas/shared/tests/__init__.py b/lambdas/shared/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/tests/test_common/test_authentication.py b/lambdas/shared/tests/test_common/test_authentication.py index de31eda964..69cc81781a 100644 --- a/lambdas/shared/tests/test_common/test_authentication.py +++ b/lambdas/shared/tests/test_common/test_authentication.py @@ -2,12 +2,15 @@ import json import time import unittest -from unittest.mock import ANY, MagicMock, patch +from unittest.mock import ANY +from unittest.mock import MagicMock +from unittest.mock import patch import responses from responses import matchers -from common.authentication import AppRestrictedAuth, Service +from common.authentication import AppRestrictedAuth +from common.authentication import Service from common.models.errors import UnhandledResponseError diff --git a/lambdas/shared/tests/test_common/test_aws_dynamodb.py b/lambdas/shared/tests/test_common/test_aws_dynamodb.py index b4a025bb58..5c94f13d69 100644 --- a/lambdas/shared/tests/test_common/test_aws_dynamodb.py +++ b/lambdas/shared/tests/test_common/test_aws_dynamodb.py @@ -1,5 +1,6 @@ import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock +from unittest.mock import patch from common.aws_dynamodb import get_dynamodb_table diff --git a/lambdas/shared/tests/test_common/test_aws_lambda_event.py b/lambdas/shared/tests/test_common/test_aws_lambda_event.py index 3f4ac87879..87e31645ce 100644 --- a/lambdas/shared/tests/test_common/test_aws_lambda_event.py +++ b/lambdas/shared/tests/test_common/test_aws_lambda_event.py @@ -1,6 +1,7 @@ import unittest -from common.aws_lambda_event import AwsEventType, AwsLambdaEvent +from common.aws_lambda_event import AwsEventType +from common.aws_lambda_event import AwsLambdaEvent class TestAwsLambdaEvent(unittest.TestCase): diff --git a/lambdas/shared/tests/test_common/test_cache.py b/lambdas/shared/tests/test_common/test_cache.py index 3e3a2bd194..8125099ac8 100644 --- a/lambdas/shared/tests/test_common/test_cache.py +++ b/lambdas/shared/tests/test_common/test_cache.py @@ -70,7 +70,7 @@ def test_write_to_file(self): self.cache.put(key, new_value) # Then - with open(self.cache.cache_file.name, "r") as stored: + with open(self.cache.cache_file.name) as stored: content = json.loads(stored.read()) self.assertDictEqual(content[key], new_value) @@ -83,6 +83,6 @@ def test_cache_create_empty(self): self.cache = Cache(tempfile.gettempdir()) # Then - with open(self.cache.cache_file.name, "r") as stored: + with open(self.cache.cache_file.name) as stored: content = stored.read() self.assertEqual(len(content), 0) diff --git a/lambdas/shared/tests/test_common/test_clients.py b/lambdas/shared/tests/test_common/test_clients.py index b6944af453..7403241ab4 100644 --- a/lambdas/shared/tests/test_common/test_clients.py +++ b/lambdas/shared/tests/test_common/test_clients.py @@ -1,7 +1,8 @@ import importlib import logging import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock +from unittest.mock import patch import common.clients as clients diff --git a/lambdas/shared/tests/test_common/test_log_decorator.py b/lambdas/shared/tests/test_common/test_log_decorator.py index cdcc613a90..b2d8f2b8f7 100644 --- a/lambdas/shared/tests/test_common/test_log_decorator.py +++ b/lambdas/shared/tests/test_common/test_log_decorator.py @@ -3,11 +3,9 @@ from datetime import datetime from unittest.mock import patch -from common.log_decorator import ( - generate_and_send_logs, - logging_decorator, - send_log_to_firehose, -) +from common.log_decorator import generate_and_send_logs +from common.log_decorator import logging_decorator +from common.log_decorator import send_log_to_firehose class TestLogDecorator(unittest.TestCase): diff --git a/lambdas/shared/tests/test_common/test_s3_reader.py b/lambdas/shared/tests/test_common/test_s3_reader.py index f56cae4bc1..967bd1f503 100644 --- a/lambdas/shared/tests/test_common/test_s3_reader.py +++ b/lambdas/shared/tests/test_common/test_s3_reader.py @@ -1,5 +1,6 @@ import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock +from unittest.mock import patch from common.s3_reader import S3Reader diff --git a/lambdas/shared/tests/test_common/validator/Xtest_application_csv_old.py b/lambdas/shared/tests/test_common/validator/Xtest_application_csv_old.py new file mode 100644 index 0000000000..1b16daa32d --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/Xtest_application_csv_old.py @@ -0,0 +1,34 @@ +# Test application file +import json +import time +from pathlib import Path + +from common.validator.validator import Validator + +# TODO this needs to be converted to unit test with success and fail cases + +parent_folder = Path(__file__).parent +data_folder = parent_folder / "data" +csvFilePath = data_folder / "test_data_ok.csv" # Passes + +dataType = "CSV" + +schemaFilePath = parent_folder / "schemas/test_school_schema.json" + + +start = time.time() + +# get the JSON of the schema, changed to cope with elasticache +with open(schemaFilePath) as JSON: + SchemaFile = json.load(JSON) + +validator = Validator(SchemaFile) +error_report = validator.validate_csv(csvFilePath, False, True, True) + +if len(error_report) > 0: + print(error_report) +else: + print("Validated Successfully") + +end = time.time() +print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/__init__.py b/lambdas/shared/tests/test_common/validator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/tests/test_common/validator/data/test_data.csv b/lambdas/shared/tests/test_common/validator/data/test_data.csv new file mode 100644 index 0000000000..d7c1e87fa6 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/test_data.csv @@ -0,0 +1,29 @@ +NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB|PERSON_GENDER_CODE|PERSON_POSTCODE|DATE_AND_TIME|SITE_CODE|SITE_CODE_TYPE_URI|UNIQUE_ID|UNIQUE_ID_URI|ACTION_FLAG|PERFORMING_PROFESSIONAL_FORENAME|PERFORMING_PROFESSIONAL_SURNAME|RECORDED_DATE|PRIMARY_SOURCE|VACCINATION_PROCEDURE_CODE|VACCINATION_PROCEDURE_TERM|DOSE_SEQUENCE|VACCINE_PRODUCT_CODE|VACCINE_PRODUCT_TERM|VACCINE_MANUFACTURER|BATCH_NUMBER|EXPIRY_DATE|SITE_OF_VACCINATION_CODE|SITE_OF_VACCINATION_TERM|ROUTE_OF_VACCINATION_CODE|ROUTE_OF_VACCINATION_TERM|DOSE_AMOUNT|DOSE_UNIT_CODE|DOSE_UNIT_TERM|INDICATION_CODE|LOCATION_CODE|LOCATION_CODE_TYPE_URI|REDUCE_VALIDATION_CODE|REDUCE_VALIDATION_REASON +"9674963871"|"SABINA"|"GREIR"|"20190131"|"2"|"GU14 6TU"|"20240610T183325"|"J82067"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0001_RSV_v5_RUN_2_CDFDPS-742_valid_dose_1"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"J82067"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9454195417"|"SHAQUILLE"|"KOLBERG"|"20110513"|"1"|"DN17 1SL"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0002_RSV_v5_RUN_2_CDFDPS-742_Dose_seq_05"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9730117489"|"STUART"|"RUDD"|"20011114"|"1"|"RG2 0JA"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0003_RSV_v5_RUN_2_CDFDPS-742_Dose_seq_09"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9694578744"|"JON EDMUND"|"HUGHES"|"19811022"|"1"|"DN37 9NJ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1012_RSV_v5_RUN_2_CDFDPS-742_Detained_Estate"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264283"|"JULIA"|"VERNON"|"19990712"|"2"|"ZZ99 3CZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1013_RSV_v5_RUN_2_CDFDPS-742_ANS_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264275"|"CASEY"|"MUMBY"|"19970921"|"9"|"ZZ99 3WZ"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1014_RSV_v5_RUN_2_CDFDPS-742_ANK_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264267"|"TESSA"|"MORREY"|"19940626"|"2"|"ZZ99 3VZ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1015_RSV_v5_RUN_2_CDFDPS-742_NFA_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244932"|"STAN"|"TEMPLE"|"19990120"|"1"|"IM1 2BT"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1016_RSV_v5_RUN_2_CDFDPS-742_IOM_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244924"|"SAM"|"SELLY"|"20000516"|"1"|"LS26 0TW"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1017_RSV_v5_RUN_2_CDFDPS-742_English_PC_IOM_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244916"|"DAVID"|"DUDLEY"|"19901001"|"1"|"IM1 2BT"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1018_RSV_v5_RUN_2_CDFDPS-742_IOM_PC_IOM_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244908"|"AMOS"|"WESTON"|"19900906"|"1"|"BT9 7GW"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1019_RSV_v5_RUN_2_CDFDPS-742_NI_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244886"|"VERNON"|"MCGURK"|"20010808"|"1"|"BT9 7GW"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1020_RSV_v5_RUN_2_CDFDPS-742_NI_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244878"|"CONRAD"|"FRENCH"|"20040827"|"1"|"EH3 7HA"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1021_RSV_v5_RUN_2_CDFDPS-742_Scottish_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244843"|"ROLAND"|"MACVAY"|"20030221"|"1"|"EH3 7HA"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1022_RSV_v5_RUN_2_CDFDPS-742_Scottish_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244835"|"GEORGE"|"HUNT"|"19900212"|"1"|"SA15 1SU"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1023_RSV_v5_RUN_2_CDFDPS-742_Welsh_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244819"|"LESTER"|"LISTER"|"20021007"|"1"|"SA43 1BU"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1025_RSV_v5_RUN_2_CDFDPS-742_Welsh_GP_Welsh_PC"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244851"|"THOMAS"|"PETTIT"|"20040809"|"1"|"WF3 3AP"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1026_RSV_v5_RUN_2_CDFDPS-742_English_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9467361590"|"WALLIS"|"ADEYEMO"|"20150419"|"1"|"LS6 1BQ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1027_RSV_v5_RUN_2_CDFDPS-742_Merged_patient_Old_NHS_NUMBER"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +|"DINAH"|"MARKIE"|"20110216"|"2"|"DN17 2QZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1028_RSV_v5_RUN_2_CDFDPS-742_Merged_patient_No_NHS_NUMBER"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9446463580"|"ANNABETH"|"BURNS"|"19800303"|"2"|"BL9 7ST"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1029_RSV_v5_RUN_2_CDFDPS-742_Deceased_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9451924108"|"NESSA"|"DENTON"|"20100530"|"0"|"FY8 3HL"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1030_RSV_v5_RUN_2_CDFDPS-742_Sensitive_Y_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9476111771"|"LEAH"|"HAY"|"20160508"|"0"|"CA12 5JT"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1031_RSV_v5_RUN_2_CDFDPS-742_Sensitive_I_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9730117489"|"STUART"|"RUDD"|"20011114"|"0"|"RG2 0JA"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1032_RSV_v5_RUN_2_CDFDPS-742"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9694561140"|"LILLY"|"BUXTON"|"20120810"|"2"|"DN34 4PA"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1033_RSV_v5_RUN_2_CDFDPS-742_MOD_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9475768947"|"IVY"|"ROSSI"|"20160109"|"2"|"BD8 7NS"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1034_RSV_v5_RUN_2_CDFDPS-742_Sensitive_S_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264232"|"DEBRA"|"HESTER"|"19990530"|"2"|"ZZ99 3VZ"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1035_RSV_v5_RUN_2_CDFDPS-742_NFA_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264240"|"ELIAS"|"REID"|"19920313"|"1"|"ZZ99 3WZ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1036_RSV_v5_RUN_2_CDFDPS-742_ANK_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264259"|"RHAN"|"KENYON"|"19930902"|"2"|"ZZ99 3CZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1037_RSV_v5_RUN_2_CDFDPS-742_ANS_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/data/test_data_ok.csv b/lambdas/shared/tests/test_common/validator/data/test_data_ok.csv new file mode 100644 index 0000000000..d7c1e87fa6 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/test_data_ok.csv @@ -0,0 +1,29 @@ +NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB|PERSON_GENDER_CODE|PERSON_POSTCODE|DATE_AND_TIME|SITE_CODE|SITE_CODE_TYPE_URI|UNIQUE_ID|UNIQUE_ID_URI|ACTION_FLAG|PERFORMING_PROFESSIONAL_FORENAME|PERFORMING_PROFESSIONAL_SURNAME|RECORDED_DATE|PRIMARY_SOURCE|VACCINATION_PROCEDURE_CODE|VACCINATION_PROCEDURE_TERM|DOSE_SEQUENCE|VACCINE_PRODUCT_CODE|VACCINE_PRODUCT_TERM|VACCINE_MANUFACTURER|BATCH_NUMBER|EXPIRY_DATE|SITE_OF_VACCINATION_CODE|SITE_OF_VACCINATION_TERM|ROUTE_OF_VACCINATION_CODE|ROUTE_OF_VACCINATION_TERM|DOSE_AMOUNT|DOSE_UNIT_CODE|DOSE_UNIT_TERM|INDICATION_CODE|LOCATION_CODE|LOCATION_CODE_TYPE_URI|REDUCE_VALIDATION_CODE|REDUCE_VALIDATION_REASON +"9674963871"|"SABINA"|"GREIR"|"20190131"|"2"|"GU14 6TU"|"20240610T183325"|"J82067"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0001_RSV_v5_RUN_2_CDFDPS-742_valid_dose_1"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"J82067"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9454195417"|"SHAQUILLE"|"KOLBERG"|"20110513"|"1"|"DN17 1SL"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0002_RSV_v5_RUN_2_CDFDPS-742_Dose_seq_05"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9730117489"|"STUART"|"RUDD"|"20011114"|"1"|"RG2 0JA"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0003_RSV_v5_RUN_2_CDFDPS-742_Dose_seq_09"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9694578744"|"JON EDMUND"|"HUGHES"|"19811022"|"1"|"DN37 9NJ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1012_RSV_v5_RUN_2_CDFDPS-742_Detained_Estate"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264283"|"JULIA"|"VERNON"|"19990712"|"2"|"ZZ99 3CZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1013_RSV_v5_RUN_2_CDFDPS-742_ANS_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264275"|"CASEY"|"MUMBY"|"19970921"|"9"|"ZZ99 3WZ"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1014_RSV_v5_RUN_2_CDFDPS-742_ANK_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264267"|"TESSA"|"MORREY"|"19940626"|"2"|"ZZ99 3VZ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1015_RSV_v5_RUN_2_CDFDPS-742_NFA_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244932"|"STAN"|"TEMPLE"|"19990120"|"1"|"IM1 2BT"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1016_RSV_v5_RUN_2_CDFDPS-742_IOM_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244924"|"SAM"|"SELLY"|"20000516"|"1"|"LS26 0TW"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1017_RSV_v5_RUN_2_CDFDPS-742_English_PC_IOM_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244916"|"DAVID"|"DUDLEY"|"19901001"|"1"|"IM1 2BT"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1018_RSV_v5_RUN_2_CDFDPS-742_IOM_PC_IOM_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244908"|"AMOS"|"WESTON"|"19900906"|"1"|"BT9 7GW"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1019_RSV_v5_RUN_2_CDFDPS-742_NI_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244886"|"VERNON"|"MCGURK"|"20010808"|"1"|"BT9 7GW"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1020_RSV_v5_RUN_2_CDFDPS-742_NI_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244878"|"CONRAD"|"FRENCH"|"20040827"|"1"|"EH3 7HA"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1021_RSV_v5_RUN_2_CDFDPS-742_Scottish_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244843"|"ROLAND"|"MACVAY"|"20030221"|"1"|"EH3 7HA"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1022_RSV_v5_RUN_2_CDFDPS-742_Scottish_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244835"|"GEORGE"|"HUNT"|"19900212"|"1"|"SA15 1SU"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1023_RSV_v5_RUN_2_CDFDPS-742_Welsh_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244819"|"LESTER"|"LISTER"|"20021007"|"1"|"SA43 1BU"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1025_RSV_v5_RUN_2_CDFDPS-742_Welsh_GP_Welsh_PC"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244851"|"THOMAS"|"PETTIT"|"20040809"|"1"|"WF3 3AP"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1026_RSV_v5_RUN_2_CDFDPS-742_English_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9467361590"|"WALLIS"|"ADEYEMO"|"20150419"|"1"|"LS6 1BQ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1027_RSV_v5_RUN_2_CDFDPS-742_Merged_patient_Old_NHS_NUMBER"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +|"DINAH"|"MARKIE"|"20110216"|"2"|"DN17 2QZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1028_RSV_v5_RUN_2_CDFDPS-742_Merged_patient_No_NHS_NUMBER"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9446463580"|"ANNABETH"|"BURNS"|"19800303"|"2"|"BL9 7ST"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1029_RSV_v5_RUN_2_CDFDPS-742_Deceased_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9451924108"|"NESSA"|"DENTON"|"20100530"|"0"|"FY8 3HL"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1030_RSV_v5_RUN_2_CDFDPS-742_Sensitive_Y_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9476111771"|"LEAH"|"HAY"|"20160508"|"0"|"CA12 5JT"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1031_RSV_v5_RUN_2_CDFDPS-742_Sensitive_I_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9730117489"|"STUART"|"RUDD"|"20011114"|"0"|"RG2 0JA"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1032_RSV_v5_RUN_2_CDFDPS-742"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9694561140"|"LILLY"|"BUXTON"|"20120810"|"2"|"DN34 4PA"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1033_RSV_v5_RUN_2_CDFDPS-742_MOD_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9475768947"|"IVY"|"ROSSI"|"20160109"|"2"|"BD8 7NS"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1034_RSV_v5_RUN_2_CDFDPS-742_Sensitive_S_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264232"|"DEBRA"|"HESTER"|"19990530"|"2"|"ZZ99 3VZ"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1035_RSV_v5_RUN_2_CDFDPS-742_NFA_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264240"|"ELIAS"|"REID"|"19920313"|"1"|"ZZ99 3WZ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1036_RSV_v5_RUN_2_CDFDPS-742_ANK_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264259"|"RHAN"|"KENYON"|"19930902"|"2"|"ZZ99 3CZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1037_RSV_v5_RUN_2_CDFDPS-742_ANS_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/data/test_small_nok.csv b/lambdas/shared/tests/test_common/validator/data/test_small_nok.csv new file mode 100644 index 0000000000..3821631506 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/test_small_nok.csv @@ -0,0 +1,4 @@ +NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB +"9674963871"|"SABINA"|"GREIR"|"20190131" +"9454195417"|"SHAQUILLE"|"KOLBERG"|"30110513" +"9730117489"||"RUDD"|"20011114" \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/data/test_small_ok.csv b/lambdas/shared/tests/test_common/validator/data/test_small_ok.csv new file mode 100644 index 0000000000..1df369f0b2 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/test_small_ok.csv @@ -0,0 +1,4 @@ +NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB +"9674963871"|"SABINA"|"GREIR"|"20190131" +"9454195417"|"SHAQUILLE"|"KOLBERG"|"20110513" +"9730117489"|"STUART"|"RUDD"|"20011114" \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/data/vaccination.json b/lambdas/shared/tests/test_common/validator/data/vaccination.json new file mode 100644 index 0000000000..7cca84c675 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/vaccination.json @@ -0,0 +1,170 @@ +{ + "resourceType": "Immunization", + "status": "completed", + "protocolApplied": [ + { + "targetDisease": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "14189004", + "display": "Measles (disorder)" + } + ] + }, + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "36989005", + "display": "Mumps (disorder)" + } + ] + }, + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "36653000", + "display": "Rubella (disorder)" + } + ] + } + ], + "doseNumberPositiveInt": 1 + } + ], + "reasonCode": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "1037351000000105" + } + ] + } + ], + "recorded": "2025-09-15T09:49:38+00:00", + "identifier": [ + { + "value": "48a4dde2-929a-47dc-ae9e-0234d6efc40a", + "system": "https://www.ravs.england.nhs.uk/" + } + ], + "patient": { + "reference": "#Patient1" + }, + "contained": [ + { + "id": "Patient1", + "resourceType": "Patient", + "birthDate": "2013-01-08", + "gender": "male", + "address": [ + { + "postalCode": "SP6 4WE" + } + ], + "identifier": [ + { + "system": "https://fhir.nhs.uk/Id/nhs-number", + "value": "9489927848" + } + ], + "name": [ + { + "family": "Taylor", + "given": ["Laura"] + } + ] + }, + { + "resourceType": "Practitioner", + "id": "Practitioner1", + "name": [ + { + "family": "Taylor", + "given": ["Anthony"] + } + ] + } + ], + "vaccineCode": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "42223111000001107", + "display": "Influenza Tetra MYL vaccine suspension for injection 0.5ml pre-filled syringes (Mylan)" + } + ] + }, + "manufacturer": { + "display": "Mylan" + }, + "expirationDate": "2025-09-20", + "lotNumber": "BN40379485850", + "extension": [ + { + "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "956951000000104", + "display": "RSV vaccination in pregnancy (procedure)" + } + ] + } + } + ], + "occurrenceDateTime": "2025-09-11T07:14:25+00:00", + "primarySource": true, + "site": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "368209003", + "display": "Right arm" + } + ] + }, + "route": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "1210999013", + "display": "Intradermal use" + } + ] + }, + "doseQuantity": { + "value": 0.3, + "unit": "Inhalation - unit of product usage", + "system": "http://snomed.info/sct", + "code": "2622896019" + }, + "performer": [ + { + "actor": { + "type": "Organization", + "identifier": { + "system": "https://fhir.nhs.uk/Id/ods-organization-code", + "value": "RJ1" + } + } + }, + { + "actor": { + "reference": "#Practitioner1" + } + } + ], + "location": { + "identifier": { + "value": "RJC02", + "system": "https://fhir.nhs.uk/Id/ods-organization-code" + } + }, + "id": "976ccbf1-5ced-4541-8664-3953bf2ff026" +} diff --git a/lambdas/shared/tests/test_common/validator/data/vaccination2.json b/lambdas/shared/tests/test_common/validator/data/vaccination2.json new file mode 100644 index 0000000000..b7e56a6d24 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/vaccination2.json @@ -0,0 +1,113 @@ +{ + "resourceType": "Immunization", + "id": "d11c69d8-7a50-4a54-a848-7648121e995f", + "contained": [ + { + "resourceType": "Patient", + "id": "Pat1", + "identifier": [ + { + "system": "https://fhir.nhs.uk/Id/nhs-number", + "value": "9000000009" + } + ], + "name": [ + { + "family": "Taylor", + "given": ["Sarah"] + } + ], + "gender": "unknown", + "birthDate": "1965-02-28", + "address": [ + { + "postalCode": "EC1A 1BB" + } + ] + } + ], + "extension": [ + { + "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "1324681000000101", + "display": "Administration of first dose of severe acute respiratory syndrome coronavirus 2 vaccine (procedure)" + } + ] + } + } + ], + "status": "completed", + "vaccineCode": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "39114911000001105", + "display": "COVID-19 Vaccine Vaxzevria (ChAdOx1 S [recombinant]) not less than 2.5x100,000,000 infectious units/0.5ml dose suspension for injection multidose vials (AstraZeneca UK Ltd) (product)" + } + ] + }, + "patient": { + "reference": "#Pat1" + }, + "occurrenceDateTime": "2021-02-07T13:28:17.271+00:00", + "recorded": "2021-02-07", + "primarySource": true, + "manufacturer": { + "display": "AstraZeneca Ltd" + }, + "location": { + "identifier": { + "value": "X99999", + "system": "https://fhir.nhs.uk/Id/ods-organization-code" + } + }, + "lotNumber": "4120Z001", + "expirationDate": "2021-07-02", + "doseQuantity": { + "value": 0.5, + "unit": "ml", + "system": "http://snomed.info/sct", + "code": "258773002" + }, + "performer": [ + { + "actor": { + "type": "Organization", + "identifier": { + "system": "https://fhir.nhs.uk/Id/ods-organization-code", + "value": "B0C4P" + } + } + } + ], + "reasonCode": [ + { + "coding": [ + { + "code": "443684005", + "system": "http://snomed.info/sct" + } + ] + } + ], + "protocolApplied": [ + { + "targetDisease": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "840539006", + "display": "Disease caused by severe acute respiratory syndrome coronavirus 2 (disorder)" + } + ] + } + ], + "doseNumberPositiveInt": 1 + } + ] +} diff --git a/lambdas/shared/tests/test_common/validator/schemas/Schema.json b/lambdas/shared/tests/test_common/validator/schemas/Schema.json new file mode 100644 index 0000000000..3b6bb98028 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/schemas/Schema.json @@ -0,0 +1,189 @@ +{ + "id": "01K5EGR0C85TPNZT71MJ10VKYY", + "schemaName": "Base Vaccination Validation", + "version": 1.0, + "releaseDate": "2024-07-17T00:00:00.000Z", + "expressions": [ + { + "expressionId": "01K5EGR0C7Y1WJ0BC803SQDWK4", + "fieldNameFHIR": "contained|#:Patient|identifier|#:https://fhir.nhs.uk/Id/nhs-number|value", + "fieldNameFlat": "NHS_NUMBER", + "errorLevel": 0, + "expression": { + "expressionName": "NHS Number Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "01K5EGR0C7QCEJMWH1R4MBPGQA", + "fieldNameFHIR": "contained|#:Patient|name|#:official|given|0", + "fieldNameFlat": "PERSON_FORENAME", + "errorLevel": 0, + "expression": { + "expressionName": "Person Forname Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C7RRG9F6FVHJ8HE4QX", + "fieldNameFHIR": "contained|#:Patient|name|#:official|family", + "fieldNameFlat": "PERSON_SURNAME", + "errorLevel": 0, + "parentExpression": "01K5EGR0C7Y1WJ0BC803SQDWK4", + "expression": { + "expressionName": "Person Surname Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C8WSJ9N8RV6T8RJJ4W", + "fieldNameFHIR": "performer|#:Organization|actor|identifier|value", + "fieldNameFlat": "SITE_CODE", + "errorLevel": 1, + "expression": { + "expressionName": "Organisation Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C8M1MVNKTQCE6MSG68", + "fieldNameFHIR": "performer|#:Organization|actor|identifier|value", + "fieldNameFlat": "SITE_CODE", + "errorLevel": 0, + "expression": { + "expressionName": "Organisation Look Up Check", + "expressionType": "KEYCHECK", + "expressionRule": "Organisation" + }, + "errorGroup": "consistency" + }, + { + "expressionId": "01K5EGR0C8SDQBTNCEP8TJNCCW", + "fieldNameFHIR": "contained|#:Practitioner|name|0|given|0", + "fieldNameFlat": "PERFORMING_PROFESSIONAL_FORENAME", + "errorLevel": 1, + "expression": { + "expressionName": "Practitioner Forename Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C822RC96QJRR2YX18S", + "fieldNameFHIR": "contained|#:Practitioner|name|0|family", + "fieldNameFlat": "PERFORMING_PROFESSIONAL_SURNAME", + "errorLevel": 1, + "expression": { + "expressionName": "Practitioner Surname Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C84CCDRR0VFSWQNFZP", + "fieldNameFHIR": "primarySource", + "fieldNameFlat": "PRIMARY_SOURCE", + "errorLevel": 1, + "expression": { + "expressionName": "Primary Source Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C8VCX8FX8A7ZV7MQ5J", + "fieldNameFHIR": "extension|0|valueCodeableConcept|coding|0|code", + "fieldNameFlat": "VACCINATION_PROCEDURE_CODE", + "errorLevel": 0, + "expression": { + "expressionName": "Procedure Code Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C85HY6MDNN6TTR1K48", + "fieldNameFHIR": "extension|0|valueCodeableConcept|coding|0|display", + "fieldNameFlat": "VACCINATION_PROCEDURE_TERM", + "errorLevel": 1, + "expression": { + "expressionName": "Procedure Term Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C84DDGW567G14AYBC6", + "fieldNameFHIR": "protocolApplied|0|doseNumberPositiveInt", + "fieldNameFlat": "DOSE_SEQUENCE", + "errorLevel": 1, + "expression": { + "expressionName": "Dose Sequence Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C8W3HXFYR80ENW73SS", + "fieldNameFHIR": "vaccineCode|coding|#:http://snomed.info/sct|code", + "fieldNameFlat": "VACCINE_PRODUCT_CODE", + "errorLevel": 0, + "expression": { + "expressionName": "Produce Code Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C885N7MMW2J5JKHTT2", + "fieldNameFHIR": "vaccineCode|coding|#:http://snomed.info/sct|display", + "fieldNameFlat": "VACCINE_PRODUCT_TERM", + "errorLevel": 1, + "expression": { + "expressionName": "Produce Term Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C86XN0AF0M9DJYFGCD", + "fieldNameFHIR": "manufacturer|display", + "fieldNameFlat": "VACCINE_MANUFACTURER", + "errorLevel": 0, + "expression": { + "expressionName": "Manufacturer Display Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "01K5EGR0C89M4CV68B7XAKDCHG", + "fieldNameFHIR": "lotNumber", + "fieldNameFlat": "BATCH_NUMBER", + "errorLevel": 0, + "expression": { + "expressionName": "Batch Number Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + } + ] +} diff --git a/lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json b/lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json new file mode 100644 index 0000000000..5fdc279f56 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json @@ -0,0 +1,164 @@ +{ + "id": "01K5EGR0C85TPNZT71MJ10VKAA", + "schemaName": "School Attendance Data Validation", + "version": 1.0, + "releaseDate": "2024-07-17T00:00:00.000Z", + "expressions": [ + { + "expressionId": "check1", + "fieldNameCSV": "academic_year", + "fieldNameFlat": "ACADEMIC_YEAR", + "errorLevel": 0, + "expression": { + "expressionName": "Academic Year Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check2", + "fieldNameCSV": "time_period", + "fieldNameFlat": "TIME_PERIOD", + "errorLevel": 0, + "expression": { + "expressionName": "Time Period Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check3", + "fieldNameCSV": "country_code", + "fieldNameFlat": "COUNTRY_CODE", + "errorLevel": 0, + "expression": { + "expressionName": "Country Code Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check4", + "fieldNameCSV": "country_name", + "fieldNameFlat": "COUNTRY_NAME", + "errorLevel": 1, + "expression": { + "expressionName": "Country Name Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check5", + "fieldNameCSV": "new_la_code", + "fieldNameFlat": "NEW_LA_CODE", + "errorLevel": 0, + "expression": { + "expressionName": "LA Code Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check6", + "fieldNameCSV": "la_name", + "fieldNameFlat": "LA_NAME", + "errorLevel": 1, + "expression": { + "expressionName": "LA Name Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check7", + "fieldNameCSV": "school_type", + "fieldNameFlat": "SCHOOL_TYPE", + "errorLevel": 0, + "expression": { + "expressionName": "School Type Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check8", + "fieldNameCSV": "num_schools", + "fieldNameFlat": "NUM_SCHOOLS", + "errorLevel": 1, + "expression": { + "expressionName": "Number of Schools Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check9", + "fieldNameCSV": "enrolments", + "fieldNameFlat": "ENROLMENTS", + "errorLevel": 1, + "expression": { + "expressionName": "Enrolments Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check10", + "fieldNameCSV": "overall_attendance", + "fieldNameFlat": "OVERALL_ATTENDANCE", + "errorLevel": 0, + "expression": { + "expressionName": "Overall Attendance Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check11", + "fieldNameCSV": "overall_absence", + "fieldNameFlat": "OVERALL_ABSENCE", + "errorLevel": 0, + "expression": { + "expressionName": "Overall Absence Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check12", + "fieldNameCSV": "authorised_absence", + "fieldNameFlat": "AUTHORISED_ABSENCE", + "errorLevel": 1, + "expression": { + "expressionName": "Authorised Absence Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check13", + "fieldNameCSV": "unauthorised_absence", + "fieldNameFlat": "UNAUTHORISED_ABSENCE", + "errorLevel": 1, + "expression": { + "expressionName": "Unauthorised Absence Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + } + ] +} diff --git a/lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json b/lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json new file mode 100644 index 0000000000..1b14f2c801 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json @@ -0,0 +1,71 @@ +{ + "id": "No 2", + "schemaName": "Small Test Validation", + "version": 1.0, + "releaseDate": "2024-07-17T00:00:00.000Z", + "expressions": [ + { + "expressionId": "check_1", + "fieldNameCSV": "NHS_NUMBER", + "fieldNameFlat": "NHS_NUMBER", + "errorLevel": 0, + "expression": { + "expressionName": "NHS Number Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check_2", + "fieldNameCSV": "PERSON_FORENAME", + "fieldNameFlat": "PERSON_FORENAME", + "errorLevel": 0, + "expression": { + "expressionName": "Person Forname Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check_3", + "fieldNameCSV": "PERSON_SURNAME", + "fieldNameFlat": "PERSON_SURNAME", + "errorLevel": 0, + "parentExpression": "check_1", + "expression": { + "expressionName": "Person Surname Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check_3", + "fieldNameCSV": "PERSON_DOB", + "fieldNameFlat": "PERSON_DOB", + "errorLevel": 0, + "parentExpression": "check_1", + "expression": { + "expressionName": "Person DOB Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check_4", + "fieldNameCSV": "PERSON_DOB", + "fieldNameFlat": "PERSON_DOB", + "errorLevel": 0, + "parentExpression": "check_1", + "expression": { + "expressionName": "Person DOB Valid Date Check", + "expressionType": "DATE", + "expressionRule": "" + }, + "errorGroup": "completeness" + } + ] +} diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv_row.py b/lambdas/shared/tests/test_common/validator/test_application_csv_row.py new file mode 100644 index 0000000000..6a9e38d684 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_application_csv_row.py @@ -0,0 +1,49 @@ +# Test application file +import json +import unittest +from pathlib import Path + +from common.validator.validator import Validator + +CSV_HEADER = ( + "academic_year,time_period,time_identifier,geographic_level," + "country_code,country_name,region_code,region_name,new_la_code,la_name," + "old_la_code,school_type,num_schools,enrolments,present_sessions,overall_attendance," + "approved_educational_activity,overall_absence,authorised_absence,unauthorised_absence," + "late_sessions,possible_sessions,reason_present_am,reason_present_pm,reason_present," + "reason_l_present_late_before_registers_closed" +) + +schema_data_folder = Path(__file__).parent / "schemas" +schemaFilePath = schema_data_folder / "test_school_schema.json" + + +class TestValidator(unittest.TestCase): + def setUp(self): + with open(schemaFilePath) as JSON: + SchemaFile = json.load(JSON) + self.validator = Validator(SchemaFile) + + def test_run_validation_csv_row_success(self): + good_row = ( + "202223,202223,Spring term,Local authority,E92000001,England,E12000004," + "East Midlands,E06000016,Leicester,856,Primary,66,23057,2367094," + "2380687,13593,166808,99826,66982,34090,2547495,1157575,1180365,2337940,29154" + ) + error_report = self.validator.validate_csv_row(good_row, CSV_HEADER, True, True, True) + self.assertTrue(error_report == []) + + def test_run_validation_csv_row_failure(self): + # empty time_period + bad_row = ( + "202223,,Spring term,Local authority,E92000001,England,E12000004," + "East Midlands,E06000016,Leicester,856,Primary,66,23057,2367094," + "2380687,13593,166808,99826,66982,34090,2547495,1157575,1180365,2337940,29154" + ) + error_report = self.validator.validate_csv_row(bad_row, CSV_HEADER, True, True, True) + self.assertTrue(len(error_report) > 0) + error = error_report[0] + self.assertEqual(error.id, "check2") + self.assertEqual(error.message, "Value not empty failure") + self.assertEqual(error.name, "Time Period Not Empty Check") + self.assertEqual(error.details, "Value is empty, not as expected") diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv_small.py b/lambdas/shared/tests/test_common/validator/test_application_csv_small.py new file mode 100644 index 0000000000..44b9d00756 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_application_csv_small.py @@ -0,0 +1,26 @@ +# Test application file +import json +import unittest +from pathlib import Path + +from common.validator.validator import Validator + + +class TestValidator(unittest.TestCase): + def setUp(self): + self.parent_folder = Path(__file__).parent + schema_file_path = self.parent_folder / "schemas/test_small_schema.json" + with open(schema_file_path) as JSON: + self.schema = json.load(JSON) + + def test_run_validation_csv_success(self): + good_file_path = self.parent_folder / "data/test_small_ok.csv" + validator = Validator(self.schema) + error_report = validator.validate_csv(good_file_path, False, True, True) + self.assertTrue(error_report == []) + + def test_run_validation_csv_fails(self): + bad_file_path = self.parent_folder / "data/test_small_nok.csv" + validator = Validator(self.schema) + error_report = validator.validate_csv(bad_file_path, False, True, True) + self.assertTrue(error_report != []) diff --git a/lambdas/shared/tests/test_common/validator/test_application_fhir.py b/lambdas/shared/tests/test_common/validator/test_application_fhir.py new file mode 100644 index 0000000000..11efba5c25 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_application_fhir.py @@ -0,0 +1,47 @@ +# Test application file +import json +import time +import unittest +from pathlib import Path + +from common.validator.validator import Validator + +# TODO this needs success and fail cases + + +class TestApplication(unittest.TestCase): + def setUp(self): + validation_folder = Path(__file__).parent + self.FHIRFilePath = validation_folder / "data/vaccination2.json" + self.schemaFilePath = validation_folder / "schemas/schema.json" + + def test_validation(self): + start = time.time() + + # get the JSON of the schema, changed to cope with elasticache + with open(self.schemaFilePath) as JSON: + SchemaFile = json.load(JSON) + + validator = Validator(SchemaFile) # FHIR File Path not needed + error_list = validator.validate_fhir(self.FHIRFilePath, True, True, True) + error_report = validator.build_error_report("25a8cc4d-1875-4191-ac6d-2d63a0ebc64b") # include eventID if known + + failed_validation = validator.has_validation_failed() + + if len(error_list) > 0: + print(error_list) + else: + print("Validated Successfully") + + print("--------------------------------------------------------------------") + print(error_report) + print("--------------------------------------------------------------------") + + if failed_validation: + print("Validation failed due to a critical validation failure...") + else: + print("Validation Successful, see reports for details") + + end = time.time() + print("Time Taken : ") + print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/test_expression_checker.py b/lambdas/shared/tests/test_common/validator/test_expression_checker.py new file mode 100644 index 0000000000..8833a1fc6d --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_expression_checker.py @@ -0,0 +1,65 @@ +import unittest +from unittest.mock import MagicMock +from unittest.mock import patch + +import common.validator.enums.exception_messages as ExceptionMessages +from common.validator.expression_checker import ExpressionChecker + +# TODO this needs to be expanded to cover all expression types + + +class TestExpressionChecker(unittest.TestCase): + def setUp(self): + self.MockLookUpData = patch("common.validator.expression_checker.LookUpData").start() + self.MockKeyData = patch("common.validator.expression_checker.KeyData").start() + + self.mock_summarise = MagicMock() + self.mock_report_exception = MagicMock() + self.mock_data_parser = MagicMock() + + self.expression_checker = ExpressionChecker( + self.mock_data_parser, self.mock_summarise, self.mock_report_exception + ) + + def tearDown(self): + patch.stopall() + + def test_validate_datetime_valid(self): + result = self.expression_checker.validate_expression( + "DATETIME", rule="", field_name="timestamp", field_value="2022-01-01T12:00:00", row={} + ) + self.assertEqual( + result.message, "Unexpected exception [ValueError]: Invalid isoformat string: '2022-01-01T12:00:00'" + ) + self.assertEqual(result.code, ExceptionMessages.UNEXPECTED_EXCEPTION) + self.assertEqual(result.field, "timestamp") + + def test_validate_uuid_valid(self): + result = self.expression_checker.validate_expression( + "UUID", rule="", field_name="id", field_value="550e8400-e29b-41d4-a716-446655440000", row={} + ) + self.assertTrue(result is None) + + def test_validate_integer_invalid(self): + result = self.expression_checker.validate_expression( + "INT", rule="", field_name="age", field_value="hello world", row={} + ) + self.assertEqual(result.code, ExceptionMessages.UNEXPECTED_EXCEPTION) + self.assertEqual(result.field, "age") + self.assertIn("invalid literal for int()", result.message) + + def test_validate_in_array(self): + # Mock data_parser.get_key_values + self.mock_data_parser.get_key_values.return_value = ["val1", "val2"] + + result = self.expression_checker.validate_expression( + "INARRAY", rule="", field_name="some_field", field_value="val2", row={} + ) + self.assertEqual(result.message, "Value not in array check failed") + self.assertEqual(result.field, "some_field") + + def test_validate_expression_type_not_found(self): + result = self.expression_checker.validate_expression( + "UNKNOWN", rule="", field_name="field", field_value="value", row={} + ) + self.assertIn("Schema expression not found", result) diff --git a/lambdas/shared/tests/test_common/validator/test_parser.py b/lambdas/shared/tests/test_common/validator/test_parser.py new file mode 100644 index 0000000000..cc4834e661 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_parser.py @@ -0,0 +1,28 @@ +# Test application file +import unittest +from pathlib import Path + +from common.validator.parsers.fhir_parser import FHIRParser + + +class TestParse(unittest.TestCase): + def setUp(self): + self.fhir_data_folder = Path(__file__).parent / "data" + + def test_parse_fhir_key_exists(self): + fhirFilePath = self.fhir_data_folder / "vaccination.json" + + fhir_parser = FHIRParser() + fhir_parser.parse_fhir_file(fhirFilePath) + my_value = fhir_parser.get_key_value("vaccineCode|coding|0|code") + self.assertEqual(my_value, ["42223111000001107"]) + + def test_parse_fhir_key_not_exists(self): + fhirFilePath = self.fhir_data_folder / "vaccination.json" + + fhir_parser = FHIRParser() + fhir_parser.parse_fhir_file(fhirFilePath) + my_value = fhir_parser.get_key_value("vaccineCode|coding|1") + self.assertEqual(my_value, [""]) + my_value = fhir_parser.get_key_value("vaccineCode|coding|1|codes") + self.assertEqual(my_value, [""]) diff --git a/lambdas/shared/tests/test_common/validator/test_validator.py b/lambdas/shared/tests/test_common/validator/test_validator.py new file mode 100644 index 0000000000..a32cbaea04 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_validator.py @@ -0,0 +1,103 @@ +import unittest +from unittest.mock import MagicMock +from unittest.mock import patch + +import common.validator.enums.error_levels as ErrorLevels +from common.validator.record_error import ErrorReport +from common.validator.validator import DataType +from common.validator.validator import Validator + + +class TestValidator(unittest.TestCase): + def setUp(self): + self.mock_expression_checker = patch("common.validator.validator.ExpressionChecker").start() + self.mock_schema_parser = patch("common.validator.validator.SchemaParser").start() + self.mock_fhir_parser = patch("common.validator.validator.FHIRParser").start() + self.mock_csv_parser = patch("common.validator.validator.CSVParser").start() + self.mock_csv_line_parser = patch("common.validator.validator.CSVLineParser").start() + + def tearDown(self): + patch.stopall() + + def test_run_validation_csv(self): + # Setup mocks + self.mock_csv_parser.parse_csv_file.return_value = None + self.mock_schema_parser.parse_schema.return_value = None + self.mock_schema_parser.getExpressions.return_value = [ + { + "fieldNameCSV": "test_field", + "expressionId": "exp1", + "errorLevel": 1, + "expression": {"expressionName": "Test", "expressionType": "type", "expressionRule": "rule"}, + "errorGroup": "group", + } + ] + self.mock_expression_checker.return_value.validate_expression.return_value = None + + validator = Validator() + result = validator.validate_csv("file.csv") + self.assertIsInstance(result, list) + self.assertEqual(len(result), 0) + + def test_run_validation_fhir_exception(self): + # Simulate exception in FHIRParser + self.mock_fhir_parser.side_effect = Exception("FHIR error") + validator = Validator() + result = validator.validate_fhir("file.fhir") + self.assertIsInstance(result, list) + self.assertEqual(result[0].code, 0) + self.assertIn("Data Parser Unexpected exception", result[0].message) + + def test_run_validation_schema_exception(self): + # Simulate exception in SchemaParser + self.mock_fhir_parser.return_value.parse_fhir_file.return_value = None + self.mock_schema_parser.side_effect = Exception("Schema error") + validator = Validator() + result = validator.validate_fhir("file.fhir") + self.assertIsInstance(result, list) + self.assertEqual(result[0].code, 0) + self.assertIn("Schema Parser Unexpected exception", result[0].message) + + def test_run_validation_expression_checker_exception(self): + # Simulate exception in ExpressionChecker + self.mock_fhir_parser.return_value.parse_fhir_file.return_value = None + self.mock_schema_parser.return_value.parse_schema.return_value = None + self.mock_schema_parser.return_value.get_expressions.return_value = [] + self.mock_expression_checker.side_effect = Exception("ExpressionChecker error") + validator = Validator() + result = validator.validate_fhir("file.fhir") + self.assertIsInstance(result, list) + self.assertEqual(result[0].code, 0) + self.assertIn("Expression Checker Unexpected exception", result[0].message) + + def test_run_validation_expression_getter_exception(self): + # Simulate exception in getExpressions + self.mock_fhir_parser.return_value.parse_fhir_file.return_value = None + self.mock_schema_parser.return_value.parse_schema.return_value = None + self.mock_schema_parser.return_value.get_expressions.side_effect = Exception("Expressions error") + self.mock_expression_checker.return_value = MagicMock() + validator = Validator() + result = validator.validate_fhir("file.fhir") + self.assertIsInstance(result, list) + self.assertEqual(result[0].code, 0) + self.assertIn("Expression Getter Unexpected exception", result[0].message) + + @patch("common.validator.validator.DQReporter") + def test_build_error_report(self, mock_dq_reporter): + mock_dq_reporter.return_value.generate_error_report.return_value = {"report": "ok"} + validator = Validator(data_type=DataType.CSV, filepath="file.csv") + validator.data_parser = MagicMock() + validator.data_parser.get_key_single_value.return_value = "2023-01-01" + result = validator.build_error_report("event123") + self.assertEqual(result, {"report": "ok"}) + mock_dq_reporter.return_value.generate_error_report.assert_called_once() + + def test_has_validation_failed_true(self): + v = Validator(data_type=DataType.CSV) + v.error_records.append(ErrorReport(error_level=ErrorLevels.CRITICAL_ERROR)) # Assuming 2 is CRITICAL_ERROR + self.assertTrue(v.has_validation_failed()) + + def test_has_validation_failed_false(self): + v = Validator(data_type=DataType.CSV) + v.error_records.append(ErrorReport(error_level=ErrorLevels.WARNING)) + self.assertFalse(v.has_validation_failed())