From e7779dc7fb46b948eed4e3c80822bcb83a80ce14 Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Wed, 8 Oct 2025 19:00:13 +0100 Subject: [PATCH 01/13] linting --- e2e_batch/Makefile | 4 +- .../common/validator/enums/error_levels.py | 10 + .../validator/enums/exception_messages.py | 31 + .../src/common/validator/lookup/key_data.py | 107 +++ .../common/validator/lookup/lookup_data.py | 105 +++ .../validator/parsers/csv_line_parser.py | 20 + .../common/validator/parsers/csv_parser.py | 26 + .../common/validator/parsers/fhir_parser.py | 108 +++ .../common/validator/parsers/schema_parser.py | 26 + .../common/validator/reporter/dq_reporter.py | 64 ++ .../validation_expression_checker.py | 653 ++++++++++++++++++ .../shared/src/common/validator/validator.py | 187 +++++ 12 files changed, 1340 insertions(+), 1 deletion(-) create mode 100644 lambdas/shared/src/common/validator/enums/error_levels.py create mode 100644 lambdas/shared/src/common/validator/enums/exception_messages.py create mode 100644 lambdas/shared/src/common/validator/lookup/key_data.py create mode 100644 lambdas/shared/src/common/validator/lookup/lookup_data.py create mode 100644 lambdas/shared/src/common/validator/parsers/csv_line_parser.py create mode 100644 lambdas/shared/src/common/validator/parsers/csv_parser.py create mode 100644 lambdas/shared/src/common/validator/parsers/fhir_parser.py create mode 100644 lambdas/shared/src/common/validator/parsers/schema_parser.py create mode 100644 lambdas/shared/src/common/validator/reporter/dq_reporter.py create mode 100644 lambdas/shared/src/common/validator/validation_expression_checker.py create mode 100644 lambdas/shared/src/common/validator/validator.py diff --git a/e2e_batch/Makefile b/e2e_batch/Makefile index eb97587d05..839037ef10 100644 --- a/e2e_batch/Makefile +++ b/e2e_batch/Makefile @@ -1,4 +1,6 @@ -include .env +.PHONY: test + test: - ENVIRONMENT=$(ENVIRONMENT) poetry run python -m unittest -v -c \ No newline at end of file + ENVIRONMENT=$(ENVIRONMENT) poetry run python -m unittest -v -c \ No newline at end of file diff --git a/lambdas/shared/src/common/validator/enums/error_levels.py b/lambdas/shared/src/common/validator/enums/error_levels.py new file mode 100644 index 0000000000..c6f8f51e0e --- /dev/null +++ b/lambdas/shared/src/common/validator/enums/error_levels.py @@ -0,0 +1,10 @@ +# all error Levels +CRITICAL_ERROR = 0 +WARNING = 1 +NOTIFICATION = 2 + +MESSAGES = { + CRITICAL_ERROR: 'Critical Validation Error [%s]: %s', + WARNING: 'Non-Critical Validation Error [%s]: %s', + NOTIFICATION: 'Quality Issue Found [%s]: %s', + } diff --git a/lambdas/shared/src/common/validator/enums/exception_messages.py b/lambdas/shared/src/common/validator/enums/exception_messages.py new file mode 100644 index 0000000000..770f25c6db --- /dev/null +++ b/lambdas/shared/src/common/validator/enums/exception_messages.py @@ -0,0 +1,31 @@ +# all exceptions and messgaes +UNEXPECTED_EXCEPTION = 0 +VALUE_CHECK_FAILED = 1 +HEADER_CHECK_FAILED = 2 +RECORD_LENGTH_CHECK_FAILED = 3 +VALUE_PREDICATE_FALSE = 4 +RECORD_CHECK_FAILED = 5 +RECORD_PREDICATE_FALSE = 6 +UNIQUE_CHECK_FAILED = 7 +ASSERT_CHECK_FAILED = 8 +FINALLY_ASSERT_CHECK_FAILED = 9 +PARSING_ERROR = 10 +PARENT_FAILED = 11 +KEY_CHECK_FAILED = 12 + + +MESSAGES = { + UNEXPECTED_EXCEPTION: 'Unexpected exception [%s]: %s', + VALUE_CHECK_FAILED: 'Value check failed.', + HEADER_CHECK_FAILED: 'Header check failed.', + RECORD_LENGTH_CHECK_FAILED: 'Record length check failed.', + RECORD_CHECK_FAILED: 'Record check failed.', + VALUE_PREDICATE_FALSE: 'Value predicate returned false.', + RECORD_PREDICATE_FALSE: 'Record predicate returned false.', + UNIQUE_CHECK_FAILED: 'Unique check failed.', + ASSERT_CHECK_FAILED: 'Assertion check failed.', + FINALLY_ASSERT_CHECK_FAILED: 'Final assertion check failed.', + PARSING_ERROR: 'Failed to parse data correctly.', + PARENT_FAILED: 'The parent expression failed to validate', + KEY_CHECK_FAILED: 'Value could not be found in the Key list' + } diff --git a/lambdas/shared/src/common/validator/lookup/key_data.py b/lambdas/shared/src/common/validator/lookup/key_data.py new file mode 100644 index 0000000000..4af7b2cad5 --- /dev/null +++ b/lambdas/shared/src/common/validator/lookup/key_data.py @@ -0,0 +1,107 @@ +#--------------------------------------------------------------------------------------------------------- +#main conversion lookup + +class KeyData: + #data settings + Procedure = ['956951000000104'] + + Organisation = ['RJ1', 'RJC02'] + + Site = ['368208006', '279549004', '74262004', '368209003', '723979003', '61396006', '723980000', '11207009', '420254004'] + + Route = [ '54471007', + '372449004', + '372450004', + '372451000', + '372452007', + '404820008', + '18246711000001107', + '372453002', + '372454008', + '127490009', + '372457001', + '9191401000001100', + '18682911000001103', + '10334211000001103', + '718329006', + '18679011000001101', + '34777511000001106', + '372458006', + '58100008', + '12130007', + '372459003', + '418821007', + '372460008', + '372461007', + '420719007', + '19537211000001108', + '372463005', + '372464004', + '372465003', + '448077001', + '38233211000001106', + '372466002', + '372467006', + '78421000', + '255559005', + '372468001', + '417255000', + '38239002', + '372469009', + '372470005', + '418586008', + '72607000', + '447122006', + '62226000', + '47625008', + '420287000', + '372471009', + '418401004', + '21856811000001103', + '127491008', + '9907001000001103', + '46713006', + '127492001', + '418730005', + '54485002', + '26643006', + '372473007', + '10547007', + '225691000001105', + '9191501000001101', + '372474001', + '39338211000001108', + '3323001000001107', + '372475000', + '11478901000001102', + '419762003', + '39337511000001107', + '37161004', + '11564311000001109', + '418321004', + '3594011000001102', + '372476004', + '34206005', + '37839007', + '419874009', + '11564211000001101', + '33770711000001104', + '6064005', + '45890007', + '11479001000001107', + '404815008', + '90028008', + '16857009'] + + # Look up the term for the code + def findKey(self, KeySource, fieldValue): + try: + match KeySource: + case "Procedure": return fieldValue in self.Procedure + case "Organisation": fieldValue in self.Organisation + case "Site": fieldValue in self.Site + case "Route": fieldValue in self.Route + case _: return False + except: + return False + return False diff --git a/lambdas/shared/src/common/validator/lookup/lookup_data.py b/lambdas/shared/src/common/validator/lookup/lookup_data.py new file mode 100644 index 0000000000..b19dd72e2a --- /dev/null +++ b/lambdas/shared/src/common/validator/lookup/lookup_data.py @@ -0,0 +1,105 @@ +#--------------------------------------------------------------------------------------------------------- +#main conversion lookup + +class LookUpData: + #data settings + allData = { '368208006':'Left upper arm structure', + '279549004':'Nasal cavity structure', + '74262004':'Oral cavity structure', + '368209003':'Right upper arm structure', + '723979003':'Structure of left buttock', + '61396006':'Structure of left thigh', + '723980000':'Structure of right buttock', + '11207009':'Structure of right thigh', + '420254004':'Body cavity', + '54471007':'Buccal', + '372449004':'Dental', + '372450004':'Endocervical', + '372451000':'Endosinusial', + '372452007':'Endotracheopulmonary', + '404820008':'Epidural', + '18246711000001107':'Epilesional', + '372453002':'Extraamniotic', + '372454008':'Gastroenteral', + '127490009':'Gastrostomy route', + '372457001':'Gingival', + '9191401000001100':'Haemodiafiltration', + '18682911000001103':'Haemodialysis', + '10334211000001103':'Haemofiltration', + '718329006':'Infiltration', + '18679011000001101':'Inhalation', + '34777511000001106':'Intestinal use', + '372458006':'Intraamniotic', + '58100008':'Intraarterial', + '12130007':'Intraarticular', + '372459003':'Intrabursal', + '418821007':'Intracameral use', + '372460008':'Intracardiac', + '372461007':'Intracavernous', + '420719007':'Intracerebroventricular', + '19537211000001108':'Intracervical route', + '372463005':'Intracoronary', + '372464004':'Intradermal', + '372465003':'Intradiscal', + '448077001':'Intraepidermal', + '38233211000001106':'Intraglandular', + '372466002':'Intralesional', + '372467006':'Intralymphatic', + '78421000':'Intramuscular', + '255559005':'Intramuscular', + '372468001':'Intraocular', + '417255000':'Intraosseous', + '38239002':'Intraperitoneal', + '372469009':'Intrapleural', + '372470005':'Intrasternal', + '418586008':'Intratendinous route', + '72607000':'Intrathecal', + '447122006':'Intratumoral', + '62226000':'Intrauterine', + '47625008':'Intravenous', + '420287000':'Intraventricular cardiac', + '372471009':'Intravesical', + '418401004':'Intravitreal', + '21856811000001103':'Iontophoresis', + '127491008':'Jejunostomy route', + '9907001000001103':'Line lock', + '46713006':'Nasal', + '127492001':'Nasogastric route', + '418730005':'Nasojejunal route', + '54485002':'Ophthalmic route', + '26643006':'Oral', + '372473007':'Oromucosal', + '10547007':'Otic', + '225691000001105':'PEG tube route', + '9191501000001101':'Percutaneous', + '372474001':'Periarticular', + '39338211000001108':'Peribulbar ocular', + '3323001000001107':'Pericardial route', + '372475000':'Perineural', + '11478901000001102':'Periosseous', + '419762003':'Peritendinous route', + '39337511000001107':'Peritumoral', + '37161004':'Rectal', + '11564311000001109':'Regional perfusion', + '418321004':'Retrobulbar route', + '3594011000001102':'Route of administration not applicable', + '372476004':'Subconjunctival', + '34206005':'Subcutaneous', + '37839007':'Sublingual', + '419874009':'Submucosal', + '11564211000001101':'Submucosal rectal', + '33770711000001104':'Subretinal', + '6064005':'Topical', + '45890007':'Transdermal', + '11479001000001107':'Translingual', + '404815008':'Transmucosal', + '90028008':'Urethral', + '16857009':'Vaginal'} + + # Look up the term for the code + def findLookUp(self, fieldValue): + try: + lookUpValue = self.allData[fieldValue] + except: + lookUpValue = '' + return lookUpValue diff --git a/lambdas/shared/src/common/validator/parsers/csv_line_parser.py b/lambdas/shared/src/common/validator/parsers/csv_line_parser.py new file mode 100644 index 0000000000..e4ba27cdf6 --- /dev/null +++ b/lambdas/shared/src/common/validator/parsers/csv_line_parser.py @@ -0,0 +1,20 @@ +# CSV Row importer and data access +import csv + +class CSVLineParser: + #parser variables + CSVFileData = {} + + # parse the CSV into a Dictionary + def parseCSVLine(self, CSVRow, CSVHeader): + #create a key value mapping + keys = list(csv.reader([CSVHeader]))[0] + values = list(csv.reader([CSVRow]))[0] + self.CSVFileData = dict(map(lambda i,j : (i,j) , keys, values)) + + + #retrieve a column of data to work with + def getKeyValues(self, fieldName): + # creating empty lists, convery to list + data = [self.CSVFileData[fieldName]] + return data \ No newline at end of file diff --git a/lambdas/shared/src/common/validator/parsers/csv_parser.py b/lambdas/shared/src/common/validator/parsers/csv_parser.py new file mode 100644 index 0000000000..81d657eee9 --- /dev/null +++ b/lambdas/shared/src/common/validator/parsers/csv_parser.py @@ -0,0 +1,26 @@ +# CSV importer and data access +import csv + +class CSVParser: + #parser variables + CSVFileData = {} + +#--------------------------------------------- +# File Management + + # parse the CSV into a Dictionary + def parseCSVFile(self, CSVFileName): + input_file = csv.DictReader(open(CSVFileName)) + self.CSVFileData = {elem: [] for elem in input_file.fieldnames} + for row in input_file: + for key in self.CSVFileData.keys(): + self.CSVFileData[key].append(row[key]) + +#--------------------------------------------- +#Scan and retrieve values + + #retrieve a column of data to work with + def getKeyValues(self, fieldName): + # creating empty lists + data = self.CSVFileData[fieldName] + return data \ No newline at end of file diff --git a/lambdas/shared/src/common/validator/parsers/fhir_parser.py b/lambdas/shared/src/common/validator/parsers/fhir_parser.py new file mode 100644 index 0000000000..d84da610ef --- /dev/null +++ b/lambdas/shared/src/common/validator/parsers/fhir_parser.py @@ -0,0 +1,108 @@ +# FHIR JSON importer and data access +import json + +class FHIRParser: + #parser variables + FHIRResource = {} + +#------------------------------------------- +#File Management + +# used for files + def parseFHIRFile(self, FHIRFileName): + with open(FHIRFileName, 'r') as JSON: + self.FHIRResource = json.load(JSON) + + +# used for JSON FHIR Resource data + def parseFHIRData(self, FHIRData): + self.FHIRResource = FHIRData + +#------------------------------------------------ +# Scan and Identify + +# scan for a key name or a value + def _scanValuesForMatch(self, parent, matchValue): + try: + for key in parent: + if (parent[key] == matchValue): + return True + return False + except: + return False + + +# locate an index for an item in a list + def _locateListId(self, parent, locator): + fieldList = locator.split(":") + nodeId = 0 + index = 0 + try: + while index < len(parent): + for key in parent[index]: + if ((parent[index][key] == fieldList[1]) or (key == fieldList[1])): + nodeId = index + break + else: + if self._scanValuesForMatch(parent[index][key], fieldList[1]): + nodeId = index + break + index += 1 + except: + return '' + return parent[nodeId] + + +# identify a node in the FHIR data + def _getNode(self, parent, child): + #check for indices + try: + result = parent[child] + except: + try: + child = int(child) + result = parent[child] + except: + result = '' + return result + + +# locate a value for a key + def _scanForValue(self, FHIRFields): + fieldList = FHIRFields.split("|") + #get root field before we iterate + rootfield = self.FHIRResource[fieldList[0]] + del fieldList[0] + try: + for field in fieldList: + if (field.startswith("#")): + rootfield = self._locateListId(rootfield, field) #check here for default index?? + else: + rootfield = self._getNode(rootfield, field) + except: + rootfield = '' + return rootfield + + + # get the value list for a key + def getKeyValue(self, fieldName): + value = [] + try: + responseValue = self._scanForValue(fieldName) + except: + responseValue = '' + + value.append(responseValue) + return value + + + # get the value list for a key + def getKeySingleValue(self, fieldName): + value = '' + try: + responseValue = self._scanForValue(fieldName) + except: + responseValue = '' + + value = responseValue + return value \ No newline at end of file diff --git a/lambdas/shared/src/common/validator/parsers/schema_parser.py b/lambdas/shared/src/common/validator/parsers/schema_parser.py new file mode 100644 index 0000000000..7d74186177 --- /dev/null +++ b/lambdas/shared/src/common/validator/parsers/schema_parser.py @@ -0,0 +1,26 @@ +# Schema Parser +# Moved from file loading to JSON string better for elasticache + +class SchemaParser: + #parser variables + SchemaFile = {} + Expressions = {} + + def parseSchema(self, schemaFile): # changed to accept JSON better for cache + self.SchemaFile = schemaFile + self.Expressions = self.SchemaFile['expressions'] + + + def expressionCount(self): + count = 0 + count = sum([1 for d in self.Expressions if 'expression' in d]) + return count + + + def getExpressions(self): + return self.Expressions + + + def getExpression(self, expressionNumber): + expression = self.Expressions[expressionNumber] + return expression \ No newline at end of file diff --git a/lambdas/shared/src/common/validator/reporter/dq_reporter.py b/lambdas/shared/src/common/validator/reporter/dq_reporter.py new file mode 100644 index 0000000000..5866d2c9dc --- /dev/null +++ b/lambdas/shared/src/common/validator/reporter/dq_reporter.py @@ -0,0 +1,64 @@ +import json +import datetime +import validator.enums.error_levels as ErrorLevels +from dateutil import parser + +ErrorReport = { + "eventId": "", + "validationDate": "", + "validated": 'true', + "results": { + "totalErrors": 0, + "completeness": { + "errors": 0, + "fields": [] + }, + "consistency": { + "errors": 0, + "fields": [] + }, + "validity": { + "errors": 0, + "fields": [] + }, + "timeliness_processed": 0 + } +} + + +class DQReporter: + + # create the date difference for the report in minutes + def diff_dates(self, date1, date2): + diffSeconds = abs(date2-date1).total_seconds() + diffMinutes = diffSeconds / 60 + return diffMinutes + + def generateErrorReport(self, eventId, Occurrence, error_records): + occurenceDate = Occurrence + occurenceDate = parser.parse(occurenceDate, ignoretz=True) + validationDate = datetime.datetime.now(tz=None) + + timeTaken = self.diff_dates(occurenceDate, validationDate) + + ErrorReport['validationDate'] = validationDate.isoformat() + ErrorReport['eventId'] = eventId + ErrorReport['results']['timeliness_processed'] = timeTaken + + for errorRecord in error_records: + self.updateReport(errorRecord) + + jsonErrorReport = json.dumps(ErrorReport) + return jsonErrorReport + + def updateReport(self, errorData): + errorGroup = errorData["errorGroup"] + if (errorData['errorLevel'] == ErrorLevels.CRITICAL_ERROR): + ErrorReport['validated'] = "false" + totalErrors = ErrorReport['results']['totalErrors'] + resultsErrorCount = ErrorReport['results'][errorGroup]['errors'] + resultsErrorCount += 1 + totalErrors += 1 + ErrorReport['results'][errorGroup]['fields'].append(errorData["name"]) + ErrorReport['results'][errorGroup]['errors'] = resultsErrorCount + ErrorReport['results']['totalErrors'] = totalErrors diff --git a/lambdas/shared/src/common/validator/validation_expression_checker.py b/lambdas/shared/src/common/validator/validation_expression_checker.py new file mode 100644 index 0000000000..b4addaee8d --- /dev/null +++ b/lambdas/shared/src/common/validator/validation_expression_checker.py @@ -0,0 +1,653 @@ +# Root and base type expression checker functions +import validator.enums.exception_messages as ExceptionMessages +import datetime +import uuid +import re +from validator.lookup.lookup_data import LookUpData +from validator.lookup.key_data import KeyData + + +class ErrorReport(): + def __init__(self, code: int, message: str, row: int = None, field: str = None, details: str = None, + summarise: bool = False): + self.code = code + self.message = message + if not summarise: + self.row = row + self.field = field + self.details = details + self.summarise = summarise + # these are set when the error is added to the report + self.errorGroup = None + self.name = None + self.id = None + self.errorLevel = None + + # function to return the object as a dictionary + def to_dict(self): + ret = { + 'code': self.code, + 'message': self.message + } + if not self.summarise: + ret.update({ + 'row': self.row, + 'field': self.field, + 'details': self.details + }) + return ret + + +# record exception capture +class RecordError(Exception): + + def __init__(self, code=None, message=None, details=None): + self.code = code + self.message = message + self.details = details + + def __str__(self): + return repr((self.code, self.message, self.details)) + + def __repr__(self): + return repr((self.code, self.message, self.details)) + + +# main expressions checker +class ExpressionChecker: + # validation settings + summarise = False + report_unexpected_exception = True + dataParser = any + dataLookUp = any + keyData = any + + def __init__(self, dataParser, summarise, report_unexpected_exception): + self.dataParser = dataParser # FHIR data parser for additional functions + self.dataLookUp = LookUpData() # used for generic look up + self.keyData = KeyData() # used for key check on data we know (Snomed / ODS etc) + self.summarise = summarise + self.report_unexpected_exception = report_unexpected_exception + + def validateExpression(self, expressionType, rule, fieldName, fieldValue, row) -> ErrorReport: + match expressionType: + case "DATETIME": + return self._validateDateTime(rule, fieldName, fieldValue, row) + case "DATE": + return self._validateDateTime(rule, fieldName, fieldValue, row) + case "UUID": + return self._validateUUID(rule, fieldName, fieldValue, row) + case "INT": + return self._validateInteger(rule, fieldName, fieldValue, row) + case "FLOAT": + return self._validateFloat(rule, fieldName, fieldValue, row) + case "REGEX": + return self._validateRegex(rule, fieldName, fieldValue, row) + case "EQUAL": + return self._validateEqual(rule, fieldName, fieldValue, row) + case "NOTEQUAL": + return self._validateNotEqual(rule, fieldName, fieldValue, row) + case "IN": + return self._validateIn(rule, fieldName, fieldValue, row) + case "NRANGE": + return self._validateNRange(rule, fieldName, fieldValue, row) + case "INARRAY": + return self._validateInArray(rule, fieldName, fieldValue, row) + case "UPPER": + return self._validateUpper(rule, fieldName, fieldValue, row) + case "LOWER": + return self._validateLower(rule, fieldName, fieldValue, row) + case "LENGTH": + return self._validateLength(rule, fieldName, fieldValue, row) + case "STARTSWITH": + return self._validateStartsWith(rule, fieldName, fieldValue, row) + case "ENDSWITH": + return self._validateEndsWith(rule, fieldName, fieldValue, row) + case "EMPTY": + return self._validateEmpty(rule, fieldName, fieldValue, row) + case "NOTEMPTY": + return self._validateNotEmpty(rule, fieldName, fieldValue, row) + case "POSITIVE": + return self._validatePositive(rule, fieldName, fieldValue, row) + case "POSTCODE": + return self._validatePostCode(rule, fieldName, fieldValue, row) + case "GENDER": + return self._validateGender(rule, fieldName, fieldValue, row) + case "NHSNUMBER": + return self._validateNHSNumber(rule, fieldName, fieldValue, row) + case "MAXOBJECTS": + return self._validateMaxObjects(rule, fieldName, fieldValue, row) + case "ONLYIF": + return self._validateOnlyIf(rule, fieldName, fieldValue, row) + case "LOOKUP": + return self._validateAgainstLookup(rule, fieldName, fieldValue, row) + case "KEYCHECK": + return self._validateAgainstKey(rule, fieldName, fieldValue, row) + case _: + return "Schema expression not found! Check your expression type : " + expressionType + + # iso8086 date time validate + def _validateDateTime(self, rule, fieldName, fieldValue, row): + try: + datetime.date.fromisoformat(fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return RecordError(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return RecordError(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # UUID validate + def _validateUUID(self, expressionRule, fieldName, fieldValue, row): + try: + uuid.UUID(str(fieldValue)) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Integer Validate + def _validateInteger(self, expressionRule, fieldName, + fieldValue, row, summarise) -> ErrorReport: + try: + int(fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + if summarise: + p = RecordError(code, message) + else: + p = RecordError(code, message, row, fieldName, details) + return p + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + if summarise: + p = RecordError(ExceptionMessages.UNEXPECTED_EXCEPTION, message) + else: + p = RecordError(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '') + return p + + # Float Validate + def _validateFloat(self, expressionRule, fieldName, fieldValue, row, summarise): + try: + float(fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + if summarise: + p = {'code': code, 'message': message} + else: + p = {'code': code, 'message': message, 'row': row, 'field': fieldName, 'details': details} + return p + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + if summarise: + p = {'code': ExceptionMessages.UNEXPECTED_EXCEPTION, 'message': message} + else: + p = {'code': ExceptionMessages.UNEXPECTED_EXCEPTION, 'message': message, + 'row': row, 'field': fieldName, 'details': ''} + return p + + # Length Validate + def _validateLength(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + strLen = len(fieldValue) + checklength = int(expressionRule) + if strLen > checklength: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value length check failed", + "Value is longer than expected") + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Regex Validate + def _validateRegex(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + result = re.search(expressionRule, fieldValue) + if not result: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "String REGEX check failed", + "Value does not meet regex rules") + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) + if e.details is not None: + details = e.details + if summarise: + p = ErrorReport(code, message) + else: + p = ErrorReport(code, message, row, fieldName, details) + return p + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + if summarise: + p = ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message) + else: + p = ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '') + return p + + # Equal Validate + def _validateEqual(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + if fieldValue != expressionRule: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value equals check failed", + "Value does not equal expected value, Expected- " + expressionRule + " found- " + + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Not Equal Validate + def _validateNotEqual(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + if fieldValue == expressionRule: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value not equals check failed", + "Value equals expected value when it should not, Expected- " + expressionRule + + " found- " + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # In Validate + def _validateIn(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + if expressionRule.lower() in fieldValue.lower(): + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Data not in Value failed", "Check Data not found in Value, List- " + + expressionRule + " Data- " + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # NRange Validate + def _validateNRange(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + value = float(fieldValue) + rule = expressionRule.split(",") + range1 = float(rule[0]) + range2 = float(rule[1]) + + if range1 <= value >= range2: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value range check failed", "Value is not within the number range, data- " + + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = ( + e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # InArray Validate + def _validateInArray(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + ruleList = expressionRule.split(",") + + if fieldValue not in ruleList: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value not in array check failed", + "Check Value not found in data array") + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Upper Validate + def _validateUpper(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + result = fieldValue.isupper() + + if not result: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value not uppercase", "Check Value not found to be uppercase, value- " + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Lower Validate + def _validateLower(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + result = fieldValue.islower() + + if not result: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value not lowercase", + "Check Value not found to be lowercase, data- " + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Starts With Validate + def _validateStartsWith(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + result = fieldValue.startswith(expressionRule) + if not result: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value starts with failure", + "Value does not start as expected, Expected- " + expressionRule + + " found- " + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Ends With Validate + def _validateEndsWith(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + result = fieldValue.endswith(expressionRule) + if not result: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value ends with failure", + "Value does not end as expected, Expected- " + expressionRule + + " found- " + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Empty Validate + def _validateEmpty(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + if fieldValue: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value is empty failure", + "Value has data, not as expected, data- " + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Not Empty Validate + def _validateNotEmpty(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + if not fieldValue: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value not empty failure", + "Value is empty, not as expected") + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Positive Validate + def _validatePositive(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + value = float(fieldValue) + if value < 0: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value is not positive failure", + "Value is not positive as expected, data- " + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # NHSNumber Validate + def _validateNHSNumber(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + regexRule = '^6[0-9]{10}$' + result = re.search(regexRule, fieldValue) + if not result: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "NHS Number check failed", + "NHS Number does not meet regex rules, data- " + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Gender Validate + def _validateGender(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + ruleList = ['0', '1', '2', '9'] + + if fieldValue not in ruleList: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Gender check failed", + "Gender value not found in array, data- " + fieldValue) + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # PostCode Validate + def _validatePostCode(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + regexRule = '^([Gg][Ii][Rr] 0[Aa]{2})|((([A-Za-z][0-9]{1,2})|(([A-Za-z][A-Ha-hJ-Yj-y]' + '[0-9]{1,2})|(([AZa-z][0-9][A-Za-z])|([A-Za-z][A-Ha-hJ-Yj-y][0-9]?[A-Za-z])))) [0-9][A-Za-z]{2})$' + result = re.search(regexRule, fieldValue) + if not result: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Postcode check failed", + "Postcode does not meet regex rules") + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Max Objects Validate + def _validateMaxObjects(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + value = len(fieldValue) + if value > int(expressionRule): + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Max Objects failure", + "Number of objects is greater than expected") + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Default to Validate + def _validateOnlyIf(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + try: + conversionList = expressionRule.split("|") + location = conversionList[0] + valueCheck = conversionList[1] + dataValue = self.dataParser.getKeyValue(location) + + if (dataValue[0] != valueCheck): + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Validate Only If failure", + "Value was not found at that position") + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Check with Lookup + def _validateAgainstLookup(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + result = self.dataLookUp.findLookUp(fieldValue) + if not result: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value lookup failure", + "Value was not found in Lookup List, Expected- " + fieldValue + " found- nothing") + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + + # Check with Key Lookup + def _validateAgainstKey(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + try: + result = self.KeyData.findKey(expressionRule, fieldValue) + if not result: + raise RecordError(ExceptionMessages.KEY_CHECK_FAILED, + "Key lookup failure", + "Value was not found in Key List, Expected- " + + fieldValue + " found- nothing") + except RecordError as e: + code = e.code if e.code is not None else ExceptionMessages.KEY_CHECK_FAILED + message = (e.message if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.KEY_CHECK_FAILED]) + if e.details is not None: + details = e.details + return ErrorReport(code, message, row, fieldName, details, self.summarise) + except Exception as e: + if self.report_unexpected_exception: + message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) diff --git a/lambdas/shared/src/common/validator/validator.py b/lambdas/shared/src/common/validator/validator.py new file mode 100644 index 0000000000..377bc2a451 --- /dev/null +++ b/lambdas/shared/src/common/validator/validator.py @@ -0,0 +1,187 @@ +# Main validation engine + +import validator.enums.exception_messages as ExceptionMessages +import validator.enums.error_levels as ErrorLevels +from validator.parsers.csv_parser import CSVParser +from validator.parsers.csv_line_parser import CSVLineParser +from validator.parsers.fhir_parser import FHIRParser +from validator.parsers.schema_parser import SchemaParser +from validator.validation_expression_checker import ExpressionChecker, ErrorReport +from validator.reporter.dq_reporter import DQReporter + + +FilePath = '' +JSONData = {} +SchemaFile = {} +CSVRow = '' +CSVHeader = '' +error_records: list[ErrorReport] = [] +isCSV = True +dataType = 'FHIR' # 'FHIR', 'FHIRJSON', 'CSV', 'CSVROW' +dataParser = any + + +class Validator: + + def __init__(self, filepath, JSONData, schemafile, CSVRow, CSVHeader, dataType): + self.FilePath = filepath + self.JSONData = JSONData + self.SchemaFile = schemafile + self.CSVRow = CSVRow + self.CSVHeader = CSVHeader + self.dataType = dataType + + def _getCSVLineParser(self, CSVRow, CSVHeader): + csvParser = CSVLineParser() + csvParser.parseCSVLine(CSVRow, CSVHeader) + return csvParser + + def _getCSVParser(self, filepath): + csvParser = CSVParser() + csvParser.parseCSVFile(filepath) + return csvParser + + def _getFHIRParser(self, filepath): + fhirParser = FHIRParser() + fhirParser.parseFHIRFile(filepath) + return fhirParser + + def _getFHIRJSONParser(self, FHIRData): + fhirParser = FHIRParser() + fhirParser.parseFHIRData(FHIRData) + return fhirParser + + def _getSchemaParser(self, schemafile): + schemaParser = SchemaParser() + schemaParser.parseSchema(schemafile) + return schemaParser + + def _addErrorRecord(self, errorRecord: ErrorReport, expressionErrorGroup, expressionName, expressionId, errorLevel): + if errorRecord is not None: + errorRecord.errorGroup = expressionErrorGroup + errorRecord.name = expressionName + errorRecord.id = expressionId + errorRecord.errorLevel = errorLevel + error_records.append(errorRecord) + + # Function to help identify a parent failure in the error list + def _checkErrorRecordForFailure(self, expressionId): + for errorRecord in error_records: + if (errorRecord.id == expressionId): + return True + return False + + # validate a single expression against the data file + def _validateExpression(self, ExpressionValidate, expression, + inc_header_in_row_count) -> ErrorReport | int: + row = 1 + if inc_header_in_row_count: + row = 2 + + if self.isCSV: + expressionFieldName = expression['fieldNameCSV'] + else: + expressionFieldName = expression['fieldNameFHIR'] + + expressionId = expression['expressionId'] + errorLevel = expression['errorLevel'] + expressionName = expression['expression']['expressionName'] + expressionType = expression['expression']['expressionType'] + expressionRule = expression['expression']['expressionRule'] + expressionErrorGroup = expression['errorGroup'] + + # Check to see if the expression has a parent, if so did the parent validate + if ('parentExpression' in expression): + parentExpression = expression['parentExpression'] + if (self._checkErrorRecordForFailure(parentExpression)): + errorRecord = {'code': ExceptionMessages.PARENT_FAILED, + 'message': ExceptionMessages.MESSAGES[ExceptionMessages.PARENT_FAILED] + + ', Parent ID: ' + parentExpression} + self._addErrorRecord(errorRecord, expressionErrorGroup, expressionName, expressionId, errorLevel) + return errorRecord + + try: + expressionValues = self.dataParser.getKeyValue(expressionFieldName) + except Exception as e: + message = 'Data get values Unexpected exception [%s]: %s' % (e.__class__.__name__, e) + error_report = ErrorReport(code=ExceptionMessages.PARSING_ERROR, message=message) + self._addErrorRecord(error_report, expressionErrorGroup, expressionName, expressionId, self.CriticalErrorLevel) + return error_report + + for value in expressionValues: + errorRecord: ErrorReport = ExpressionValidate.validateExpression(expressionType, expressionRule, + expressionFieldName, value, row) + if errorRecord is not None: + self._addErrorRecord(errorRecord, expressionErrorGroup, expressionName, expressionId, errorLevel) + row += 1 + return row + + # run the validation against the data + def runValidation(self, summarise=False, report_unexpected_exception=True, + inc_header_in_row_count=True) -> list[ErrorReport]: + try: + error_records.clear() + + match self.dataType: # 'FHIR', 'FHIRJSON', 'CSV', 'CSVROW' + case 'FHIR': + self.dataParser = self._getFHIRParser(self.FilePath) + self.isCSV = False + case 'FHIRJSON': + self.dataParser = self._getFHIRJSONParser(self.JSONData) + self.isCSV = False + case 'CSV': + self.dataParser = self._getCSVParser(self.FilePath) + self.isCSV = True + case 'CSVROW': + self.dataParser = self._getCSVLineParser(self.CSVRow, self.CSVHeader) + self.isCSV = True + + except Exception as e: + if report_unexpected_exception: + message = 'Data Parser Unexpected exception [%s]: %s' % (e.__class__.__name__, e) + return [ErrorReport(code=0, message=message)] + + try: + schemaParser = self._getSchemaParser(self.SchemaFile) + except Exception as e: + if report_unexpected_exception: + message = 'Schema Parser Unexpected exception [%s]: %s' % (e.__class__.__name__, e) + return [ErrorReport(code=0, message=message)] + + try: + ExpressionValidate = ExpressionChecker(dataParser, summarise, report_unexpected_exception) + except Exception as e: + if report_unexpected_exception: + message = 'Expression Checker Unexpected exception [%s]: %s' % (e.__class__.__name__, e) + return [ErrorReport(code=0, message=message)] + + # get list of expressions + try: + expressions = schemaParser.getExpressions() + except Exception as e: + if report_unexpected_exception: + message = 'Expression Getter Unexpected exception [%s]: %s' % (e.__class__.__name__, e) + return [ErrorReport(code=0, message=message)] + + for expression in expressions: + # rows = self._validateExpression(ExpressionValidate, expression, inc_header_in_row_count) + self._validateExpression(ExpressionValidate, expression, inc_header_in_row_count) + + return error_records + + # ------------------------------------------------------------------------- + # Report Generation + # Build the error Report + def buildErrorReport(self, eventId): + OccurrenceDateTime = self.dataParser.getKeySingleValue('occurrenceDateTime') + dqReporter = DQReporter() + dqReport = dqReporter.generateErrorReport(eventId, OccurrenceDateTime, error_records) + + return dqReport + + # Check all errors to see if we have a critical error that would fail the validation + def hasValidationFailed(self): + for errorRecord in error_records: + if (errorRecord['errorLevel'] == ErrorLevels.CRITICAL_ERROR): + return True + return False From 2ac8163ad5b99875f7d6f107a582f06c33f94817 Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Thu, 9 Oct 2025 16:11:59 +0100 Subject: [PATCH 02/13] Wip --- .../common/validator/enums/error_levels.py | 8 +- .../validator/enums/exception_messages.py | 28 +- .../src/common/validator/lookup/key_data.py | 186 +++++------ .../common/validator/lookup/lookup_data.py | 204 ++++++------- .../validator/parsers/csv_line_parser.py | 21 +- .../common/validator/parsers/csv_parser.py | 35 ++- .../common/validator/parsers/fhir_parser.py | 92 +++--- .../common/validator/parsers/schema_parser.py | 34 +-- .../validation_expression_checker.py | 289 ++++++++---------- .../shared/src/common/validator/validator.py | 149 ++++----- 10 files changed, 509 insertions(+), 537 deletions(-) diff --git a/lambdas/shared/src/common/validator/enums/error_levels.py b/lambdas/shared/src/common/validator/enums/error_levels.py index c6f8f51e0e..a2689e53ac 100644 --- a/lambdas/shared/src/common/validator/enums/error_levels.py +++ b/lambdas/shared/src/common/validator/enums/error_levels.py @@ -4,7 +4,7 @@ NOTIFICATION = 2 MESSAGES = { - CRITICAL_ERROR: 'Critical Validation Error [%s]: %s', - WARNING: 'Non-Critical Validation Error [%s]: %s', - NOTIFICATION: 'Quality Issue Found [%s]: %s', - } + CRITICAL_ERROR: 'Critical Validation Error [%s]: %s', + WARNING: 'Non-Critical Validation Error [%s]: %s', + NOTIFICATION: 'Quality Issue Found [%s]: %s', +} diff --git a/lambdas/shared/src/common/validator/enums/exception_messages.py b/lambdas/shared/src/common/validator/enums/exception_messages.py index 770f25c6db..d96a839c4e 100644 --- a/lambdas/shared/src/common/validator/enums/exception_messages.py +++ b/lambdas/shared/src/common/validator/enums/exception_messages.py @@ -15,17 +15,17 @@ MESSAGES = { - UNEXPECTED_EXCEPTION: 'Unexpected exception [%s]: %s', - VALUE_CHECK_FAILED: 'Value check failed.', - HEADER_CHECK_FAILED: 'Header check failed.', - RECORD_LENGTH_CHECK_FAILED: 'Record length check failed.', - RECORD_CHECK_FAILED: 'Record check failed.', - VALUE_PREDICATE_FALSE: 'Value predicate returned false.', - RECORD_PREDICATE_FALSE: 'Record predicate returned false.', - UNIQUE_CHECK_FAILED: 'Unique check failed.', - ASSERT_CHECK_FAILED: 'Assertion check failed.', - FINALLY_ASSERT_CHECK_FAILED: 'Final assertion check failed.', - PARSING_ERROR: 'Failed to parse data correctly.', - PARENT_FAILED: 'The parent expression failed to validate', - KEY_CHECK_FAILED: 'Value could not be found in the Key list' - } + UNEXPECTED_EXCEPTION: 'Unexpected exception [%s]: %s', + VALUE_CHECK_FAILED: 'Value check failed.', + HEADER_CHECK_FAILED: 'Header check failed.', + RECORD_LENGTH_CHECK_FAILED: 'Record length check failed.', + RECORD_CHECK_FAILED: 'Record check failed.', + VALUE_PREDICATE_FALSE: 'Value predicate returned false.', + RECORD_PREDICATE_FALSE: 'Record predicate returned false.', + UNIQUE_CHECK_FAILED: 'Unique check failed.', + ASSERT_CHECK_FAILED: 'Assertion check failed.', + FINALLY_ASSERT_CHECK_FAILED: 'Final assertion check failed.', + PARSING_ERROR: 'Failed to parse data correctly.', + PARENT_FAILED: 'The parent expression failed to validate', + KEY_CHECK_FAILED: 'Value could not be found in the Key list' +} diff --git a/lambdas/shared/src/common/validator/lookup/key_data.py b/lambdas/shared/src/common/validator/lookup/key_data.py index 4af7b2cad5..61cdf6c4be 100644 --- a/lambdas/shared/src/common/validator/lookup/key_data.py +++ b/lambdas/shared/src/common/validator/lookup/key_data.py @@ -1,98 +1,100 @@ -#--------------------------------------------------------------------------------------------------------- -#main conversion lookup +# --------------------------------------------------------------------------------------------------------- +# main conversion lookup class KeyData: - #data settings - Procedure = ['956951000000104'] + # data settings + def __init__(self): + self.procedure = ['956951000000104'] - Organisation = ['RJ1', 'RJC02'] + self.organisation = ['RJ1', 'RJC02'] - Site = ['368208006', '279549004', '74262004', '368209003', '723979003', '61396006', '723980000', '11207009', '420254004'] + self.site = ['368208006', '279549004', '74262004', '368209003', + '723979003', '61396006', '723980000', '11207009', '420254004'] + + self.route = ['54471007', + '372449004', + '372450004', + '372451000', + '372452007', + '404820008', + '18246711000001107', + '372453002', + '372454008', + '127490009', + '372457001', + '9191401000001100', + '18682911000001103', + '10334211000001103', + '718329006', + '18679011000001101', + '34777511000001106', + '372458006', + '58100008', + '12130007', + '372459003', + '418821007', + '372460008', + '372461007', + '420719007', + '19537211000001108', + '372463005', + '372464004', + '372465003', + '448077001', + '38233211000001106', + '372466002', + '372467006', + '78421000', + '255559005', + '372468001', + '417255000', + '38239002', + '372469009', + '372470005', + '418586008', + '72607000', + '447122006', + '62226000', + '47625008', + '420287000', + '372471009', + '418401004', + '21856811000001103', + '127491008', + '9907001000001103', + '46713006', + '127492001', + '418730005', + '54485002', + '26643006', + '372473007', + '10547007', + '225691000001105', + '9191501000001101', + '372474001', + '39338211000001108', + '3323001000001107', + '372475000', + '11478901000001102', + '419762003', + '39337511000001107', + '37161004', + '11564311000001109', + '418321004', + '3594011000001102', + '372476004', + '34206005', + '37839007', + '419874009', + '11564211000001101', + '33770711000001104', + '6064005', + '45890007', + '11479001000001107', + '404815008', + '90028008', + '16857009'] - Route = [ '54471007', - '372449004', - '372450004', - '372451000', - '372452007', - '404820008', - '18246711000001107', - '372453002', - '372454008', - '127490009', - '372457001', - '9191401000001100', - '18682911000001103', - '10334211000001103', - '718329006', - '18679011000001101', - '34777511000001106', - '372458006', - '58100008', - '12130007', - '372459003', - '418821007', - '372460008', - '372461007', - '420719007', - '19537211000001108', - '372463005', - '372464004', - '372465003', - '448077001', - '38233211000001106', - '372466002', - '372467006', - '78421000', - '255559005', - '372468001', - '417255000', - '38239002', - '372469009', - '372470005', - '418586008', - '72607000', - '447122006', - '62226000', - '47625008', - '420287000', - '372471009', - '418401004', - '21856811000001103', - '127491008', - '9907001000001103', - '46713006', - '127492001', - '418730005', - '54485002', - '26643006', - '372473007', - '10547007', - '225691000001105', - '9191501000001101', - '372474001', - '39338211000001108', - '3323001000001107', - '372475000', - '11478901000001102', - '419762003', - '39337511000001107', - '37161004', - '11564311000001109', - '418321004', - '3594011000001102', - '372476004', - '34206005', - '37839007', - '419874009', - '11564211000001101', - '33770711000001104', - '6064005', - '45890007', - '11479001000001107', - '404815008', - '90028008', - '16857009'] - # Look up the term for the code def findKey(self, KeySource, fieldValue): try: @@ -101,7 +103,7 @@ def findKey(self, KeySource, fieldValue): case "Organisation": fieldValue in self.Organisation case "Site": fieldValue in self.Site case "Route": fieldValue in self.Route - case _: return False - except: + case _: return False + except Exception: return False return False diff --git a/lambdas/shared/src/common/validator/lookup/lookup_data.py b/lambdas/shared/src/common/validator/lookup/lookup_data.py index b19dd72e2a..3431ea67f4 100644 --- a/lambdas/shared/src/common/validator/lookup/lookup_data.py +++ b/lambdas/shared/src/common/validator/lookup/lookup_data.py @@ -1,105 +1,105 @@ -#--------------------------------------------------------------------------------------------------------- -#main conversion lookup - +# --------------------------------------------------------------------------------------------------------- +# main conversion lookup class LookUpData: - #data settings - allData = { '368208006':'Left upper arm structure', - '279549004':'Nasal cavity structure', - '74262004':'Oral cavity structure', - '368209003':'Right upper arm structure', - '723979003':'Structure of left buttock', - '61396006':'Structure of left thigh', - '723980000':'Structure of right buttock', - '11207009':'Structure of right thigh', - '420254004':'Body cavity', - '54471007':'Buccal', - '372449004':'Dental', - '372450004':'Endocervical', - '372451000':'Endosinusial', - '372452007':'Endotracheopulmonary', - '404820008':'Epidural', - '18246711000001107':'Epilesional', - '372453002':'Extraamniotic', - '372454008':'Gastroenteral', - '127490009':'Gastrostomy route', - '372457001':'Gingival', - '9191401000001100':'Haemodiafiltration', - '18682911000001103':'Haemodialysis', - '10334211000001103':'Haemofiltration', - '718329006':'Infiltration', - '18679011000001101':'Inhalation', - '34777511000001106':'Intestinal use', - '372458006':'Intraamniotic', - '58100008':'Intraarterial', - '12130007':'Intraarticular', - '372459003':'Intrabursal', - '418821007':'Intracameral use', - '372460008':'Intracardiac', - '372461007':'Intracavernous', - '420719007':'Intracerebroventricular', - '19537211000001108':'Intracervical route', - '372463005':'Intracoronary', - '372464004':'Intradermal', - '372465003':'Intradiscal', - '448077001':'Intraepidermal', - '38233211000001106':'Intraglandular', - '372466002':'Intralesional', - '372467006':'Intralymphatic', - '78421000':'Intramuscular', - '255559005':'Intramuscular', - '372468001':'Intraocular', - '417255000':'Intraosseous', - '38239002':'Intraperitoneal', - '372469009':'Intrapleural', - '372470005':'Intrasternal', - '418586008':'Intratendinous route', - '72607000':'Intrathecal', - '447122006':'Intratumoral', - '62226000':'Intrauterine', - '47625008':'Intravenous', - '420287000':'Intraventricular cardiac', - '372471009':'Intravesical', - '418401004':'Intravitreal', - '21856811000001103':'Iontophoresis', - '127491008':'Jejunostomy route', - '9907001000001103':'Line lock', - '46713006':'Nasal', - '127492001':'Nasogastric route', - '418730005':'Nasojejunal route', - '54485002':'Ophthalmic route', - '26643006':'Oral', - '372473007':'Oromucosal', - '10547007':'Otic', - '225691000001105':'PEG tube route', - '9191501000001101':'Percutaneous', - '372474001':'Periarticular', - '39338211000001108':'Peribulbar ocular', - '3323001000001107':'Pericardial route', - '372475000':'Perineural', - '11478901000001102':'Periosseous', - '419762003':'Peritendinous route', - '39337511000001107':'Peritumoral', - '37161004':'Rectal', - '11564311000001109':'Regional perfusion', - '418321004':'Retrobulbar route', - '3594011000001102':'Route of administration not applicable', - '372476004':'Subconjunctival', - '34206005':'Subcutaneous', - '37839007':'Sublingual', - '419874009':'Submucosal', - '11564211000001101':'Submucosal rectal', - '33770711000001104':'Subretinal', - '6064005':'Topical', - '45890007':'Transdermal', - '11479001000001107':'Translingual', - '404815008':'Transmucosal', - '90028008':'Urethral', - '16857009':'Vaginal'} - + # data settings + def __init__(self): + self.all_data = {'368208006': 'Left upper arm structure', + '279549004': 'Nasal cavity structure', + '74262004': 'Oral cavity structure', + '368209003': 'Right upper arm structure', + '723979003': 'Structure of left buttock', + '61396006': 'Structure of left thigh', + '723980000': 'Structure of right buttock', + '11207009': 'Structure of right thigh', + '420254004': 'Body cavity', + '54471007': 'Buccal', + '372449004': 'Dental', + '372450004': 'Endocervical', + '372451000': 'Endosinusial', + '372452007': 'Endotracheopulmonary', + '404820008': 'Epidural', + '18246711000001107': 'Epilesional', + '372453002': 'Extraamniotic', + '372454008': 'Gastroenteral', + '127490009': 'Gastrostomy route', + '372457001': 'Gingival', + '9191401000001100': 'Haemodiafiltration', + '18682911000001103': 'Haemodialysis', + '10334211000001103': 'Haemofiltration', + '718329006': 'Infiltration', + '18679011000001101': 'Inhalation', + '34777511000001106': 'Intestinal use', + '372458006': 'Intraamniotic', + '58100008': 'Intraarterial', + '12130007': 'Intraarticular', + '372459003': 'Intrabursal', + '418821007': 'Intracameral use', + '372460008': 'Intracardiac', + '372461007': 'Intracavernous', + '420719007': 'Intracerebroventricular', + '19537211000001108': 'Intracervical route', + '372463005': 'Intracoronary', + '372464004': 'Intradermal', + '372465003': 'Intradiscal', + '448077001': 'Intraepidermal', + '38233211000001106': 'Intraglandular', + '372466002': 'Intralesional', + '372467006': 'Intralymphatic', + '78421000': 'Intramuscular', + '255559005': 'Intramuscular', + '372468001': 'Intraocular', + '417255000': 'Intraosseous', + '38239002': 'Intraperitoneal', + '372469009': 'Intrapleural', + '372470005': 'Intrasternal', + '418586008': 'Intratendinous route', + '72607000': 'Intrathecal', + '447122006': 'Intratumoral', + '62226000': 'Intrauterine', + '47625008': 'Intravenous', + '420287000': 'Intraventricular cardiac', + '372471009': 'Intravesical', + '418401004': 'Intravitreal', + '21856811000001103': 'Iontophoresis', + '127491008': 'Jejunostomy route', + '9907001000001103': 'Line lock', + '46713006': 'Nasal', + '127492001': 'Nasogastric route', + '418730005': 'Nasojejunal route', + '54485002': 'Ophthalmic route', + '26643006': 'Oral', + '372473007': 'Oromucosal', + '10547007': 'Otic', + '225691000001105': 'PEG tube route', + '9191501000001101': 'Percutaneous', + '372474001': 'Periarticular', + '39338211000001108': 'Peribulbar ocular', + '3323001000001107': 'Pericardial route', + '372475000': 'Perineural', + '11478901000001102': 'Periosseous', + '419762003': 'Peritendinous route', + '39337511000001107': 'Peritumoral', + '37161004': 'Rectal', + '11564311000001109': 'Regional perfusion', + '418321004': 'Retrobulbar route', + '3594011000001102': 'Route of administration not applicable', + '372476004': 'Subconjunctival', + '34206005': 'Subcutaneous', + '37839007': 'Sublingual', + '419874009': 'Submucosal', + '11564211000001101': 'Submucosal rectal', + '33770711000001104': 'Subretinal', + '6064005': 'Topical', + '45890007': 'Transdermal', + '11479001000001107': 'Translingual', + '404815008': 'Transmucosal', + '90028008': 'Urethral', + '16857009': 'Vaginal'} + # Look up the term for the code - def findLookUp(self, fieldValue): + def find_lookup(self, field_value): try: - lookUpValue = self.allData[fieldValue] - except: - lookUpValue = '' - return lookUpValue + lookup_value = self.all_data[field_value] + except Exception: + lookup_value = '' + return lookup_value diff --git a/lambdas/shared/src/common/validator/parsers/csv_line_parser.py b/lambdas/shared/src/common/validator/parsers/csv_line_parser.py index e4ba27cdf6..df3f37fe10 100644 --- a/lambdas/shared/src/common/validator/parsers/csv_line_parser.py +++ b/lambdas/shared/src/common/validator/parsers/csv_line_parser.py @@ -1,20 +1,21 @@ # CSV Row importer and data access import csv + class CSVLineParser: - #parser variables - CSVFileData = {} + # parser variables + def __init__(self): + self.csv_file_data = {} # parse the CSV into a Dictionary def parseCSVLine(self, CSVRow, CSVHeader): - #create a key value mapping + # create a key value mapping keys = list(csv.reader([CSVHeader]))[0] - values = list(csv.reader([CSVRow]))[0] - self.CSVFileData = dict(map(lambda i,j : (i,j) , keys, values)) - + values = list(csv.reader([CSVRow]))[0] + self.csv_file_data = dict(map(lambda i, j: (i, j), keys, values)) - #retrieve a column of data to work with + # retrieve a column of data to work with def getKeyValues(self, fieldName): - # creating empty lists, convery to list - data = [self.CSVFileData[fieldName]] - return data \ No newline at end of file + # creating empty lists, convert to list + data = [self.csv_file_data[fieldName]] + return data diff --git a/lambdas/shared/src/common/validator/parsers/csv_parser.py b/lambdas/shared/src/common/validator/parsers/csv_parser.py index 81d657eee9..5c1ca140e9 100644 --- a/lambdas/shared/src/common/validator/parsers/csv_parser.py +++ b/lambdas/shared/src/common/validator/parsers/csv_parser.py @@ -1,26 +1,25 @@ # CSV importer and data access import csv -class CSVParser: - #parser variables - CSVFileData = {} -#--------------------------------------------- -# File Management +class CSVParser: + """ File Management""" + # parser variables + def __init__(self): + self.csv_file_data = {} # parse the CSV into a Dictionary - def parseCSVFile(self, CSVFileName): - input_file = csv.DictReader(open(CSVFileName)) - self.CSVFileData = {elem: [] for elem in input_file.fieldnames} + def parse_csv_file(self, csv_filename): + input_file = csv.DictReader(open(csv_filename)) + self.csv_file_data = {elem: [] for elem in input_file.fieldnames} for row in input_file: - for key in self.CSVFileData.keys(): - self.CSVFileData[key].append(row[key]) - -#--------------------------------------------- -#Scan and retrieve values + for key in self.csv_file_data.keys(): + self.csv_file_data[key].append(row[key]) - #retrieve a column of data to work with - def getKeyValues(self, fieldName): - # creating empty lists - data = self.CSVFileData[fieldName] - return data \ No newline at end of file + # --------------------------------------------- + # Scan and retrieve values + # retrieve a column of data to work with + def get_key_values(self, field_name): + # creating empty lists + data = self.csv_file_data[field_name] + return data diff --git a/lambdas/shared/src/common/validator/parsers/fhir_parser.py b/lambdas/shared/src/common/validator/parsers/fhir_parser.py index d84da610ef..5b6a1893d3 100644 --- a/lambdas/shared/src/common/validator/parsers/fhir_parser.py +++ b/lambdas/shared/src/common/validator/parsers/fhir_parser.py @@ -1,108 +1,102 @@ # FHIR JSON importer and data access import json -class FHIRParser: - #parser variables - FHIRResource = {} - -#------------------------------------------- -#File Management - -# used for files - def parseFHIRFile(self, FHIRFileName): - with open(FHIRFileName, 'r') as JSON: - self.FHIRResource = json.load(JSON) +class FHIRParser: + # parser variables + def __init__(self): + self.FHIRResource = {} -# used for JSON FHIR Resource data - def parseFHIRData(self, FHIRData): - self.FHIRResource = FHIRData + # ------------------------------------------- + # File Management + # used for files + def parse_fhir_file(self, fhir_file_name): + with open(fhir_file_name, 'r') as json_file: + self.FHIRResource = json.load(json_file) -#------------------------------------------------ -# Scan and Identify + # used for JSON FHIR Resource data + def parse_fhir_data(self, fhir_data): + self.FHIRResource = fhir_data -# scan for a key name or a value - def _scanValuesForMatch(self, parent, matchValue): + # ------------------------------------------------ + # Scan and Identify + # scan for a key name or a value + def _scan_values_for_match(self, parent, match_value): try: for key in parent: - if (parent[key] == matchValue): + if (parent[key] == match_value): return True return False - except: + except Exception: return False - -# locate an index for an item in a list - def _locateListId(self, parent, locator): - fieldList = locator.split(":") - nodeId = 0 + # locate an index for an item in a list + def _locate_list_id(self, parent, locator): + field_list = locator.split(":") + node_id = 0 index = 0 try: while index < len(parent): for key in parent[index]: - if ((parent[index][key] == fieldList[1]) or (key == fieldList[1])): - nodeId = index + if ((parent[index][key] == field_list[1]) or (key == field_list[1])): + node_id = index break else: - if self._scanValuesForMatch(parent[index][key], fieldList[1]): - nodeId = index + if self._scan_values_for_match(parent[index][key], field_list[1]): + node_id = index break index += 1 - except: + except Exception: return '' - return parent[nodeId] - + return parent[node_id] -# identify a node in the FHIR data - def _getNode(self, parent, child): - #check for indices + # identify a node in the FHIR data + def _get_node(self, parent, child): + # check for indices try: result = parent[child] - except: + except Exception: try: child = int(child) result = parent[child] - except: + except Exception: result = '' return result - -# locate a value for a key + # locate a value for a key def _scanForValue(self, FHIRFields): fieldList = FHIRFields.split("|") - #get root field before we iterate + # get root field before we iterate rootfield = self.FHIRResource[fieldList[0]] del fieldList[0] try: for field in fieldList: if (field.startswith("#")): - rootfield = self._locateListId(rootfield, field) #check here for default index?? + rootfield = self._locate_list_id(rootfield, field) # check here for default index?? else: - rootfield = self._getNode(rootfield, field) - except: + rootfield = self._get_node(rootfield, field) + except Exception: rootfield = '' - return rootfield - + return rootfield # get the value list for a key def getKeyValue(self, fieldName): value = [] try: responseValue = self._scanForValue(fieldName) - except: + except Exception: responseValue = '' value.append(responseValue) return value - # get the value list for a key def getKeySingleValue(self, fieldName): value = '' try: responseValue = self._scanForValue(fieldName) - except: + except Exception: responseValue = '' value = responseValue - return value \ No newline at end of file + return value diff --git a/lambdas/shared/src/common/validator/parsers/schema_parser.py b/lambdas/shared/src/common/validator/parsers/schema_parser.py index 7d74186177..5bce18d9de 100644 --- a/lambdas/shared/src/common/validator/parsers/schema_parser.py +++ b/lambdas/shared/src/common/validator/parsers/schema_parser.py @@ -1,26 +1,24 @@ # Schema Parser -# Moved from file loading to JSON string better for elasticache +# Moved from file loading to JSON string better for elasticache class SchemaParser: - #parser variables - SchemaFile = {} - Expressions = {} + def __init__(self): + # parser variables + self.schema_file = {} + self.expressions = {} - def parseSchema(self, schemaFile): # changed to accept JSON better for cache - self.SchemaFile = schemaFile - self.Expressions = self.SchemaFile['expressions'] + def parse_schema(self, schema_file): # changed to accept JSON better for cache + self.schema_file = schema_file + self.expressions = self.schema_file['expressions'] - - def expressionCount(self): + def expression_count(self): count = 0 - count = sum([1 for d in self.Expressions if 'expression' in d]) + count = sum([1 for d in self.expressions if 'expression' in d]) return count - - - def getExpressions(self): - return self.Expressions - - def getExpression(self, expressionNumber): - expression = self.Expressions[expressionNumber] - return expression \ No newline at end of file + def get_expressions(self): + return self.expressions + + def get_expression(self, expression_number): + expression = self.expressions[expression_number] + return expression diff --git a/lambdas/shared/src/common/validator/validation_expression_checker.py b/lambdas/shared/src/common/validator/validation_expression_checker.py index b4addaee8d..240f50e925 100644 --- a/lambdas/shared/src/common/validator/validation_expression_checker.py +++ b/lambdas/shared/src/common/validator/validation_expression_checker.py @@ -18,10 +18,10 @@ def __init__(self, code: int, message: str, row: int = None, field: str = None, self.details = details self.summarise = summarise # these are set when the error is added to the report - self.errorGroup = None + self.error_group = None self.name = None self.id = None - self.errorLevel = None + self.error_level = None # function to return the object as a dictionary def to_dict(self): @@ -56,78 +56,78 @@ def __repr__(self): # main expressions checker class ExpressionChecker: # validation settings - summarise = False - report_unexpected_exception = True - dataParser = any - dataLookUp = any - keyData = any - - def __init__(self, dataParser, summarise, report_unexpected_exception): - self.dataParser = dataParser # FHIR data parser for additional functions - self.dataLookUp = LookUpData() # used for generic look up - self.keyData = KeyData() # used for key check on data we know (Snomed / ODS etc) + # summarise = False + # report_unexpected_exception = True + # dataParser = any + # dataLookUp = any + # keyData = any + + def __init__(self, data_parser, summarise, report_unexpected_exception): + self.data_parser = data_parser # FHIR data parser for additional functions + self.data_look_up = LookUpData() # used for generic look up + self.key_data = KeyData() # used for key check on data we know (Snomed / ODS etc) self.summarise = summarise self.report_unexpected_exception = report_unexpected_exception - def validateExpression(self, expressionType, rule, fieldName, fieldValue, row) -> ErrorReport: - match expressionType: + def validateExpression(self, expression_type, rule, field_name, field_value, row) -> ErrorReport: + match expression_type: case "DATETIME": - return self._validateDateTime(rule, fieldName, fieldValue, row) + return self._validate_datetime(rule, field_name, field_value, row) case "DATE": - return self._validateDateTime(rule, fieldName, fieldValue, row) + return self._validate_datetime(rule, field_name, field_value, row) case "UUID": - return self._validateUUID(rule, fieldName, fieldValue, row) + return self._validate_uuid(rule, field_name, field_value, row) case "INT": - return self._validateInteger(rule, fieldName, fieldValue, row) + return self._validate_integer(rule, field_name, field_value, row) case "FLOAT": - return self._validateFloat(rule, fieldName, fieldValue, row) + return self._validate_float(rule, field_name, field_value, row) case "REGEX": - return self._validateRegex(rule, fieldName, fieldValue, row) + return self._validate_regex(rule, field_name, field_value, row) case "EQUAL": - return self._validateEqual(rule, fieldName, fieldValue, row) + return self._validate_equal(rule, field_name, field_value, row) case "NOTEQUAL": - return self._validateNotEqual(rule, fieldName, fieldValue, row) + return self._validate_not_equal(rule, field_name, field_value, row) case "IN": - return self._validateIn(rule, fieldName, fieldValue, row) + return self._validate_in(rule, field_name, field_value, row) case "NRANGE": - return self._validateNRange(rule, fieldName, fieldValue, row) + return self._validate_n_range(rule, field_name, field_value, row) case "INARRAY": - return self._validateInArray(rule, fieldName, fieldValue, row) + return self._validate_in_array(rule, field_name, field_value, row) case "UPPER": - return self._validateUpper(rule, fieldName, fieldValue, row) + return self._validate_upper(rule, field_name, field_value, row) case "LOWER": - return self._validateLower(rule, fieldName, fieldValue, row) + return self._validate_lower(rule, field_name, field_value, row) case "LENGTH": - return self._validateLength(rule, fieldName, fieldValue, row) + return self._validate_length(rule, field_name, field_value, row) case "STARTSWITH": - return self._validateStartsWith(rule, fieldName, fieldValue, row) + return self._validate_starts_with(rule, field_name, field_value, row) case "ENDSWITH": - return self._validateEndsWith(rule, fieldName, fieldValue, row) + return self._validate_ends_with(rule, field_name, field_value, row) case "EMPTY": - return self._validateEmpty(rule, fieldName, fieldValue, row) + return self._validate_empty(rule, field_name, field_value, row) case "NOTEMPTY": - return self._validateNotEmpty(rule, fieldName, fieldValue, row) + return self._validate_not_empty(rule, field_name, field_value, row) case "POSITIVE": - return self._validatePositive(rule, fieldName, fieldValue, row) + return self._validate_positive(rule, field_name, field_value, row) case "POSTCODE": - return self._validatePostCode(rule, fieldName, fieldValue, row) + return self._validate_post_code(rule, field_name, field_value, row) case "GENDER": - return self._validateGender(rule, fieldName, fieldValue, row) + return self._validate_gender(rule, field_name, field_value, row) case "NHSNUMBER": - return self._validateNHSNumber(rule, fieldName, fieldValue, row) + return self._validate_nhs_number(rule, field_name, field_value, row) case "MAXOBJECTS": - return self._validateMaxObjects(rule, fieldName, fieldValue, row) + return self._validate_max_objects(rule, field_name, field_value, row) case "ONLYIF": - return self._validateOnlyIf(rule, fieldName, fieldValue, row) + return self._validate_only_if(rule, field_name, field_value, row) case "LOOKUP": - return self._validateAgainstLookup(rule, fieldName, fieldValue, row) + return self._validate_against_lookup(rule, field_name, field_value, row) case "KEYCHECK": - return self._validateAgainstKey(rule, fieldName, fieldValue, row) + return self._validate_against_key(rule, field_name, field_value, row) case _: - return "Schema expression not found! Check your expression type : " + expressionType + return "Schema expression not found! Check your expression type : " + expression_type # iso8086 date time validate - def _validateDateTime(self, rule, fieldName, fieldValue, row): + def _validate_datetime(self, rule, fieldName, fieldValue, row): try: datetime.date.fromisoformat(fieldValue) except RecordError as e: @@ -143,77 +143,60 @@ def _validateDateTime(self, rule, fieldName, fieldValue, row): return RecordError(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # UUID validate - def _validateUUID(self, expressionRule, fieldName, fieldValue, row): + def _validate_uuid(self, expressionRule, field_name, field_value, row): try: - uuid.UUID(str(fieldValue)) + uuid.UUID(str(field_value)) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Integer Validate - def _validateInteger(self, expressionRule, fieldName, - fieldValue, row, summarise) -> ErrorReport: + def _validate_integer(self, expression_rule, field_name, + field_value, row, summarise) -> ErrorReport: try: - int(fieldValue) + int(field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - if summarise: - p = RecordError(code, message) - else: - p = RecordError(code, message, row, fieldName, details) - return p + return RecordError(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - if summarise: - p = RecordError(ExceptionMessages.UNEXPECTED_EXCEPTION, message) - else: - p = RecordError(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '') - return p + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Float Validate - def _validateFloat(self, expressionRule, fieldName, fieldValue, row, summarise): + def _validate_float(self, expression_rule, field_name, field_value, row, summarise): try: - float(fieldValue) + float(field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - if summarise: - p = {'code': code, 'message': message} - else: - p = {'code': code, 'message': message, 'row': row, 'field': fieldName, 'details': details} - return p + return RecordError(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - if summarise: - p = {'code': ExceptionMessages.UNEXPECTED_EXCEPTION, 'message': message} - else: - p = {'code': ExceptionMessages.UNEXPECTED_EXCEPTION, 'message': message, - 'row': row, 'field': fieldName, 'details': ''} - return p + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Length Validate - def _validateLength(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_length(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: - strLen = len(fieldValue) - checklength = int(expressionRule) - if strLen > checklength: + str_len = len(field_value) + check_length = int(expression_rule) + if str_len > check_length: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value length check failed", "Value is longer than expected") @@ -224,16 +207,16 @@ def _validateLength(self, expressionRule, fieldName, fieldValue, row) -> ErrorR else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Regex Validate - def _validateRegex(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_regex(self, expression_rule, field_name, field_value, row, summarise) -> ErrorReport: try: - result = re.search(expressionRule, fieldValue) + result = re.search(expression_rule, field_value) if not result: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "String REGEX check failed", "Value does not meet regex rules") @@ -245,27 +228,19 @@ def _validateRegex(self, expressionRule, fieldName, fieldValue, row, summarise) ) if e.details is not None: details = e.details - if summarise: - p = ErrorReport(code, message) - else: - p = ErrorReport(code, message, row, fieldName, details) - return p + return RecordError(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - if summarise: - p = ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message) - else: - p = ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '') - return p + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Equal Validate - def _validateEqual(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_equal(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: - if fieldValue != expressionRule: + if field_value != expression_rule: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value equals check failed", - "Value does not equal expected value, Expected- " + expressionRule + " found- " - + fieldValue) + "Value does not equal expected value, Expected- " + expression_rule + " found- " + + field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = ( @@ -273,62 +248,62 @@ def _validateEqual(self, expressionRule, fieldName, fieldValue, row) -> ErrorRe else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Not Equal Validate - def _validateNotEqual(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_not_equal(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: - if fieldValue == expressionRule: + if field_value == expression_rule: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value not equals check failed", - "Value equals expected value when it should not, Expected- " + expressionRule - + " found- " + fieldValue) + "Value equals expected value when it should not, Expected- " + expression_rule + + " found- " + field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # In Validate - def _validateIn(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_in(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: - if expressionRule.lower() in fieldValue.lower(): + if expression_rule.lower() in field_value.lower(): raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Data not in Value failed", "Check Data not found in Value, List- " - + expressionRule + " Data- " + fieldValue) + + expression_rule + " Data- " + field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # NRange Validate - def _validateNRange(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_n_range(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: - value = float(fieldValue) - rule = expressionRule.split(",") + value = float(field_value) + rule = expression_rule.split(",") range1 = float(rule[0]) range2 = float(rule[1]) if range1 <= value >= range2: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value range check failed", "Value is not within the number range, data- " - + fieldValue) + + field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = ( @@ -336,18 +311,18 @@ def _validateNRange(self, expressionRule, fieldName, fieldValue, row) -> ErrorR else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # InArray Validate - def _validateInArray(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_in_array(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: - ruleList = expressionRule.split(",") + rule_list = expression_rule.split(",") - if fieldValue not in ruleList: + if field_value not in rule_list: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value not in array check failed", "Check Value not found in data array") except RecordError as e: @@ -356,118 +331,118 @@ def _validateInArray(self, expressionRule, fieldName, fieldValue, row) -> Error else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Upper Validate - def _validateUpper(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_upper(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: - result = fieldValue.isupper() + result = field_value.isupper() if not result: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value not uppercase", "Check Value not found to be uppercase, value- " + fieldValue) + "Value not uppercase", "Check Value not found to be uppercase, value- " + field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Lower Validate - def _validateLower(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_lower(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: - result = fieldValue.islower() + result = field_value.islower() if not result: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value not lowercase", - "Check Value not found to be lowercase, data- " + fieldValue) + "Check Value not found to be lowercase, data- " + field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Starts With Validate - def _validateStartsWith(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_starts_with(self, expression_rule, field_name, field_value, row, summarise) -> ErrorReport: try: - result = fieldValue.startswith(expressionRule) + result = field_value.startswith(expression_rule) if not result: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value starts with failure", - "Value does not start as expected, Expected- " + expressionRule - + " found- " + fieldValue) + "Value does not start as expected, Expected- " + expression_rule + + " found- " + field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Ends With Validate - def _validateEndsWith(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_ends_with(self, expression_rule, field_name, field_value, row, summarise) -> ErrorReport: try: - result = fieldValue.endswith(expressionRule) + result = field_value.endswith(expression_rule) if not result: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value ends with failure", - "Value does not end as expected, Expected- " + expressionRule - + " found- " + fieldValue) + "Value does not end as expected, Expected- " + expression_rule + + " found- " + field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Empty Validate - def _validateEmpty(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_empty(self, expression_rule, field_name, field_value, row, summarise) -> ErrorReport: try: - if fieldValue: + if field_value: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value is empty failure", - "Value has data, not as expected, data- " + fieldValue) + "Value has data, not as expected, data- " + field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Not Empty Validate - def _validateNotEmpty(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_not_empty(self, expression_rule, field_name, field_value, row, summarise) -> ErrorReport: try: - if not fieldValue: + if not field_value: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value not empty failure", "Value is empty, not as expected") @@ -477,14 +452,14 @@ def _validateNotEmpty(self, expressionRule, fieldName, fieldValue, row, summari else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return ErrorReport(code, message, row, fieldName, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Positive Validate - def _validatePositive(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_positive(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: try: value = float(fieldValue) if value < 0: @@ -504,7 +479,7 @@ def _validatePositive(self, expressionRule, fieldName, fieldValue, row, summari return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # NHSNumber Validate - def _validateNHSNumber(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_nhs_number(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: try: regexRule = '^6[0-9]{10}$' result = re.search(regexRule, fieldValue) @@ -525,7 +500,7 @@ def _validateNHSNumber(self, expressionRule, fieldName, fieldValue, row, summar return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # Gender Validate - def _validateGender(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_gender(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: try: ruleList = ['0', '1', '2', '9'] @@ -546,7 +521,7 @@ def _validateGender(self, expressionRule, fieldName, fieldValue, row, summarise return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # PostCode Validate - def _validatePostCode(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_post_code(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: try: regexRule = '^([Gg][Ii][Rr] 0[Aa]{2})|((([A-Za-z][0-9]{1,2})|(([A-Za-z][A-Ha-hJ-Yj-y]' '[0-9]{1,2})|(([AZa-z][0-9][A-Za-z])|([A-Za-z][A-Ha-hJ-Yj-y][0-9]?[A-Za-z])))) [0-9][A-Za-z]{2})$' @@ -568,7 +543,7 @@ def _validatePostCode(self, expressionRule, fieldName, fieldValue, row, summari return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # Max Objects Validate - def _validateMaxObjects(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_max_objects(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: try: value = len(fieldValue) if value > int(expressionRule): @@ -588,7 +563,7 @@ def _validateMaxObjects(self, expressionRule, fieldName, fieldValue, row, summa return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # Default to Validate - def _validateOnlyIf(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_only_if(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: conversionList = expressionRule.split("|") location = conversionList[0] @@ -612,7 +587,7 @@ def _validateOnlyIf(self, expressionRule, fieldName, fieldValue, row) -> ErrorRe return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # Check with Lookup - def _validateAgainstLookup(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_against_lookup(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: try: result = self.dataLookUp.findLookUp(fieldValue) if not result: @@ -632,7 +607,7 @@ def _validateAgainstLookup(self, expressionRule, fieldName, fieldValue, row, su return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # Check with Key Lookup - def _validateAgainstKey(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_against_key(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: try: result = self.KeyData.findKey(expressionRule, fieldValue) if not result: diff --git a/lambdas/shared/src/common/validator/validator.py b/lambdas/shared/src/common/validator/validator.py index 377bc2a451..5907f8b8e5 100644 --- a/lambdas/shared/src/common/validator/validator.py +++ b/lambdas/shared/src/common/validator/validator.py @@ -12,7 +12,7 @@ FilePath = '' JSONData = {} -SchemaFile = {} +# SchemaFile = {} # Doesnt seem to be used CSVRow = '' CSVHeader = '' error_records: list[ErrorReport] = [] @@ -23,102 +23,105 @@ class Validator: - def __init__(self, filepath, JSONData, schemafile, CSVRow, CSVHeader, dataType): - self.FilePath = filepath - self.JSONData = JSONData - self.SchemaFile = schemafile - self.CSVRow = CSVRow - self.CSVHeader = CSVHeader - self.dataType = dataType - - def _getCSVLineParser(self, CSVRow, CSVHeader): - csvParser = CSVLineParser() - csvParser.parseCSVLine(CSVRow, CSVHeader) - return csvParser - - def _getCSVParser(self, filepath): - csvParser = CSVParser() - csvParser.parseCSVFile(filepath) - return csvParser - - def _getFHIRParser(self, filepath): - fhirParser = FHIRParser() - fhirParser.parseFHIRFile(filepath) - return fhirParser - - def _getFHIRJSONParser(self, FHIRData): - fhirParser = FHIRParser() - fhirParser.parseFHIRData(FHIRData) - return fhirParser - - def _getSchemaParser(self, schemafile): - schemaParser = SchemaParser() - schemaParser.parseSchema(schemafile) - return schemaParser - - def _addErrorRecord(self, errorRecord: ErrorReport, expressionErrorGroup, expressionName, expressionId, errorLevel): - if errorRecord is not None: - errorRecord.errorGroup = expressionErrorGroup - errorRecord.name = expressionName - errorRecord.id = expressionId - errorRecord.errorLevel = errorLevel - error_records.append(errorRecord) + def __init__(self, filepath, json_data, schemafile, csv_row, csv_header, data_type): + self.filepath = filepath + self.json_data = json_data + self.schema_file = schemafile + self.csv_row = csv_row + self.csv_header = csv_header + self.data_type = data_type + + def _get_csv_line_parser(self, csv_row, csv_header): + csv_parser = CSVLineParser() + csv_parser.parse_csv_line(csv_row, csv_header) + return csv_parser + + def _get_csv_parser(self, filepath): + csv_parser = CSVParser() + csv_parser.parse_csv_file(filepath) + return csv_parser + + def _get_fhir_parser(self, filepath): + fhir_parser = FHIRParser() + fhir_parser.parse_fhir_file(filepath) + return fhir_parser + + def _get_fhir_json_parser(self, fhir_data): + fhir_parser = FHIRParser() + fhir_parser.parse_fhir_data(fhir_data) + return fhir_parser + + def _get_schema_parser(self, schemafile): + schema_parser = SchemaParser() + schema_parser.parse_schema(schemafile) + return schema_parser + + def _add_error_record(self, error_record: ErrorReport, + expression_error_group, expression_name, expression_id, error_level): + if error_record is not None: + error_record.error_group = expression_error_group + error_record.name = expression_name + error_record.id = expression_id + error_record.error_level = error_level + error_records.append(error_record) # Function to help identify a parent failure in the error list - def _checkErrorRecordForFailure(self, expressionId): - for errorRecord in error_records: - if (errorRecord.id == expressionId): + def _check_error_record_for_fail(self, expression_id): + for error_record in error_records: + if (error_record.id == expression_id): return True return False # validate a single expression against the data file - def _validateExpression(self, ExpressionValidate, expression, - inc_header_in_row_count) -> ErrorReport | int: + def _validate_expression(self, expression_validate, expression, + inc_header_in_row_count) -> ErrorReport | int: row = 1 if inc_header_in_row_count: row = 2 if self.isCSV: - expressionFieldName = expression['fieldNameCSV'] + expression_fieldname = expression['fieldNameCSV'] else: - expressionFieldName = expression['fieldNameFHIR'] + expression_fieldname = expression['fieldNameFHIR'] - expressionId = expression['expressionId'] - errorLevel = expression['errorLevel'] - expressionName = expression['expression']['expressionName'] - expressionType = expression['expression']['expressionType'] - expressionRule = expression['expression']['expressionRule'] - expressionErrorGroup = expression['errorGroup'] + expression_id = expression['expressionId'] + error_level = expression['errorLevel'] + expression_name = expression['expression']['expressionName'] + expression_type = expression['expression']['expressionType'] + expression_rule = expression['expression']['expressionRule'] + expression_error_group = expression['errorGroup'] # Check to see if the expression has a parent, if so did the parent validate if ('parentExpression' in expression): - parentExpression = expression['parentExpression'] - if (self._checkErrorRecordForFailure(parentExpression)): - errorRecord = {'code': ExceptionMessages.PARENT_FAILED, - 'message': ExceptionMessages.MESSAGES[ExceptionMessages.PARENT_FAILED] - + ', Parent ID: ' + parentExpression} - self._addErrorRecord(errorRecord, expressionErrorGroup, expressionName, expressionId, errorLevel) - return errorRecord + parent_expression = expression['parentExpression'] + if (self._check_error_record_for_fail(parent_expression)): + error_record = {'code': ExceptionMessages.PARENT_FAILED, + 'message': ExceptionMessages.MESSAGES[ExceptionMessages.PARENT_FAILED] + + ', Parent ID: ' + parent_expression} + self._add_error_record(error_record, expression_error_group, expression_name, expression_id, error_level) + return error_record try: - expressionValues = self.dataParser.getKeyValue(expressionFieldName) + expression_values = self.data_parser.getKeyValue(expression_fieldname) except Exception as e: message = 'Data get values Unexpected exception [%s]: %s' % (e.__class__.__name__, e) error_report = ErrorReport(code=ExceptionMessages.PARSING_ERROR, message=message) - self._addErrorRecord(error_report, expressionErrorGroup, expressionName, expressionId, self.CriticalErrorLevel) + self._addErrorRecord(error_report, + expression_error_group, expression_name, expression_id, self.CriticalErrorLevel) return error_report - for value in expressionValues: - errorRecord: ErrorReport = ExpressionValidate.validateExpression(expressionType, expressionRule, - expressionFieldName, value, row) - if errorRecord is not None: - self._addErrorRecord(errorRecord, expressionErrorGroup, expressionName, expressionId, errorLevel) + for value in expression_values: + error_record: ErrorReport = expression_validate.validate_expression( + expression_type, expression_rule, expression_fieldname, value, row) + if error_record is not None: + self._addErrorRecord(error_record, expression_error_group, + expression_name, expression_id, error_level) row += 1 return row # run the validation against the data - def runValidation(self, summarise=False, report_unexpected_exception=True, - inc_header_in_row_count=True) -> list[ErrorReport]: + def run_validation(self, summarise=False, report_unexpected_exception=True, + inc_header_in_row_count=True) -> list[ErrorReport]: try: error_records.clear() @@ -149,7 +152,7 @@ def runValidation(self, summarise=False, report_unexpected_exception=True, return [ErrorReport(code=0, message=message)] try: - ExpressionValidate = ExpressionChecker(dataParser, summarise, report_unexpected_exception) + expression_validate = ExpressionChecker(dataParser, summarise, report_unexpected_exception) except Exception as e: if report_unexpected_exception: message = 'Expression Checker Unexpected exception [%s]: %s' % (e.__class__.__name__, e) @@ -164,8 +167,8 @@ def runValidation(self, summarise=False, report_unexpected_exception=True, return [ErrorReport(code=0, message=message)] for expression in expressions: - # rows = self._validateExpression(ExpressionValidate, expression, inc_header_in_row_count) - self._validateExpression(ExpressionValidate, expression, inc_header_in_row_count) + # rows = self._validate_expression(expression_validate, expression, inc_header_in_row_count) + self._validate_expression(expression_validate, expression, inc_header_in_row_count) return error_records From 45bc465cf06bf107cf3d2d89347222f235689871 Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Fri, 10 Oct 2025 14:41:57 +0100 Subject: [PATCH 03/13] linted --- .../common/validator/enums/error_levels.py | 2 + .../validator/enums/exception_messages.py | 1 - .../src/common/validator/lookup/key_data.py | 14 +- .../common/validator/lookup/lookup_data.py | 2 + .../validator/parsers/csv_line_parser.py | 10 +- .../common/validator/parsers/csv_parser.py | 1 + .../common/validator/parsers/fhir_parser.py | 32 ++-- .../common/validator/parsers/schema_parser.py | 1 + .../common/validator/reporter/dq_reporter.py | 94 +++++----- .../validation_expression_checker.py | 8 +- .../shared/src/common/validator/validator.py | 5 +- .../test_common/validator/data/test_data.csv | 29 +++ .../validator/data/test_data_ok.csv | 29 +++ .../validator/data/vaccination.json | 174 ++++++++++++++++++ .../validator/data/vaccination2.json | 115 ++++++++++++ .../validator/test_application_csv.py | 32 ++++ .../validator/test_application_csv_row.py | 38 ++++ .../validator/test_application_fhir.py | 50 +++++ .../test_common/validator/test_parser.py | 17 ++ 19 files changed, 574 insertions(+), 80 deletions(-) create mode 100644 lambdas/shared/tests/test_common/validator/data/test_data.csv create mode 100644 lambdas/shared/tests/test_common/validator/data/test_data_ok.csv create mode 100644 lambdas/shared/tests/test_common/validator/data/vaccination.json create mode 100644 lambdas/shared/tests/test_common/validator/data/vaccination2.json create mode 100644 lambdas/shared/tests/test_common/validator/test_application_csv.py create mode 100644 lambdas/shared/tests/test_common/validator/test_application_csv_row.py create mode 100644 lambdas/shared/tests/test_common/validator/test_application_fhir.py create mode 100644 lambdas/shared/tests/test_common/validator/test_parser.py diff --git a/lambdas/shared/src/common/validator/enums/error_levels.py b/lambdas/shared/src/common/validator/enums/error_levels.py index a2689e53ac..4a5e8d9db3 100644 --- a/lambdas/shared/src/common/validator/enums/error_levels.py +++ b/lambdas/shared/src/common/validator/enums/error_levels.py @@ -1,3 +1,5 @@ + + # all error Levels CRITICAL_ERROR = 0 WARNING = 1 diff --git a/lambdas/shared/src/common/validator/enums/exception_messages.py b/lambdas/shared/src/common/validator/enums/exception_messages.py index d96a839c4e..cbb348ece5 100644 --- a/lambdas/shared/src/common/validator/enums/exception_messages.py +++ b/lambdas/shared/src/common/validator/enums/exception_messages.py @@ -13,7 +13,6 @@ PARENT_FAILED = 11 KEY_CHECK_FAILED = 12 - MESSAGES = { UNEXPECTED_EXCEPTION: 'Unexpected exception [%s]: %s', VALUE_CHECK_FAILED: 'Value check failed.', diff --git a/lambdas/shared/src/common/validator/lookup/key_data.py b/lambdas/shared/src/common/validator/lookup/key_data.py index 61cdf6c4be..bcddbce463 100644 --- a/lambdas/shared/src/common/validator/lookup/key_data.py +++ b/lambdas/shared/src/common/validator/lookup/key_data.py @@ -1,7 +1,9 @@ # --------------------------------------------------------------------------------------------------------- # main conversion lookup + class KeyData: + # data settings def __init__(self): self.procedure = ['956951000000104'] @@ -96,13 +98,13 @@ def __init__(self): '16857009'] # Look up the term for the code - def findKey(self, KeySource, fieldValue): + def findKey(self, key_source, field_value): try: - match KeySource: - case "Procedure": return fieldValue in self.Procedure - case "Organisation": fieldValue in self.Organisation - case "Site": fieldValue in self.Site - case "Route": fieldValue in self.Route + match key_source: + case "Procedure": return field_value in self.Procedure + case "Organisation": field_value in self.Organisation + case "Site": field_value in self.Site + case "Route": field_value in self.Route case _: return False except Exception: return False diff --git a/lambdas/shared/src/common/validator/lookup/lookup_data.py b/lambdas/shared/src/common/validator/lookup/lookup_data.py index 3431ea67f4..47bbf5a07a 100644 --- a/lambdas/shared/src/common/validator/lookup/lookup_data.py +++ b/lambdas/shared/src/common/validator/lookup/lookup_data.py @@ -1,5 +1,7 @@ # --------------------------------------------------------------------------------------------------------- # main conversion lookup + + class LookUpData: # data settings def __init__(self): diff --git a/lambdas/shared/src/common/validator/parsers/csv_line_parser.py b/lambdas/shared/src/common/validator/parsers/csv_line_parser.py index df3f37fe10..989561c02b 100644 --- a/lambdas/shared/src/common/validator/parsers/csv_line_parser.py +++ b/lambdas/shared/src/common/validator/parsers/csv_line_parser.py @@ -8,14 +8,14 @@ def __init__(self): self.csv_file_data = {} # parse the CSV into a Dictionary - def parseCSVLine(self, CSVRow, CSVHeader): + def parse_csv_line(self, csv_row, csv_header): # create a key value mapping - keys = list(csv.reader([CSVHeader]))[0] - values = list(csv.reader([CSVRow]))[0] + keys = list(csv.reader([csv_header]))[0] + values = list(csv.reader([csv_row]))[0] self.csv_file_data = dict(map(lambda i, j: (i, j), keys, values)) # retrieve a column of data to work with - def getKeyValues(self, fieldName): + def get_key_values(self, field_name): # creating empty lists, convert to list - data = [self.csv_file_data[fieldName]] + data = [self.csv_file_data[field_name]] return data diff --git a/lambdas/shared/src/common/validator/parsers/csv_parser.py b/lambdas/shared/src/common/validator/parsers/csv_parser.py index 5c1ca140e9..32544574d4 100644 --- a/lambdas/shared/src/common/validator/parsers/csv_parser.py +++ b/lambdas/shared/src/common/validator/parsers/csv_parser.py @@ -3,6 +3,7 @@ class CSVParser: + """ File Management""" # parser variables def __init__(self): diff --git a/lambdas/shared/src/common/validator/parsers/fhir_parser.py b/lambdas/shared/src/common/validator/parsers/fhir_parser.py index 5b6a1893d3..c7c62c9b37 100644 --- a/lambdas/shared/src/common/validator/parsers/fhir_parser.py +++ b/lambdas/shared/src/common/validator/parsers/fhir_parser.py @@ -5,18 +5,18 @@ class FHIRParser: # parser variables def __init__(self): - self.FHIRResource = {} + self.fhir_resource = {} # ------------------------------------------- # File Management # used for files def parse_fhir_file(self, fhir_file_name): with open(fhir_file_name, 'r') as json_file: - self.FHIRResource = json.load(json_file) + self.fhir_resource = json.load(json_file) # used for JSON FHIR Resource data def parse_fhir_data(self, fhir_data): - self.FHIRResource = fhir_data + self.fhir_resource = fhir_data # ------------------------------------------------ # Scan and Identify @@ -64,13 +64,13 @@ def _get_node(self, parent, child): return result # locate a value for a key - def _scanForValue(self, FHIRFields): - fieldList = FHIRFields.split("|") + def _scan_for_value(self, fhir_fields): + field_list = fhir_fields.split("|") # get root field before we iterate - rootfield = self.FHIRResource[fieldList[0]] - del fieldList[0] + rootfield = self.fhir_resource[field_list[0]] + del field_list[0] try: - for field in fieldList: + for field in field_list: if (field.startswith("#")): rootfield = self._locate_list_id(rootfield, field) # check here for default index?? else: @@ -80,23 +80,23 @@ def _scanForValue(self, FHIRFields): return rootfield # get the value list for a key - def getKeyValue(self, fieldName): + def get_key_value(self, field_name): value = [] try: - responseValue = self._scanForValue(fieldName) + response_value = self._scan_for_value(field_name) except Exception: - responseValue = '' + response_value = '' - value.append(responseValue) + value.append(response_value) return value # get the value list for a key - def getKeySingleValue(self, fieldName): + def get_key_single_value(self, field_name): value = '' try: - responseValue = self._scanForValue(fieldName) + response_value = self._scan_for_value(field_name) except Exception: - responseValue = '' + response_value = '' - value = responseValue + value = response_value return value diff --git a/lambdas/shared/src/common/validator/parsers/schema_parser.py b/lambdas/shared/src/common/validator/parsers/schema_parser.py index 5bce18d9de..6278487a77 100644 --- a/lambdas/shared/src/common/validator/parsers/schema_parser.py +++ b/lambdas/shared/src/common/validator/parsers/schema_parser.py @@ -1,6 +1,7 @@ # Schema Parser # Moved from file loading to JSON string better for elasticache + class SchemaParser: def __init__(self): # parser variables diff --git a/lambdas/shared/src/common/validator/reporter/dq_reporter.py b/lambdas/shared/src/common/validator/reporter/dq_reporter.py index 5866d2c9dc..ed09c9ce43 100644 --- a/lambdas/shared/src/common/validator/reporter/dq_reporter.py +++ b/lambdas/shared/src/common/validator/reporter/dq_reporter.py @@ -3,62 +3,64 @@ import validator.enums.error_levels as ErrorLevels from dateutil import parser -ErrorReport = { - "eventId": "", - "validationDate": "", - "validated": 'true', - "results": { - "totalErrors": 0, - "completeness": { - "errors": 0, - "fields": [] - }, - "consistency": { - "errors": 0, - "fields": [] - }, - "validity": { - "errors": 0, - "fields": [] - }, - "timeliness_processed": 0 - } -} - class DQReporter: + def __init__(self): + # parser variables + self.error_report = { + "eventId": "", + "validationDate": "", + "validated": 'true', + "results": { + "totalErrors": 0, + "completeness": { + "errors": 0, + "fields": [] + }, + "consistency": { + "errors": 0, + "fields": [] + }, + "validity": { + "errors": 0, + "fields": [] + }, + "timeliness_processed": 0 + } + } + # create the date difference for the report in minutes def diff_dates(self, date1, date2): - diffSeconds = abs(date2-date1).total_seconds() - diffMinutes = diffSeconds / 60 - return diffMinutes + diff_seconds = abs(date2-date1).total_seconds() + diff_minutes = diff_seconds / 60 + return diff_minutes - def generateErrorReport(self, eventId, Occurrence, error_records): - occurenceDate = Occurrence - occurenceDate = parser.parse(occurenceDate, ignoretz=True) - validationDate = datetime.datetime.now(tz=None) + def generate_error_report(self, event_id, occurrence, error_records): + occurrence_date = occurrence + occurrence_date = parser.parse(occurrence_date, ignoretz=True) + validation_date = datetime.datetime.now(tz=None) - timeTaken = self.diff_dates(occurenceDate, validationDate) + time_taken = self.diff_dates(occurrence_date, validation_date) - ErrorReport['validationDate'] = validationDate.isoformat() - ErrorReport['eventId'] = eventId - ErrorReport['results']['timeliness_processed'] = timeTaken + self.error_report['validationDate'] = validation_date.isoformat() + self.error_report['eventId'] = event_id + self.error_report['results']['timeliness_processed'] = time_taken for errorRecord in error_records: self.updateReport(errorRecord) - jsonErrorReport = json.dumps(ErrorReport) - return jsonErrorReport + json_error_report = json.dumps(self.error_report) + return json_error_report - def updateReport(self, errorData): - errorGroup = errorData["errorGroup"] - if (errorData['errorLevel'] == ErrorLevels.CRITICAL_ERROR): - ErrorReport['validated'] = "false" - totalErrors = ErrorReport['results']['totalErrors'] - resultsErrorCount = ErrorReport['results'][errorGroup]['errors'] - resultsErrorCount += 1 - totalErrors += 1 - ErrorReport['results'][errorGroup]['fields'].append(errorData["name"]) - ErrorReport['results'][errorGroup]['errors'] = resultsErrorCount - ErrorReport['results']['totalErrors'] = totalErrors + def update_report(self, error_data): + error_group = error_data["errorGroup"] + if (error_data['errorLevel'] == ErrorLevels.CRITICAL_ERROR): + self.error_report['validated'] = "false" + total_errors = self.error_report['results']['totalErrors'] + results_error_count = self.error_report['results'][error_group]['errors'] + results_error_count += 1 + total_errors += 1 + self.error_report['results'][error_group]['fields'].append(error_data["name"]) + self.error_report['results'][error_group]['errors'] = results_error_count + self.error_report['results']['totalErrors'] = total_errors diff --git a/lambdas/shared/src/common/validator/validation_expression_checker.py b/lambdas/shared/src/common/validator/validation_expression_checker.py index 240f50e925..a57f636f7a 100644 --- a/lambdas/shared/src/common/validator/validation_expression_checker.py +++ b/lambdas/shared/src/common/validator/validation_expression_checker.py @@ -127,20 +127,20 @@ def validateExpression(self, expression_type, rule, field_name, field_value, ro return "Schema expression not found! Check your expression type : " + expression_type # iso8086 date time validate - def _validate_datetime(self, rule, fieldName, fieldValue, row): + def _validate_datetime(self, rule, field_name, field_value, row): try: - datetime.date.fromisoformat(fieldValue) + datetime.date.fromisoformat(field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return RecordError(code, message, row, fieldName, details, self.summarise) + return RecordError(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return RecordError(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return RecordError(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # UUID validate def _validate_uuid(self, expressionRule, field_name, field_value, row): diff --git a/lambdas/shared/src/common/validator/validator.py b/lambdas/shared/src/common/validator/validator.py index 5907f8b8e5..24ff02efa1 100644 --- a/lambdas/shared/src/common/validator/validator.py +++ b/lambdas/shared/src/common/validator/validator.py @@ -106,8 +106,9 @@ def _validate_expression(self, expression_validate, expression, except Exception as e: message = 'Data get values Unexpected exception [%s]: %s' % (e.__class__.__name__, e) error_report = ErrorReport(code=ExceptionMessages.PARSING_ERROR, message=message) - self._addErrorRecord(error_report, - expression_error_group, expression_name, expression_id, self.CriticalErrorLevel) + # original code had self.CriticalErrorLevel. Replaced with error_level + self._add_error_record(error_report, + expression_error_group, expression_name, expression_id, error_level) return error_report for value in expression_values: diff --git a/lambdas/shared/tests/test_common/validator/data/test_data.csv b/lambdas/shared/tests/test_common/validator/data/test_data.csv new file mode 100644 index 0000000000..d7c1e87fa6 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/test_data.csv @@ -0,0 +1,29 @@ +NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB|PERSON_GENDER_CODE|PERSON_POSTCODE|DATE_AND_TIME|SITE_CODE|SITE_CODE_TYPE_URI|UNIQUE_ID|UNIQUE_ID_URI|ACTION_FLAG|PERFORMING_PROFESSIONAL_FORENAME|PERFORMING_PROFESSIONAL_SURNAME|RECORDED_DATE|PRIMARY_SOURCE|VACCINATION_PROCEDURE_CODE|VACCINATION_PROCEDURE_TERM|DOSE_SEQUENCE|VACCINE_PRODUCT_CODE|VACCINE_PRODUCT_TERM|VACCINE_MANUFACTURER|BATCH_NUMBER|EXPIRY_DATE|SITE_OF_VACCINATION_CODE|SITE_OF_VACCINATION_TERM|ROUTE_OF_VACCINATION_CODE|ROUTE_OF_VACCINATION_TERM|DOSE_AMOUNT|DOSE_UNIT_CODE|DOSE_UNIT_TERM|INDICATION_CODE|LOCATION_CODE|LOCATION_CODE_TYPE_URI|REDUCE_VALIDATION_CODE|REDUCE_VALIDATION_REASON +"9674963871"|"SABINA"|"GREIR"|"20190131"|"2"|"GU14 6TU"|"20240610T183325"|"J82067"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0001_RSV_v5_RUN_2_CDFDPS-742_valid_dose_1"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"J82067"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9454195417"|"SHAQUILLE"|"KOLBERG"|"20110513"|"1"|"DN17 1SL"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0002_RSV_v5_RUN_2_CDFDPS-742_Dose_seq_05"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9730117489"|"STUART"|"RUDD"|"20011114"|"1"|"RG2 0JA"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0003_RSV_v5_RUN_2_CDFDPS-742_Dose_seq_09"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9694578744"|"JON EDMUND"|"HUGHES"|"19811022"|"1"|"DN37 9NJ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1012_RSV_v5_RUN_2_CDFDPS-742_Detained_Estate"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264283"|"JULIA"|"VERNON"|"19990712"|"2"|"ZZ99 3CZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1013_RSV_v5_RUN_2_CDFDPS-742_ANS_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264275"|"CASEY"|"MUMBY"|"19970921"|"9"|"ZZ99 3WZ"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1014_RSV_v5_RUN_2_CDFDPS-742_ANK_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264267"|"TESSA"|"MORREY"|"19940626"|"2"|"ZZ99 3VZ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1015_RSV_v5_RUN_2_CDFDPS-742_NFA_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244932"|"STAN"|"TEMPLE"|"19990120"|"1"|"IM1 2BT"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1016_RSV_v5_RUN_2_CDFDPS-742_IOM_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244924"|"SAM"|"SELLY"|"20000516"|"1"|"LS26 0TW"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1017_RSV_v5_RUN_2_CDFDPS-742_English_PC_IOM_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244916"|"DAVID"|"DUDLEY"|"19901001"|"1"|"IM1 2BT"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1018_RSV_v5_RUN_2_CDFDPS-742_IOM_PC_IOM_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244908"|"AMOS"|"WESTON"|"19900906"|"1"|"BT9 7GW"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1019_RSV_v5_RUN_2_CDFDPS-742_NI_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244886"|"VERNON"|"MCGURK"|"20010808"|"1"|"BT9 7GW"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1020_RSV_v5_RUN_2_CDFDPS-742_NI_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244878"|"CONRAD"|"FRENCH"|"20040827"|"1"|"EH3 7HA"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1021_RSV_v5_RUN_2_CDFDPS-742_Scottish_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244843"|"ROLAND"|"MACVAY"|"20030221"|"1"|"EH3 7HA"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1022_RSV_v5_RUN_2_CDFDPS-742_Scottish_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244835"|"GEORGE"|"HUNT"|"19900212"|"1"|"SA15 1SU"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1023_RSV_v5_RUN_2_CDFDPS-742_Welsh_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244819"|"LESTER"|"LISTER"|"20021007"|"1"|"SA43 1BU"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1025_RSV_v5_RUN_2_CDFDPS-742_Welsh_GP_Welsh_PC"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244851"|"THOMAS"|"PETTIT"|"20040809"|"1"|"WF3 3AP"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1026_RSV_v5_RUN_2_CDFDPS-742_English_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9467361590"|"WALLIS"|"ADEYEMO"|"20150419"|"1"|"LS6 1BQ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1027_RSV_v5_RUN_2_CDFDPS-742_Merged_patient_Old_NHS_NUMBER"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +|"DINAH"|"MARKIE"|"20110216"|"2"|"DN17 2QZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1028_RSV_v5_RUN_2_CDFDPS-742_Merged_patient_No_NHS_NUMBER"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9446463580"|"ANNABETH"|"BURNS"|"19800303"|"2"|"BL9 7ST"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1029_RSV_v5_RUN_2_CDFDPS-742_Deceased_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9451924108"|"NESSA"|"DENTON"|"20100530"|"0"|"FY8 3HL"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1030_RSV_v5_RUN_2_CDFDPS-742_Sensitive_Y_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9476111771"|"LEAH"|"HAY"|"20160508"|"0"|"CA12 5JT"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1031_RSV_v5_RUN_2_CDFDPS-742_Sensitive_I_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9730117489"|"STUART"|"RUDD"|"20011114"|"0"|"RG2 0JA"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1032_RSV_v5_RUN_2_CDFDPS-742"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9694561140"|"LILLY"|"BUXTON"|"20120810"|"2"|"DN34 4PA"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1033_RSV_v5_RUN_2_CDFDPS-742_MOD_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9475768947"|"IVY"|"ROSSI"|"20160109"|"2"|"BD8 7NS"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1034_RSV_v5_RUN_2_CDFDPS-742_Sensitive_S_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264232"|"DEBRA"|"HESTER"|"19990530"|"2"|"ZZ99 3VZ"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1035_RSV_v5_RUN_2_CDFDPS-742_NFA_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264240"|"ELIAS"|"REID"|"19920313"|"1"|"ZZ99 3WZ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1036_RSV_v5_RUN_2_CDFDPS-742_ANK_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264259"|"RHAN"|"KENYON"|"19930902"|"2"|"ZZ99 3CZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1037_RSV_v5_RUN_2_CDFDPS-742_ANS_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/data/test_data_ok.csv b/lambdas/shared/tests/test_common/validator/data/test_data_ok.csv new file mode 100644 index 0000000000..d7c1e87fa6 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/test_data_ok.csv @@ -0,0 +1,29 @@ +NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB|PERSON_GENDER_CODE|PERSON_POSTCODE|DATE_AND_TIME|SITE_CODE|SITE_CODE_TYPE_URI|UNIQUE_ID|UNIQUE_ID_URI|ACTION_FLAG|PERFORMING_PROFESSIONAL_FORENAME|PERFORMING_PROFESSIONAL_SURNAME|RECORDED_DATE|PRIMARY_SOURCE|VACCINATION_PROCEDURE_CODE|VACCINATION_PROCEDURE_TERM|DOSE_SEQUENCE|VACCINE_PRODUCT_CODE|VACCINE_PRODUCT_TERM|VACCINE_MANUFACTURER|BATCH_NUMBER|EXPIRY_DATE|SITE_OF_VACCINATION_CODE|SITE_OF_VACCINATION_TERM|ROUTE_OF_VACCINATION_CODE|ROUTE_OF_VACCINATION_TERM|DOSE_AMOUNT|DOSE_UNIT_CODE|DOSE_UNIT_TERM|INDICATION_CODE|LOCATION_CODE|LOCATION_CODE_TYPE_URI|REDUCE_VALIDATION_CODE|REDUCE_VALIDATION_REASON +"9674963871"|"SABINA"|"GREIR"|"20190131"|"2"|"GU14 6TU"|"20240610T183325"|"J82067"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0001_RSV_v5_RUN_2_CDFDPS-742_valid_dose_1"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"J82067"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9454195417"|"SHAQUILLE"|"KOLBERG"|"20110513"|"1"|"DN17 1SL"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0002_RSV_v5_RUN_2_CDFDPS-742_Dose_seq_05"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9730117489"|"STUART"|"RUDD"|"20011114"|"1"|"RG2 0JA"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"0003_RSV_v5_RUN_2_CDFDPS-742_Dose_seq_09"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9694578744"|"JON EDMUND"|"HUGHES"|"19811022"|"1"|"DN37 9NJ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1012_RSV_v5_RUN_2_CDFDPS-742_Detained_Estate"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264283"|"JULIA"|"VERNON"|"19990712"|"2"|"ZZ99 3CZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1013_RSV_v5_RUN_2_CDFDPS-742_ANS_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264275"|"CASEY"|"MUMBY"|"19970921"|"9"|"ZZ99 3WZ"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1014_RSV_v5_RUN_2_CDFDPS-742_ANK_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264267"|"TESSA"|"MORREY"|"19940626"|"2"|"ZZ99 3VZ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1015_RSV_v5_RUN_2_CDFDPS-742_NFA_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244932"|"STAN"|"TEMPLE"|"19990120"|"1"|"IM1 2BT"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1016_RSV_v5_RUN_2_CDFDPS-742_IOM_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244924"|"SAM"|"SELLY"|"20000516"|"1"|"LS26 0TW"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1017_RSV_v5_RUN_2_CDFDPS-742_English_PC_IOM_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244916"|"DAVID"|"DUDLEY"|"19901001"|"1"|"IM1 2BT"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1018_RSV_v5_RUN_2_CDFDPS-742_IOM_PC_IOM_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244908"|"AMOS"|"WESTON"|"19900906"|"1"|"BT9 7GW"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1019_RSV_v5_RUN_2_CDFDPS-742_NI_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244886"|"VERNON"|"MCGURK"|"20010808"|"1"|"BT9 7GW"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1020_RSV_v5_RUN_2_CDFDPS-742_NI_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244878"|"CONRAD"|"FRENCH"|"20040827"|"1"|"EH3 7HA"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1021_RSV_v5_RUN_2_CDFDPS-742_Scottish_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244843"|"ROLAND"|"MACVAY"|"20030221"|"1"|"EH3 7HA"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1022_RSV_v5_RUN_2_CDFDPS-742_Scottish_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244835"|"GEORGE"|"HUNT"|"19900212"|"1"|"SA15 1SU"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1023_RSV_v5_RUN_2_CDFDPS-742_Welsh_PC_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244819"|"LESTER"|"LISTER"|"20021007"|"1"|"SA43 1BU"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1025_RSV_v5_RUN_2_CDFDPS-742_Welsh_GP_Welsh_PC"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728244851"|"THOMAS"|"PETTIT"|"20040809"|"1"|"WF3 3AP"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1026_RSV_v5_RUN_2_CDFDPS-742_English_PC_No_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9467361590"|"WALLIS"|"ADEYEMO"|"20150419"|"1"|"LS6 1BQ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1027_RSV_v5_RUN_2_CDFDPS-742_Merged_patient_Old_NHS_NUMBER"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +|"DINAH"|"MARKIE"|"20110216"|"2"|"DN17 2QZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1028_RSV_v5_RUN_2_CDFDPS-742_Merged_patient_No_NHS_NUMBER"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9446463580"|"ANNABETH"|"BURNS"|"19800303"|"2"|"BL9 7ST"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1029_RSV_v5_RUN_2_CDFDPS-742_Deceased_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9451924108"|"NESSA"|"DENTON"|"20100530"|"0"|"FY8 3HL"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1030_RSV_v5_RUN_2_CDFDPS-742_Sensitive_Y_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9476111771"|"LEAH"|"HAY"|"20160508"|"0"|"CA12 5JT"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1031_RSV_v5_RUN_2_CDFDPS-742_Sensitive_I_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9730117489"|"STUART"|"RUDD"|"20011114"|"0"|"RG2 0JA"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1032_RSV_v5_RUN_2_CDFDPS-742"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9694561140"|"LILLY"|"BUXTON"|"20120810"|"2"|"DN34 4PA"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1033_RSV_v5_RUN_2_CDFDPS-742_MOD_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9475768947"|"IVY"|"ROSSI"|"20160109"|"2"|"BD8 7NS"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1034_RSV_v5_RUN_2_CDFDPS-742_Sensitive_S_flag_patient"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264232"|"DEBRA"|"HESTER"|"19990530"|"2"|"ZZ99 3VZ"|"20240610T193618"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1035_RSV_v5_RUN_2_CDFDPS-742_NFA_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264240"|"ELIAS"|"REID"|"19920313"|"1"|"ZZ99 3WZ"|"20240610T183325"|"RVVKC"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1036_RSV_v5_RUN_2_CDFDPS-742_ANK_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"Ellena"|"O'Reilly"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy +"9728264259"|"RHAN"|"KENYON"|"19930902"|"2"|"ZZ99 3CZ"|"20240610T173135"|"RJ1"|"https://fhir.nhs.uk/Id/ods-organization-code"|"1037_RSV_v5_RUN_2_CDFDPS-742_ANS_English_GP"|"https://www.ravs.england.nhs.uk/"|"new"|"John"|"Greaves"|"20240609"|"TRUE"|"1303503001"|"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"|1|"42605811000001109"|"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product) "|"Pfizer"|"RSVTEST"|"20241231"|"368208006"|"Left upper arm structure (body structure)"|"78421000"|"Intramuscular route (qualifier value)"|"0.5"|"258773002"|"Milliliter (qualifier value)"||"0DEAE"|"https://fhir.nhs.uk/Id/ods-organization-code"|dummy|dummy \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/data/vaccination.json b/lambdas/shared/tests/test_common/validator/data/vaccination.json new file mode 100644 index 0000000000..def97ad4e1 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/vaccination.json @@ -0,0 +1,174 @@ +{ + "resourceType": "Immunization", + "status": "completed", + "protocolApplied": [ + { + "targetDisease": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "14189004", + "display": "Measles (disorder)" + } + ] + }, + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "36989005", + "display": "Mumps (disorder)" + } + ] + }, + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "36653000", + "display": "Rubella (disorder)" + } + ] + } + ], + "doseNumberPositiveInt": 1 + } + ], + "reasonCode": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "1037351000000105" + } + ] + } + ], + "recorded": "2025-09-15T09:49:38+00:00", + "identifier": [ + { + "value": "48a4dde2-929a-47dc-ae9e-0234d6efc40a", + "system": "https://www.ravs.england.nhs.uk/" + } + ], + "patient": { + "reference": "#Patient1" + }, + "contained": [ + { + "id": "Patient1", + "resourceType": "Patient", + "birthDate": "2013-01-08", + "gender": "male", + "address": [ + { + "postalCode": "SP6 4WE" + } + ], + "identifier": [ + { + "system": "https://fhir.nhs.uk/Id/nhs-number", + "value": "9489927848" + } + ], + "name": [ + { + "family": "Taylor", + "given": [ + "Laura" + ] + } + ] + }, + { + "resourceType": "Practitioner", + "id": "Practitioner1", + "name": [ + { + "family": "Taylor", + "given": [ + "Anthony" + ] + } + ] + } + ], + "vaccineCode": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "42223111000001107", + "display": "Influenza Tetra MYL vaccine suspension for injection 0.5ml pre-filled syringes (Mylan)" + } + ] + }, + "manufacturer": { + "display": "Mylan" + }, + "expirationDate": "2025-09-20", + "lotNumber": "BN40379485850", + "extension": [ + { + "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "956951000000104", + "display": "RSV vaccination in pregnancy (procedure)" + } + ] + } + } + ], + "occurrenceDateTime": "2025-09-11T07:14:25+00:00", + "primarySource": true, + "site": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "368209003", + "display": "Right arm" + } + ] + }, + "route": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "1210999013", + "display": "Intradermal use" + } + ] + }, + "doseQuantity": { + "value": 0.3, + "unit": "Inhalation - unit of product usage", + "system": "http://snomed.info/sct", + "code": "2622896019" + }, + "performer": [ + { + "actor": { + "type": "Organization", + "identifier": { + "system": "https://fhir.nhs.uk/Id/ods-organization-code", + "value": "RJ1" + } + } + }, + { + "actor": { + "reference": "#Practitioner1" + } + } + ], + "location": { + "identifier": { + "value": "RJC02", + "system": "https://fhir.nhs.uk/Id/ods-organization-code" + } + }, + "id": "976ccbf1-5ced-4541-8664-3953bf2ff026" +} \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/data/vaccination2.json b/lambdas/shared/tests/test_common/validator/data/vaccination2.json new file mode 100644 index 0000000000..c5bcc102a4 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/vaccination2.json @@ -0,0 +1,115 @@ +{ + "resourceType": "Immunization", + "id": "d11c69d8-7a50-4a54-a848-7648121e995f", + "contained": [ + { + "resourceType": "Patient", + "id" : "Pat1", + "identifier": [ + { + "system": "https://fhir.nhs.uk/Id/nhs-number", + "value": "9000000009" + } + ], + "name": [ + { + "family": "Taylor", + "given": [ + "Sarah" + ] + } + ], + "gender": "unknown", + "birthDate": "1965-02-28", + "address": [ + { + "postalCode": "EC1A 1BB" + } + ] + } + ], + "extension": [ + { + "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "1324681000000101", + "display": "Administration of first dose of severe acute respiratory syndrome coronavirus 2 vaccine (procedure)" + } + ] + } + } + ], + "status": "completed", + "vaccineCode": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "39114911000001105", + "display": "COVID-19 Vaccine Vaxzevria (ChAdOx1 S [recombinant]) not less than 2.5x100,000,000 infectious units/0.5ml dose suspension for injection multidose vials (AstraZeneca UK Ltd) (product)" + } + ] + }, + "patient": { + "reference" : "#Pat1" + }, + "occurrenceDateTime": "2021-02-07T13:28:17.271+00:00", + "recorded": "2021-02-07", + "primarySource": true, + "manufacturer": { + "display": "AstraZeneca Ltd" + }, + "location": { + "identifier": { + "value": "X99999", + "system": "https://fhir.nhs.uk/Id/ods-organization-code" + } + }, + "lotNumber": "4120Z001", + "expirationDate": "2021-07-02", + "doseQuantity": { + "value": 0.5, + "unit": "ml", + "system": "http://snomed.info/sct", + "code": "258773002" + }, + "performer": [ + { + "actor" : { + "type" : "Organization", + "identifier": { + "system": "https://fhir.nhs.uk/Id/ods-organization-code", + "value": "B0C4P" + } + } + } + ], + "reasonCode": [ + { + "coding": [ + { + "code": "443684005", + "system": "http://snomed.info/sct" + } + ] + } + ], + "protocolApplied": [ + { + "targetDisease": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "840539006", + "display": "Disease caused by severe acute respiratory syndrome coronavirus 2 (disorder)" + } + ] + } + ], + "doseNumberPositiveInt": 1 + } + ] + } \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv.py b/lambdas/shared/tests/test_common/validator/test_application_csv.py new file mode 100644 index 0000000000..10ebc84761 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_application_csv.py @@ -0,0 +1,32 @@ +# Test application file +from pathlib import Path +from common.validator.validator import Validator +import json +import time + +csv_data_folder = Path("./data/csv-data/data") +# csvFilePath = csv_data_folder / "test_data.csv" # Medium +csvFilePath = csv_data_folder / "test_data_ok.csv" # Passes + +dataType = 'CSV' + +schema_data_folder = Path("C:/Source Code/CSV Validator/Schemas") +schemaFilePath = schema_data_folder / "test1.json" + + +start = time.time() + +# get the JSON of the schema, changed to cope with elasticache +with open(schemaFilePath, 'r') as JSON: + SchemaFile = json.load(JSON) + +validator = Validator(csvFilePath, SchemaFile, '', '', dataType) +error_report = validator.run_validation(False, True, True) + +if len(error_report) > 0: + print(error_report) +else: + print('Validated Successfully') + +end = time.time() +print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv_row.py b/lambdas/shared/tests/test_common/validator/test_application_csv_row.py new file mode 100644 index 0000000000..235d0bb51d --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_application_csv_row.py @@ -0,0 +1,38 @@ +# Test application file +from pathlib import Path +from validator.validator import Validator +import json +import time + +CSV_HEADER = 'academic_year,time_period,time_identifier,geographic_level,' +'country_code,country_name,region_code,region_name,new_la_code,la_name,' +'old_la_code,school_type,num_schools,enrolments,present_sessions,overall_attendance,' +'approved_educational_activity,overall_absence,authorised_absence,unauthorised_absence,' +'late_sessions,possible_sessions,reason_present_am,reason_present_pm,reason_present,' +'reason_l_present_late_before_registers_closed' +CSV_ROW = '202223,202223,Spring term,Local authority,E92000001,England,E12000004,' +'East Midlands,E06000016,Leicester,856,Primary,66.94915254,23057.94915,2367094,' +'2380687,13593,166808,99826,66982,34090,2547495,1157575,1180365,2337940,29154' +DATA_TYPE = 'CSVROW' + +schema_data_folder = Path("C:/Source Code/CSV Validator/Schemas") +schemaFilePath = schema_data_folder / "test1.json" +# schemaFilePath = schema_data_folder / "test2.json" + +start = time.time() + +# get the JSON of the schema, moved from internal file to json string +# changed to cope with elasticache +with open(schemaFilePath, 'r') as JSON: + SchemaFile = json.load(JSON) + +Validator = Validator('', SchemaFile, CSV_ROW, CSV_HEADER, DATA_TYPE) +ErrorReport = Validator.runValidation(True, True, True) + +if len(ErrorReport) > 0: + print(ErrorReport) +else: + print('Validated Successfully') + +end = time.time() +print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/test_application_fhir.py b/lambdas/shared/tests/test_common/validator/test_application_fhir.py new file mode 100644 index 0000000000..7b63f4e7f0 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_application_fhir.py @@ -0,0 +1,50 @@ +# Test application file +from pathlib import Path +from validator.validator import Validator +import json +import time + + +FHIR_data_folder = Path("C:/Source Code/CSV Validator/FHIR-data") +FHIRFilePath = FHIR_data_folder / "vaccination.json" + +schema_data_folder = Path("C:/Source Code/CSV Validator/Schemas") +schemaFilePath = schema_data_folder / "schema.json" + +DATA_TYPE = 'FHIRJSON' # 'FHIR', 'FHIRJSON', 'CSV', 'CSVROW' + +start = time.time() + +# get the JSON of the schema, changed to cope with elasticache +with open(schemaFilePath, 'r') as JSON: + SchemaFile = json.load(JSON) + +# get the FHIR Data as JSON +with open(FHIRFilePath, 'r') as JSON: + FHIRData = json.load(JSON) + + +validator = Validator(FHIRFilePath, FHIRData, SchemaFile, '', '', DATA_TYPE) # FHIR File Path not needed +error_list = validator.runValidation(True, True, True) +error_report = validator.buildErrorReport('25a8cc4d-1875-4191-ac6d-2d63a0ebc64b') # include the eventID if known + +failed_validation = validator.hasValidationFailed() + +# if len(ErrorList) > 0: +# print(ErrorList) +# else: +# print('Validated Successfully') + +print('--------------------------------------------------------------------') +print(error_report) +print('--------------------------------------------------------------------') + +if (failed_validation): + print('Validation failed due to a critical validation failure...') +else: + print('Validation Successful, see reports for details') + + +end = time.time() +print('Time Taken : ') +print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/test_parser.py b/lambdas/shared/tests/test_common/validator/test_parser.py new file mode 100644 index 0000000000..dbcb684f4d --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_parser.py @@ -0,0 +1,17 @@ +# Test application file +from pathlib import Path +from validator.parsers.fhir_parser import FHIRParser +import time + +fhir_data_folder = Path("./data") +fhirFilePath = fhir_data_folder / "vaccination.json" + +start = time.time() + +fhir_parser = FHIRParser() +fhir_parser.parse_fhir_file(fhirFilePath) +my_value = fhir_parser.get_key_value('vaccineCode|coding|0|code') +print('Value = ', my_value) + +end = time.time() +print('Time to Run : ', end - start) From cfa3ad11ae9c52565138c0d8bbee6614be44f782 Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Mon, 13 Oct 2025 20:00:35 +0100 Subject: [PATCH 04/13] unit tests --- immunisation-fhir-api.code-workspace | 70 +++---- lambdas/shared/Makefile | 7 +- .../shared/src/common/validator/__init__.py | 0 .../src/common/validator/enums/__init__.py | 0 .../common/validator/enums/error_levels.py | 1 + .../validator/enums/exception_messages.py | 2 + ...ssion_checker.py => expression_checker.py} | 83 +++----- .../src/common/validator/lookup/__init__.py | 0 .../src/common/validator/parsers/__init__.py | 0 .../src/common/validator/record_error.py | 46 +++++ .../src/common/validator/reporter/__init__.py | 0 .../common/validator/reporter/dq_reporter.py | 2 +- .../shared/src/common/validator/validator.py | 76 ++++--- lambdas/shared/tests/__init__.py | 0 .../tests/test_common/validator/__init__.py | 0 ...ication_csv.py => _est_application_csv.py} | 0 ...sv_row.py => _test_application_csv_row.py} | 2 +- ...tion_fhir.py => _test_application_fhir.py} | 2 +- .../test_common/validator/schemas/Schema.json | 188 ++++++++++++++++++ .../validator/test_expression_checker.py | 86 ++++++++ .../test_common/validator/test_parser.py | 22 +- .../test_common/validator/test_validator.py | 112 +++++++++++ 22 files changed, 549 insertions(+), 150 deletions(-) create mode 100644 lambdas/shared/src/common/validator/__init__.py create mode 100644 lambdas/shared/src/common/validator/enums/__init__.py rename lambdas/shared/src/common/validator/{validation_expression_checker.py => expression_checker.py} (95%) create mode 100644 lambdas/shared/src/common/validator/lookup/__init__.py create mode 100644 lambdas/shared/src/common/validator/parsers/__init__.py create mode 100644 lambdas/shared/src/common/validator/record_error.py create mode 100644 lambdas/shared/src/common/validator/reporter/__init__.py create mode 100644 lambdas/shared/tests/__init__.py create mode 100644 lambdas/shared/tests/test_common/validator/__init__.py rename lambdas/shared/tests/test_common/validator/{test_application_csv.py => _est_application_csv.py} (100%) rename lambdas/shared/tests/test_common/validator/{test_application_csv_row.py => _test_application_csv_row.py} (96%) rename lambdas/shared/tests/test_common/validator/{test_application_fhir.py => _test_application_fhir.py} (96%) create mode 100644 lambdas/shared/tests/test_common/validator/schemas/Schema.json create mode 100644 lambdas/shared/tests/test_common/validator/test_expression_checker.py create mode 100644 lambdas/shared/tests/test_common/validator/test_validator.py diff --git a/immunisation-fhir-api.code-workspace b/immunisation-fhir-api.code-workspace index 70e325df82..aec454be13 100644 --- a/immunisation-fhir-api.code-workspace +++ b/immunisation-fhir-api.code-workspace @@ -1,44 +1,44 @@ { "folders": [ - { - "path": "." - }, - { - "path": "backend" - }, - { - "path": "filenameprocessor" - }, - { - "path": "recordprocessor" - }, - { - "path": "delta_backend" - }, - { - "path": "mesh_processor" - }, - { - "path": "e2e" - }, - { - "path": "e2e_batch" - }, - { - "path": "lambdas/ack_backend" - }, + { + "path": "." + }, + { + "path": "backend" + }, + { + "path": "filenameprocessor" + }, + { + "path": "recordprocessor" + }, + { + "path": "delta_backend" + }, + { + "path": "mesh_processor" + }, + { + "path": "e2e" + }, + { + "path": "e2e_batch" + }, + { + "path": "lambdas/ack_backend" + }, { "path": "lambdas/redis_sync" }, - { - "path": "lambdas/id_sync" + { + "path": "lambdas/id_sync" }, - { + { "path": "lambdas/mns_subscription" - }, - { - "path": "lambdas/shared" - } - ], + }, + { + "path": "lambdas/shared" + } + ], "settings": {} } diff --git a/lambdas/shared/Makefile b/lambdas/shared/Makefile index 0ecd1882c5..dbb8b04c9b 100644 --- a/lambdas/shared/Makefile +++ b/lambdas/shared/Makefile @@ -1,5 +1,10 @@ +TEST_ENV := @PYTHONPATH=src:tests:src/common + test: - @PYTHONPATH=src:../ python -m unittest discover -s tests -p "test_*.py" -v + $(TEST_ENV) python -m unittest discover -s tests -p "test_*.py" -v + +testv: + $(TEST_ENV) python -m unittest discover -s tests/test_common/validator -p "test_*.py" -v test-list: @PYTHONPATH=src:tests python -m unittest discover -s tests -p "test_*.py" --verbose | grep test_ diff --git a/lambdas/shared/src/common/validator/__init__.py b/lambdas/shared/src/common/validator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/src/common/validator/enums/__init__.py b/lambdas/shared/src/common/validator/enums/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/src/common/validator/enums/error_levels.py b/lambdas/shared/src/common/validator/enums/error_levels.py index 4a5e8d9db3..d3111dab98 100644 --- a/lambdas/shared/src/common/validator/enums/error_levels.py +++ b/lambdas/shared/src/common/validator/enums/error_levels.py @@ -5,6 +5,7 @@ WARNING = 1 NOTIFICATION = 2 + MESSAGES = { CRITICAL_ERROR: 'Critical Validation Error [%s]: %s', WARNING: 'Non-Critical Validation Error [%s]: %s', diff --git a/lambdas/shared/src/common/validator/enums/exception_messages.py b/lambdas/shared/src/common/validator/enums/exception_messages.py index cbb348ece5..b5cb9990ee 100644 --- a/lambdas/shared/src/common/validator/enums/exception_messages.py +++ b/lambdas/shared/src/common/validator/enums/exception_messages.py @@ -1,3 +1,5 @@ + + # all exceptions and messgaes UNEXPECTED_EXCEPTION = 0 VALUE_CHECK_FAILED = 1 diff --git a/lambdas/shared/src/common/validator/validation_expression_checker.py b/lambdas/shared/src/common/validator/expression_checker.py similarity index 95% rename from lambdas/shared/src/common/validator/validation_expression_checker.py rename to lambdas/shared/src/common/validator/expression_checker.py index a57f636f7a..ce27df842f 100644 --- a/lambdas/shared/src/common/validator/validation_expression_checker.py +++ b/lambdas/shared/src/common/validator/expression_checker.py @@ -1,66 +1,14 @@ # Root and base type expression checker functions -import validator.enums.exception_messages as ExceptionMessages +import common.validator.enums.exception_messages as ExceptionMessages import datetime import uuid import re -from validator.lookup.lookup_data import LookUpData -from validator.lookup.key_data import KeyData - - -class ErrorReport(): - def __init__(self, code: int, message: str, row: int = None, field: str = None, details: str = None, - summarise: bool = False): - self.code = code - self.message = message - if not summarise: - self.row = row - self.field = field - self.details = details - self.summarise = summarise - # these are set when the error is added to the report - self.error_group = None - self.name = None - self.id = None - self.error_level = None - - # function to return the object as a dictionary - def to_dict(self): - ret = { - 'code': self.code, - 'message': self.message - } - if not self.summarise: - ret.update({ - 'row': self.row, - 'field': self.field, - 'details': self.details - }) - return ret - - -# record exception capture -class RecordError(Exception): - - def __init__(self, code=None, message=None, details=None): - self.code = code - self.message = message - self.details = details - - def __str__(self): - return repr((self.code, self.message, self.details)) - - def __repr__(self): - return repr((self.code, self.message, self.details)) - - -# main expressions checker +from common.validator.lookup.lookup_data import LookUpData +from common.validator.lookup.key_data import KeyData +from common.validator.record_error import RecordError, ErrorReport + + class ExpressionChecker: - # validation settings - # summarise = False - # report_unexpected_exception = True - # dataParser = any - # dataLookUp = any - # keyData = any def __init__(self, data_parser, summarise, report_unexpected_exception): self.data_parser = data_parser # FHIR data parser for additional functions @@ -160,9 +108,26 @@ def _validate_uuid(self, expressionRule, field_name, field_value, row): # Integer Validate def _validate_integer(self, expression_rule, field_name, - field_value, row, summarise) -> ErrorReport: + field_value, row, summarise=False) -> ErrorReport: try: int(field_value) + if expression_rule: + # TODO - code is incomplete here. It appears there should be a check + # against expression_rule but it's not implemented. eg max, min, equal etc + # eg "1" means value must be 1 + # "1:10" means value must be between 1 to 10 + # "1,10" means value must be either 1 or 10 + # ":10" means value must be less than or equal to 10 + # "1:" means value must be greater than or equal to 1 + # ">10" means value must be greater than 10 + # "<10" means value must be less than 10 + + check_value = int(expression_rule) + if field_value != check_value: + raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, + "Value integer check failed", + "Value does not equal expected value, Expected- " + expression_rule + + " found- " + field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None diff --git a/lambdas/shared/src/common/validator/lookup/__init__.py b/lambdas/shared/src/common/validator/lookup/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/src/common/validator/parsers/__init__.py b/lambdas/shared/src/common/validator/parsers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/src/common/validator/record_error.py b/lambdas/shared/src/common/validator/record_error.py new file mode 100644 index 0000000000..95ec0e6056 --- /dev/null +++ b/lambdas/shared/src/common/validator/record_error.py @@ -0,0 +1,46 @@ + + +class ErrorReport(): + def __init__(self, code: int = None, message: str = None, row: int = None, field: str = None, details: str = None, + summarise: bool = False, error_level: int = None): + self.code = code + self.message = message + if not summarise: + self.row = row + self.field = field + self.details = details + self.summarise = summarise + # these are set when the error is added to the report + self.error_group = None + self.name = None + self.id = None + self.error_level = error_level + + # function to return the object as a dictionary + def to_dict(self): + ret = { + 'code': self.code, + 'message': self.message + } + if not self.summarise: + ret.update({ + 'row': self.row, + 'field': self.field, + 'details': self.details + }) + return ret + + +# record exception capture +class RecordError(Exception): + + def __init__(self, code=None, message=None, details=None): + self.code = code + self.message = message + self.details = details + + def __str__(self): + return repr((self.code, self.message, self.details)) + + def __repr__(self): + return repr((self.code, self.message, self.details)) diff --git a/lambdas/shared/src/common/validator/reporter/__init__.py b/lambdas/shared/src/common/validator/reporter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/src/common/validator/reporter/dq_reporter.py b/lambdas/shared/src/common/validator/reporter/dq_reporter.py index ed09c9ce43..5a281916a5 100644 --- a/lambdas/shared/src/common/validator/reporter/dq_reporter.py +++ b/lambdas/shared/src/common/validator/reporter/dq_reporter.py @@ -1,6 +1,6 @@ import json import datetime -import validator.enums.error_levels as ErrorLevels +import common.validator.enums.error_levels as ErrorLevels from dateutil import parser diff --git a/lambdas/shared/src/common/validator/validator.py b/lambdas/shared/src/common/validator/validator.py index 24ff02efa1..c2dbf0258d 100644 --- a/lambdas/shared/src/common/validator/validator.py +++ b/lambdas/shared/src/common/validator/validator.py @@ -1,35 +1,29 @@ # Main validation engine -import validator.enums.exception_messages as ExceptionMessages -import validator.enums.error_levels as ErrorLevels -from validator.parsers.csv_parser import CSVParser -from validator.parsers.csv_line_parser import CSVLineParser -from validator.parsers.fhir_parser import FHIRParser -from validator.parsers.schema_parser import SchemaParser -from validator.validation_expression_checker import ExpressionChecker, ErrorReport -from validator.reporter.dq_reporter import DQReporter - - -FilePath = '' -JSONData = {} -# SchemaFile = {} # Doesnt seem to be used -CSVRow = '' -CSVHeader = '' -error_records: list[ErrorReport] = [] -isCSV = True -dataType = 'FHIR' # 'FHIR', 'FHIRJSON', 'CSV', 'CSVROW' -dataParser = any +import common.validator.enums.exception_messages as ExceptionMessages +import common.validator.enums.error_levels as ErrorLevels +from common.validator.parsers.csv_parser import CSVParser +from common.validator.parsers.csv_line_parser import CSVLineParser +from common.validator.parsers.fhir_parser import FHIRParser +from common.validator.parsers.schema_parser import SchemaParser +from common.validator.expression_checker import ExpressionChecker +from common.validator.record_error import ErrorReport +from common.validator.reporter.dq_reporter import DQReporter class Validator: - def __init__(self, filepath, json_data, schemafile, csv_row, csv_header, data_type): + def __init__(self, + filepath='', json_data={}, schemafile={}, csv_row='', + csv_header='', data_type='FHIR', data_parser=None): self.filepath = filepath self.json_data = json_data self.schema_file = schemafile self.csv_row = csv_row self.csv_header = csv_header self.data_type = data_type + self.data_parser = data_parser + self.error_records: list[ErrorReport] = [] def _get_csv_line_parser(self, csv_row, csv_header): csv_parser = CSVLineParser() @@ -63,11 +57,11 @@ def _add_error_record(self, error_record: ErrorReport, error_record.name = expression_name error_record.id = expression_id error_record.error_level = error_level - error_records.append(error_record) + self.error_records.append(error_record) # Function to help identify a parent failure in the error list def _check_error_record_for_fail(self, expression_id): - for error_record in error_records: + for error_record in self.error_records: if (error_record.id == expression_id): return True return False @@ -102,7 +96,7 @@ def _validate_expression(self, expression_validate, expression, return error_record try: - expression_values = self.data_parser.getKeyValue(expression_fieldname) + expression_values = self.data_parser.get_key_value(expression_fieldname) except Exception as e: message = 'Data get values Unexpected exception [%s]: %s' % (e.__class__.__name__, e) error_report = ErrorReport(code=ExceptionMessages.PARSING_ERROR, message=message) @@ -124,20 +118,20 @@ def _validate_expression(self, expression_validate, expression, def run_validation(self, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True) -> list[ErrorReport]: try: - error_records.clear() + self.error_records.clear() - match self.dataType: # 'FHIR', 'FHIRJSON', 'CSV', 'CSVROW' + match self.data_type: # 'FHIR', 'FHIRJSON', 'CSV', 'CSVROW' case 'FHIR': - self.dataParser = self._getFHIRParser(self.FilePath) + self.data_parser = self._get_fhir_parser(self.filepath) self.isCSV = False case 'FHIRJSON': - self.dataParser = self._getFHIRJSONParser(self.JSONData) + self.data_parser = self._get_fhir_json_parser(self.json_data) self.isCSV = False case 'CSV': - self.dataParser = self._getCSVParser(self.FilePath) + self.data_parser = self._get_csv_parser(self.filepath) self.isCSV = True case 'CSVROW': - self.dataParser = self._getCSVLineParser(self.CSVRow, self.CSVHeader) + self.data_parser = self._get_csv_line_parser(self.csv_row, self.csv_header) self.isCSV = True except Exception as e: @@ -146,14 +140,14 @@ def run_validation(self, summarise=False, report_unexpected_exception=True, return [ErrorReport(code=0, message=message)] try: - schemaParser = self._getSchemaParser(self.SchemaFile) + schemaParser = self._get_schema_parser(self.schema_file) except Exception as e: if report_unexpected_exception: message = 'Schema Parser Unexpected exception [%s]: %s' % (e.__class__.__name__, e) return [ErrorReport(code=0, message=message)] try: - expression_validate = ExpressionChecker(dataParser, summarise, report_unexpected_exception) + expression_validate = ExpressionChecker(self.data_parser, summarise, report_unexpected_exception) except Exception as e: if report_unexpected_exception: message = 'Expression Checker Unexpected exception [%s]: %s' % (e.__class__.__name__, e) @@ -161,7 +155,7 @@ def run_validation(self, summarise=False, report_unexpected_exception=True, # get list of expressions try: - expressions = schemaParser.getExpressions() + expressions = schemaParser.get_expressions() except Exception as e: if report_unexpected_exception: message = 'Expression Getter Unexpected exception [%s]: %s' % (e.__class__.__name__, e) @@ -171,21 +165,21 @@ def run_validation(self, summarise=False, report_unexpected_exception=True, # rows = self._validate_expression(expression_validate, expression, inc_header_in_row_count) self._validate_expression(expression_validate, expression, inc_header_in_row_count) - return error_records + return self.error_records # ------------------------------------------------------------------------- # Report Generation # Build the error Report - def buildErrorReport(self, eventId): - OccurrenceDateTime = self.dataParser.getKeySingleValue('occurrenceDateTime') - dqReporter = DQReporter() - dqReport = dqReporter.generateErrorReport(eventId, OccurrenceDateTime, error_records) + def build_error_report(self, eventId): + OccurrenceDateTime = self.data_parser.get_key_single_value('occurrenceDateTime') + dq_reporter = DQReporter() + dq_report = dq_reporter.generate_error_report(eventId, OccurrenceDateTime, self.error_records) - return dqReport + return dq_report # Check all errors to see if we have a critical error that would fail the validation - def hasValidationFailed(self): - for errorRecord in error_records: - if (errorRecord['errorLevel'] == ErrorLevels.CRITICAL_ERROR): + def has_validation_failed(self): + for error_record in self.error_records: + if (error_record.error_level == ErrorLevels.CRITICAL_ERROR): return True return False diff --git a/lambdas/shared/tests/__init__.py b/lambdas/shared/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/tests/test_common/validator/__init__.py b/lambdas/shared/tests/test_common/validator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv.py b/lambdas/shared/tests/test_common/validator/_est_application_csv.py similarity index 100% rename from lambdas/shared/tests/test_common/validator/test_application_csv.py rename to lambdas/shared/tests/test_common/validator/_est_application_csv.py diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv_row.py b/lambdas/shared/tests/test_common/validator/_test_application_csv_row.py similarity index 96% rename from lambdas/shared/tests/test_common/validator/test_application_csv_row.py rename to lambdas/shared/tests/test_common/validator/_test_application_csv_row.py index 235d0bb51d..17cd7c4337 100644 --- a/lambdas/shared/tests/test_common/validator/test_application_csv_row.py +++ b/lambdas/shared/tests/test_common/validator/_test_application_csv_row.py @@ -1,6 +1,6 @@ # Test application file from pathlib import Path -from validator.validator import Validator +from common.validator.validator import Validator import json import time diff --git a/lambdas/shared/tests/test_common/validator/test_application_fhir.py b/lambdas/shared/tests/test_common/validator/_test_application_fhir.py similarity index 96% rename from lambdas/shared/tests/test_common/validator/test_application_fhir.py rename to lambdas/shared/tests/test_common/validator/_test_application_fhir.py index 7b63f4e7f0..78f8bdb8db 100644 --- a/lambdas/shared/tests/test_common/validator/test_application_fhir.py +++ b/lambdas/shared/tests/test_common/validator/_test_application_fhir.py @@ -1,6 +1,6 @@ # Test application file from pathlib import Path -from validator.validator import Validator +from common.validator.validator import Validator import json import time diff --git a/lambdas/shared/tests/test_common/validator/schemas/Schema.json b/lambdas/shared/tests/test_common/validator/schemas/Schema.json new file mode 100644 index 0000000000..8f6eb8e73c --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/schemas/Schema.json @@ -0,0 +1,188 @@ +{ + "id": "01K5EGR0C85TPNZT71MJ10VKYY", + "schemaName": "Base Vaccination Validation", + "version": 1.0, + "releaseDate": "2024-07-17T00:00:00.000Z", + "expressions": [ + { "expressionId" : "01K5EGR0C7Y1WJ0BC803SQDWK4", + "fieldNameFHIR": "contained|#:Patient|identifier|#:https://fhir.nhs.uk/Id/nhs-number|value", + "fieldNameFlat": "NHS_NUMBER", + "errorLevel" : 0, + "expression": { + "expressionName": "NHS Number Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId" : "01K5EGR0C7QCEJMWH1R4MBPGQA", + "fieldNameFHIR": "contained|#:Patient|name|#:official|given|0", + "fieldNameFlat": "PERSON_FORENAME", + "errorLevel" : 0, + "expression": { + "expressionName": "Person Forname Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C7RRG9F6FVHJ8HE4QX", + "fieldNameFHIR": "contained|#:Patient|name|#:official|family", + "fieldNameFlat": "PERSON_SURNAME", + "errorLevel" : 0, + "parentExpression": "01K5EGR0C7Y1WJ0BC803SQDWK4", + "expression": { + "expressionName": "Person Surname Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C8WSJ9N8RV6T8RJJ4W", + "fieldNameFHIR": "performer|#:Organization|actor|identifier|value", + "fieldNameFlat": "SITE_CODE", + "errorLevel" : 1, + "expression": { + "expressionName": "Organisation Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C8M1MVNKTQCE6MSG68", + "fieldNameFHIR": "performer|#:Organization|actor|identifier|value", + "fieldNameFlat": "SITE_CODE", + "errorLevel" : 0, + "expression": { + "expressionName": "Organisation Look Up Check", + "expressionType": "KEYCHECK", + "expressionRule": "Organisation" + }, + "errorGroup": "consistency" + }, + { + "expressionId" : "01K5EGR0C8SDQBTNCEP8TJNCCW", + "fieldNameFHIR": "contained|#:Practitioner|name|0|given|0", + "fieldNameFlat": "PERFORMING_PROFESSIONAL_FORENAME", + "errorLevel" : 1, + "expression": { + "expressionName": "Practitioner Forename Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C822RC96QJRR2YX18S", + "fieldNameFHIR": "contained|#:Practitioner|name|0|family", + "fieldNameFlat": "PERFORMING_PROFESSIONAL_SURNAME", + "errorLevel" : 1, + "expression": { + "expressionName": "Practitioner Surname Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C84CCDRR0VFSWQNFZP", + "fieldNameFHIR": "primarySource", + "fieldNameFlat": "PRIMARY_SOURCE", + "errorLevel" : 1, + "expression": { + "expressionName": "Primary Source Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C8VCX8FX8A7ZV7MQ5J", + "fieldNameFHIR": "extension|0|valueCodeableConcept|coding|0|code", + "fieldNameFlat": "VACCINATION_PROCEDURE_CODE", + "errorLevel" : 0, + "expression": { + "expressionName": "Procedure Code Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C85HY6MDNN6TTR1K48", + "fieldNameFHIR": "extension|0|valueCodeableConcept|coding|0|display", + "fieldNameFlat": "VACCINATION_PROCEDURE_TERM", + "errorLevel" : 1, + "expression": { + "expressionName": "Procedure Term Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C84DDGW567G14AYBC6", + "fieldNameFHIR": "protocolApplied|0|doseNumberPositiveInt", + "fieldNameFlat": "DOSE_SEQUENCE", + "errorLevel" : 1, + "expression": { + "expressionName": "Dose Sequence Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C8W3HXFYR80ENW73SS", + "fieldNameFHIR": "vaccineCode|coding|#:http://snomed.info/sct|code", + "fieldNameFlat": "VACCINE_PRODUCT_CODE", + "errorLevel" : 0, + "expression": { + "expressionName": "Produce Code Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C885N7MMW2J5JKHTT2", + "fieldNameFHIR": "vaccineCode|coding|#:http://snomed.info/sct|display", + "fieldNameFlat": "VACCINE_PRODUCT_TERM", + "errorLevel" : 1, + "expression": { + "expressionName": "Produce Term Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C86XN0AF0M9DJYFGCD", + "fieldNameFHIR": "manufacturer|display", + "fieldNameFlat": "VACCINE_MANUFACTURER", + "errorLevel" : 0, + "expression": { + "expressionName": "Manufacturer Display Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "01K5EGR0C89M4CV68B7XAKDCHG", + "fieldNameFHIR": "lotNumber", + "fieldNameFlat": "BATCH_NUMBER", + "errorLevel" : 0, + "expression": { + "expressionName": "Batch Number Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + } + ] +} diff --git a/lambdas/shared/tests/test_common/validator/test_expression_checker.py b/lambdas/shared/tests/test_common/validator/test_expression_checker.py new file mode 100644 index 0000000000..7d083ce4d3 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_expression_checker.py @@ -0,0 +1,86 @@ + +import unittest +from unittest.mock import patch, MagicMock +from common.validator.expression_checker import ExpressionChecker + + +class TestExpressionChecker(unittest.TestCase): + + def setUp(self): + patcher1 = patch('common.validator.expression_checker.LookUpData') + patcher2 = patch('common.validator.expression_checker.KeyData') + patcher3 = patch('common.validator.expression_checker.RecordError') + patcher4 = patch('common.validator.expression_checker.ErrorReport') + + self.MockLookUpData = patcher1.start() + self.MockKeyData = patcher2.start() + self.MockRecordError = patcher3.start() + self.MockErrorReport = patcher4.start() + + self.addCleanup(patcher1.stop) + self.addCleanup(patcher2.stop) + self.addCleanup(patcher3.stop) + self.addCleanup(patcher4.stop) + + self.mock_summarise = MagicMock() + self.mock_report_exception = MagicMock() + self.mock_data_parser = MagicMock() + + self.expression_checker = ExpressionChecker( + self.mock_data_parser, + self.mock_summarise, + self.mock_report_exception + ) + + def test_validate_datetime_valid(self): + result = self.expression_checker.validateExpression( + "DATETIME", + rule="", + field_name="timestamp", + field_value="2022-01-01T12:00:00", + row={} + ) + self.assertTrue(self.MockErrorReport.called or result is None) + + def test_validate_uuid_valid(self): + result = self.expression_checker.validateExpression( + "UUID", + rule="", + field_name="id", + field_value="550e8400-e29b-41d4-a716-446655440000", + row={} + ) + self.assertTrue(self.MockErrorReport.called or result is None) + + def test_validate_integer_invalid(self): + result = self.expression_checker.validateExpression( + "INT", + rule="", + field_name="age", + field_value="notanint", + row={} + ) + self.assertTrue(self.MockRecordError.called or result is not None) + + def test_validate_in_array(self): + # Mock data_parser.get_key_values + self.mock_data_parser.get_key_values.return_value = ["val1", "val2"] + + result = self.expression_checker.validateExpression( + "INARRAY", + rule="", + field_name="some_field", + field_value="val2", + row={} + ) + self.assertTrue(result is None or self.MockErrorReport.called) + + def test_validate_expression_type_not_found(self): + result = self.expression_checker.validateExpression( + "UNKNOWN", + rule="", + field_name="field", + field_value="value", + row={} + ) + self.assertIn("Schema expression not found", result) diff --git a/lambdas/shared/tests/test_common/validator/test_parser.py b/lambdas/shared/tests/test_common/validator/test_parser.py index dbcb684f4d..877adea706 100644 --- a/lambdas/shared/tests/test_common/validator/test_parser.py +++ b/lambdas/shared/tests/test_common/validator/test_parser.py @@ -1,17 +1,17 @@ # Test application file +import unittest + from pathlib import Path -from validator.parsers.fhir_parser import FHIRParser -import time +from common.validator.parsers.fhir_parser import FHIRParser -fhir_data_folder = Path("./data") -fhirFilePath = fhir_data_folder / "vaccination.json" -start = time.time() +class TestParse(unittest.TestCase): + def test_parse_fhir_file(self): -fhir_parser = FHIRParser() -fhir_parser.parse_fhir_file(fhirFilePath) -my_value = fhir_parser.get_key_value('vaccineCode|coding|0|code') -print('Value = ', my_value) + fhir_data_folder = Path("./data") + fhirFilePath = fhir_data_folder / "vaccination.json" -end = time.time() -print('Time to Run : ', end - start) + fhir_parser = FHIRParser() + fhir_parser.parse_fhir_file(fhirFilePath) + my_value = fhir_parser.get_key_value('vaccineCode|coding|0|code') + self.assertEqual(my_value, ['42223111000001107']) diff --git a/lambdas/shared/tests/test_common/validator/test_validator.py b/lambdas/shared/tests/test_common/validator/test_validator.py new file mode 100644 index 0000000000..058c90ef65 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_validator.py @@ -0,0 +1,112 @@ +# import unittest +# from unittest.mock import patch, MagicMock +# from common.validator.validator import Validator +# from common.validator.enums.exception_messages import PARSING_ERROR, PARENT_FAILED +# from common.validator.enums.error_levels import CRITICAL_ERROR +# from common.validator.validation_expression_checker import ErrorReport + + +# class TestValidator(unittest.TestCase): + +# def setUp(self): +# self.mock_logger_info = patch("validator.logger.info").start() +# self.mock_csv_parser = patch("validator.parsers.csv_parser").start() +# self.mock_schema_parser = patch("validator.parsers.schema_parser").start() +# self.mock_expression_checker = patch("validator.validation_expression_checker.ExpressionChecker").start() + +import unittest +from unittest.mock import patch, MagicMock +from common.validator.validator import Validator +from common.validator.record_error import ErrorReport +import common.validator.enums.error_levels as ErrorLevels + + +class TestValidator(unittest.TestCase): + def setUp(self): + self.mock_expression_checker = patch('common.validator.validator.ExpressionChecker').start() + self.mock_schema_parser = patch('common.validator.validator.SchemaParser').start() + self.mock_fhir_parser = patch('common.validator.validator.FHIRParser').start() + self.mock_csv_parser = patch('common.validator.validator.CSVParser').start() + self.mock_csv_line_parser = patch('common.validator.validator.CSVLineParser').start() + + def tearDown(self): + patch.stopall() + + def test_run_validation_csv(self): + # Setup mocks + self.mock_csv_parser.parse_csv_file.return_value = None + self.mock_schema_parser.parse_schema.return_value = None + self.mock_schema_parser.getExpressions.return_value = [ + {'fieldNameCSV': 'test_field', 'expressionId': 'exp1', 'errorLevel': 1, + 'expression': {'expressionName': 'Test', 'expressionType': 'type', 'expressionRule': 'rule'}, + 'errorGroup': 'group'} + ] + self.mock_expression_checker.return_value.validate_expression.return_value = None + + validator = Validator(filepath='file.csv', data_type='CSV') + result = validator.run_validation() + self.assertIsInstance(result, list) + self.assertEqual(len(result), 0) + + def test_run_validation_fhir_exception(self): + # Simulate exception in FHIRParser + self.mock_fhir_parser.side_effect = Exception("FHIR error") + validator = Validator(filepath='file.fhir', data_type='FHIR') + result = validator.run_validation() + self.assertIsInstance(result, list) + self.assertEqual(result[0].code, 0) + self.assertIn('Data Parser Unexpected exception', result[0].message) + + def test_run_validation_schema_exception(self): + # Simulate exception in SchemaParser + self.mock_fhir_parser.return_value.parse_fhir_file.return_value = None + self.mock_schema_parser.side_effect = Exception("Schema error") + validator = Validator(filepath='file.fhir', data_type='FHIR') + result = validator.run_validation() + self.assertIsInstance(result, list) + self.assertEqual(result[0].code, 0) + self.assertIn('Schema Parser Unexpected exception', result[0].message) + + def test_run_validation_expression_checker_exception(self): + # Simulate exception in ExpressionChecker + self.mock_fhir_parser.return_value.parse_fhir_file.return_value = None + self.mock_schema_parser.return_value.parse_schema.return_value = None + self.mock_schema_parser.return_value.get_expressions.return_value = [] + self.mock_expression_checker.side_effect = Exception("ExpressionChecker error") + validator = Validator(filepath='file.fhir', data_type='FHIR') + result = validator.run_validation() + self.assertIsInstance(result, list) + self.assertEqual(result[0].code, 0) + self.assertIn('Expression Checker Unexpected exception', result[0].message) + + def test_run_validation_expression_getter_exception(self): + # Simulate exception in getExpressions + self.mock_fhir_parser.return_value.parse_fhir_file.return_value = None + self.mock_schema_parser.return_value.parse_schema.return_value = None + self.mock_schema_parser.return_value.get_expressions.side_effect = Exception("Expressions error") + self.mock_expression_checker.return_value = MagicMock() + validator = Validator(filepath='file.fhir', data_type='FHIR') + result = validator.run_validation() + self.assertIsInstance(result, list) + self.assertEqual(result[0].code, 0) + self.assertIn('Expression Getter Unexpected exception', result[0].message) + + @patch('common.validator.validator.DQReporter') + def test_build_error_report(self, mock_dq_reporter): + mock_dq_reporter.return_value.generate_error_report.return_value = {'report': 'ok'} + validator = Validator(filepath='file.csv', data_type='CSV') + validator.data_parser = MagicMock() + validator.data_parser.get_key_single_value.return_value = '2023-01-01' + result = validator.build_error_report('event123') + self.assertEqual(result, {'report': 'ok'}) + mock_dq_reporter.return_value.generate_error_report.assert_called_once() + + def test_has_validation_failed_true(self): + v = Validator(data_type='CSV') + v.error_records.append(ErrorReport(error_level=ErrorLevels.CRITICAL_ERROR)) # Assuming 2 is CRITICAL_ERROR + self.assertTrue(v.has_validation_failed()) + + def test_has_validation_failed_false(self): + v = Validator(data_type='CSV') + v.error_records.append(ErrorReport(error_level=ErrorLevels.WARNING)) + self.assertFalse(v.has_validation_failed()) From 535e57796502e2efc0dd9a62f589741816046a4a Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Tue, 14 Oct 2025 09:12:02 +0100 Subject: [PATCH 05/13] tidy --- .../common/validator/expression_checker.py | 4 ++ .../validator/_test_application_fhir.py | 50 ------------------- .../validator/test_application_fhir.py | 49 ++++++++++++++++++ .../test_common/validator/test_parser.py | 20 ++++++-- 4 files changed, 70 insertions(+), 53 deletions(-) delete mode 100644 lambdas/shared/tests/test_common/validator/_test_application_fhir.py create mode 100644 lambdas/shared/tests/test_common/validator/test_application_fhir.py diff --git a/lambdas/shared/src/common/validator/expression_checker.py b/lambdas/shared/src/common/validator/expression_checker.py index ce27df842f..1fdd9ca160 100644 --- a/lambdas/shared/src/common/validator/expression_checker.py +++ b/lambdas/shared/src/common/validator/expression_checker.py @@ -78,6 +78,10 @@ def validateExpression(self, expression_type, rule, field_name, field_value, ro def _validate_datetime(self, rule, field_name, field_value, row): try: datetime.date.fromisoformat(field_value) + # TODO - rule is not used - could be date only, date time, past, future etc + if rule: + pass + except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = (e.message if e.message is not None diff --git a/lambdas/shared/tests/test_common/validator/_test_application_fhir.py b/lambdas/shared/tests/test_common/validator/_test_application_fhir.py deleted file mode 100644 index 78f8bdb8db..0000000000 --- a/lambdas/shared/tests/test_common/validator/_test_application_fhir.py +++ /dev/null @@ -1,50 +0,0 @@ -# Test application file -from pathlib import Path -from common.validator.validator import Validator -import json -import time - - -FHIR_data_folder = Path("C:/Source Code/CSV Validator/FHIR-data") -FHIRFilePath = FHIR_data_folder / "vaccination.json" - -schema_data_folder = Path("C:/Source Code/CSV Validator/Schemas") -schemaFilePath = schema_data_folder / "schema.json" - -DATA_TYPE = 'FHIRJSON' # 'FHIR', 'FHIRJSON', 'CSV', 'CSVROW' - -start = time.time() - -# get the JSON of the schema, changed to cope with elasticache -with open(schemaFilePath, 'r') as JSON: - SchemaFile = json.load(JSON) - -# get the FHIR Data as JSON -with open(FHIRFilePath, 'r') as JSON: - FHIRData = json.load(JSON) - - -validator = Validator(FHIRFilePath, FHIRData, SchemaFile, '', '', DATA_TYPE) # FHIR File Path not needed -error_list = validator.runValidation(True, True, True) -error_report = validator.buildErrorReport('25a8cc4d-1875-4191-ac6d-2d63a0ebc64b') # include the eventID if known - -failed_validation = validator.hasValidationFailed() - -# if len(ErrorList) > 0: -# print(ErrorList) -# else: -# print('Validated Successfully') - -print('--------------------------------------------------------------------') -print(error_report) -print('--------------------------------------------------------------------') - -if (failed_validation): - print('Validation failed due to a critical validation failure...') -else: - print('Validation Successful, see reports for details') - - -end = time.time() -print('Time Taken : ') -print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/test_application_fhir.py b/lambdas/shared/tests/test_common/validator/test_application_fhir.py new file mode 100644 index 0000000000..d2ae176ce0 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_application_fhir.py @@ -0,0 +1,49 @@ +# Test application file +from pathlib import Path +from common.validator.validator import Validator +import json +import time +import unittest + + +class TestApplication(unittest.TestCase): + def setUp(self): + + fhir_data_folder = Path("./data") + self.FHIRFilePath = fhir_data_folder / "vaccination.json" + + self.schema_data_folder = Path("./schemas") + self.schemaFilePath = self.schema_data_folder / "schema.json" + + def test_validation(self): + + DATA_TYPE = 'FHIR' + + start = time.time() + + # get the JSON of the schema, changed to cope with elasticache + with open(self.schemaFilePath, 'r') as JSON: + SchemaFile = json.load(JSON) + + # get the FHIR Data as JSON + with open(self.FHIRFilePath, 'r') as JSON: + FHIRData = json.load(JSON) + + validator = Validator(self.FHIRFilePath, FHIRData, SchemaFile, '', '', + DATA_TYPE) # FHIR File Path not needed + error_list = validator.run_validation(True, True, True) + error_report = validator.build_error_report( + '25a8cc4d-1875-4191-ac6d-2d63a0ebc64b') # include eventID if known + + failed_validation = validator.has_validation_failed() + + self.assertTrue(len(error_list) == 0, + f"Validation failed. Errors: {error_list}") + self.assertTrue(len(error_report['errors']) == 0, + f"Validation failed. Errors: {error_report['errors']}") + + self.assertFalse(failed_validation, 'Validation failed') + + end = time.time() + print('Time Taken : ') + print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/test_parser.py b/lambdas/shared/tests/test_common/validator/test_parser.py index 877adea706..01fd29d415 100644 --- a/lambdas/shared/tests/test_common/validator/test_parser.py +++ b/lambdas/shared/tests/test_common/validator/test_parser.py @@ -6,12 +6,26 @@ class TestParse(unittest.TestCase): - def test_parse_fhir_file(self): - fhir_data_folder = Path("./data") - fhirFilePath = fhir_data_folder / "vaccination.json" + def setUp(self): + self.fhir_data_folder = Path("./data") + + def test_parse_fhir_key_exists(self): + + fhirFilePath = self.fhir_data_folder / "vaccination.json" fhir_parser = FHIRParser() fhir_parser.parse_fhir_file(fhirFilePath) my_value = fhir_parser.get_key_value('vaccineCode|coding|0|code') self.assertEqual(my_value, ['42223111000001107']) + + def test_parse_fhir_key_not_exists(self): + + fhirFilePath = self.fhir_data_folder / "vaccination.json" + + fhir_parser = FHIRParser() + fhir_parser.parse_fhir_file(fhirFilePath) + my_value = fhir_parser.get_key_value('vaccineCode|coding|1') + self.assertEqual(my_value, ['']) + my_value = fhir_parser.get_key_value('vaccineCode|coding|1|codes') + self.assertEqual(my_value, ['']) From 81b86a5e518b9909eff405e5fe96cc70360ab27d Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Tue, 14 Oct 2025 21:17:06 +0100 Subject: [PATCH 06/13] Ready for Handover --- lambdas/shared/poetry.lock | 33 +++- lambdas/shared/pyproject.toml | 17 ++ .../common/validator/expression_checker.py | 85 ++++++--- .../validator/parsers/csv_line_parser.py | 2 +- .../common/validator/parsers/csv_parser.py | 2 +- .../common/validator/parsers/fhir_parser.py | 2 +- .../src/common/validator/record_error.py | 10 +- .../common/validator/reporter/dq_reporter.py | 19 +- .../shared/src/common/validator/validator.py | 94 ++++++---- .../tests/test_common/test_authentication.py | 10 +- .../tests/test_common/test_aws_dynamodb.py | 3 +- .../test_common/test_aws_lambda_event.py | 3 +- .../shared/tests/test_common/test_clients.py | 3 +- .../tests/test_common/test_log_decorator.py | 8 +- .../tests/test_common/test_pds_service.py | 3 +- .../tests/test_common/test_s3_reader.py | 3 +- .../validator/_est_application_csv.py | 32 ---- .../validator/_test_application_csv_row.py | 38 ---- .../validator/schemas/test_school_schema.json | 164 ++++++++++++++++++ .../validator/test_application_csv.py | 34 ++++ .../validator/test_application_csv_row.py | 45 +++++ .../validator/test_application_fhir.py | 49 +++--- .../validator/test_expression_checker.py | 52 +++--- .../test_common/validator/test_parser.py | 4 +- .../test_common/validator/test_validator.py | 35 ++-- 25 files changed, 527 insertions(+), 223 deletions(-) delete mode 100644 lambdas/shared/tests/test_common/validator/_est_application_csv.py delete mode 100644 lambdas/shared/tests/test_common/validator/_test_application_csv_row.py create mode 100644 lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json create mode 100644 lambdas/shared/tests/test_common/validator/test_application_csv.py create mode 100644 lambdas/shared/tests/test_common/validator/test_application_csv_row.py diff --git a/lambdas/shared/poetry.lock b/lambdas/shared/poetry.lock index 0c0539dadf..d564634790 100644 --- a/lambdas/shared/poetry.lock +++ b/lambdas/shared/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "async-timeout" @@ -818,6 +818,35 @@ urllib3 = ">=1.25.10,<3.0" [package.extras] tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-PyYAML", "types-requests"] +[[package]] +name = "ruff" +version = "0.14.0" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "ruff-0.14.0-py3-none-linux_armv6l.whl", hash = "sha256:58e15bffa7054299becf4bab8a1187062c6f8cafbe9f6e39e0d5aface455d6b3"}, + {file = "ruff-0.14.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:838d1b065f4df676b7c9957992f2304e41ead7a50a568185efd404297d5701e8"}, + {file = "ruff-0.14.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:703799d059ba50f745605b04638fa7e9682cc3da084b2092feee63500ff3d9b8"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ba9a8925e90f861502f7d974cc60e18ca29c72bb0ee8bfeabb6ade35a3abde7"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e41f785498bd200ffc276eb9e1570c019c1d907b07cfb081092c8ad51975bbe7"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30a58c087aef4584c193aebf2700f0fbcfc1e77b89c7385e3139956fa90434e2"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f8d07350bc7af0a5ce8812b7d5c1a7293cf02476752f23fdfc500d24b79b783c"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eec3bbbf3a7d5482b5c1f42d5fc972774d71d107d447919fca620b0be3e3b75e"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16b68e183a0e28e5c176d51004aaa40559e8f90065a10a559176713fcf435206"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb732d17db2e945cfcbbc52af0143eda1da36ca8ae25083dd4f66f1542fdf82e"}, + {file = "ruff-0.14.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:c958f66ab884b7873e72df38dcabee03d556a8f2ee1b8538ee1c2bbd619883dd"}, + {file = "ruff-0.14.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7eb0499a2e01f6e0c285afc5bac43ab380cbfc17cd43a2e1dd10ec97d6f2c42d"}, + {file = "ruff-0.14.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4c63b2d99fafa05efca0ab198fd48fa6030d57e4423df3f18e03aa62518c565f"}, + {file = "ruff-0.14.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:668fce701b7a222f3f5327f86909db2bbe99c30877c8001ff934c5413812ac02"}, + {file = "ruff-0.14.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a86bf575e05cb68dcb34e4c7dfe1064d44d3f0c04bbc0491949092192b515296"}, + {file = "ruff-0.14.0-py3-none-win32.whl", hash = "sha256:7450a243d7125d1c032cb4b93d9625dea46c8c42b4f06c6b709baac168e10543"}, + {file = "ruff-0.14.0-py3-none-win_amd64.whl", hash = "sha256:ea95da28cd874c4d9c922b39381cbd69cb7e7b49c21b8152b014bd4f52acddc2"}, + {file = "ruff-0.14.0-py3-none-win_arm64.whl", hash = "sha256:f42c9495f5c13ff841b1da4cb3c2a42075409592825dada7c5885c2c844ac730"}, + {file = "ruff-0.14.0.tar.gz", hash = "sha256:62ec8969b7510f77945df916de15da55311fade8d6050995ff7f680afe582c57"}, +] + [[package]] name = "s3transfer" version = "0.14.0" @@ -911,4 +940,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = "~3.11" -content-hash = "bb81e1d561bff2f0ba6b13575d1617f37456d36e5b2140fbf70382102a984aff" +content-hash = "819920e5e6f30601ceb20849edf39d63ab5612815209cad3770026db70cebe90" diff --git a/lambdas/shared/pyproject.toml b/lambdas/shared/pyproject.toml index f3f3c8b9f4..bdfa301777 100644 --- a/lambdas/shared/pyproject.toml +++ b/lambdas/shared/pyproject.toml @@ -29,7 +29,24 @@ pyjwt = "^2.10.1" [tool.poetry.group.dev.dependencies] coverage = "^7.10.7" +ruff = "^0.14.0" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.ruff] +line-length = 120 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "B", "C4", "I", "UP"] # Common sensible defaults +ignore = ["E501"] # Ignore line-too-long if needed + +[tool.ruff.lint.isort] +force-single-line = true # Optional: force single-line imports + +[tool.ruff.format] +quote-style = "double" # Or "single", depending on your code style +indent-style = "space" +line-ending = "lf" diff --git a/lambdas/shared/src/common/validator/expression_checker.py b/lambdas/shared/src/common/validator/expression_checker.py index 1fdd9ca160..36b2aa1a96 100644 --- a/lambdas/shared/src/common/validator/expression_checker.py +++ b/lambdas/shared/src/common/validator/expression_checker.py @@ -1,11 +1,43 @@ # Root and base type expression checker functions -import common.validator.enums.exception_messages as ExceptionMessages import datetime -import uuid import re -from common.validator.lookup.lookup_data import LookUpData +import uuid +from enum import Enum + +import common.validator.enums.exception_messages as ExceptionMessages from common.validator.lookup.key_data import KeyData -from common.validator.record_error import RecordError, ErrorReport +from common.validator.lookup.lookup_data import LookUpData +from common.validator.record_error import ErrorReport +from common.validator.record_error import RecordError + + +class ExpressionType(Enum): + DATETIME = 'DATETIME' + DATE = 'DATE' + UUID = 'UUID' + INT = 'INT' + FLOAT = 'FLOAT' + REGEX = 'REGEX' + EQUAL = 'EQUAL' + NOTEQUAL = 'NOTEQUAL' + IN = 'IN' + NRANGE = 'NRANGE' + INARRAY = 'INARRAY' + UPPER = 'UPPER' + LOWER = 'LOWER' + LENGTH = 'LENGTH' + STARTSWITH = 'STARTSWITH' + ENDSWITH = 'ENDSWITH' + EMPTY = 'EMPTY' + NOTEMPTY = 'NOTEMPTY' + POSITIVE = 'POSITIVE' + GENDER = 'GENDER' + NHSNUMBER = 'NHSNUMBER' + MAXOBJECTS = 'MAXOBJECTS' + POSTCODE = 'POSTCODE' + ONLYIF = 'ONLYIF' + LOOKUP = 'LOOKUP' + KEYCHECK = 'KEYCHECK' class ExpressionChecker: @@ -17,7 +49,7 @@ def __init__(self, data_parser, summarise, report_unexpected_exception): self.summarise = summarise self.report_unexpected_exception = report_unexpected_exception - def validateExpression(self, expression_type, rule, field_name, field_value, row) -> ErrorReport: + def validate_expression(self, expression_type: str, rule, field_name, field_value, row) -> ErrorReport: match expression_type: case "DATETIME": return self._validate_datetime(rule, field_name, field_value, row) @@ -75,7 +107,7 @@ def validateExpression(self, expression_type, rule, field_name, field_value, ro return "Schema expression not found! Check your expression type : " + expression_type # iso8086 date time validate - def _validate_datetime(self, rule, field_name, field_value, row): + def _validate_datetime(self, rule, field_name, field_value, row) -> ErrorReport: try: datetime.date.fromisoformat(field_value) # TODO - rule is not used - could be date only, date time, past, future etc @@ -88,14 +120,14 @@ def _validate_datetime(self, rule, field_name, field_value, row): else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return RecordError(code, message, row, field_name, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return RecordError(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # UUID validate - def _validate_uuid(self, expressionRule, field_name, field_value, row): + def _validate_uuid(self, expressionRule, field_name, field_value, row) -> ErrorReport: try: uuid.UUID(str(field_value)) except RecordError as e: @@ -112,7 +144,7 @@ def _validate_uuid(self, expressionRule, field_name, field_value, row): # Integer Validate def _validate_integer(self, expression_rule, field_name, - field_value, row, summarise=False) -> ErrorReport: + field_value, row) -> ErrorReport: try: int(field_value) if expression_rule: @@ -138,14 +170,14 @@ def _validate_integer(self, expression_rule, field_name, else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return RecordError(code, message, row, field_name, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Float Validate - def _validate_float(self, expression_rule, field_name, field_value, row, summarise): + def _validate_float(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: float(field_value) except RecordError as e: @@ -154,7 +186,7 @@ def _validate_float(self, expression_rule, field_name, field_value, row, summar else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) if e.details is not None: details = e.details - return RecordError(code, message, row, field_name, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) @@ -183,7 +215,7 @@ def _validate_length(self, expression_rule, field_name, field_value, row) -> Er return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Regex Validate - def _validate_regex(self, expression_rule, field_name, field_value, row, summarise) -> ErrorReport: + def _validate_regex(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: result = re.search(expression_rule, field_value) if not result: @@ -197,7 +229,7 @@ def _validate_regex(self, expression_rule, field_name, field_value, row, summar ) if e.details is not None: details = e.details - return RecordError(code, message, row, field_name, details, self.summarise) + return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) @@ -273,6 +305,7 @@ def _validate_n_range(self, expression_rule, field_name, field_value, row) -> E raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value range check failed", "Value is not within the number range, data- " + field_value) + return None except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = ( @@ -348,7 +381,7 @@ def _validate_lower(self, expression_rule, field_name, field_value, row) -> Erro return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Starts With Validate - def _validate_starts_with(self, expression_rule, field_name, field_value, row, summarise) -> ErrorReport: + def _validate_starts_with(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: result = field_value.startswith(expression_rule) if not result: @@ -369,7 +402,7 @@ def _validate_starts_with(self, expression_rule, field_name, field_value, row, s return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Ends With Validate - def _validate_ends_with(self, expression_rule, field_name, field_value, row, summarise) -> ErrorReport: + def _validate_ends_with(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: result = field_value.endswith(expression_rule) if not result: @@ -390,7 +423,7 @@ def _validate_ends_with(self, expression_rule, field_name, field_value, row, sum return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Empty Validate - def _validate_empty(self, expression_rule, field_name, field_value, row, summarise) -> ErrorReport: + def _validate_empty(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: if field_value: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, @@ -409,7 +442,7 @@ def _validate_empty(self, expression_rule, field_name, field_value, row, summari return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Not Empty Validate - def _validate_not_empty(self, expression_rule, field_name, field_value, row, summarise) -> ErrorReport: + def _validate_not_empty(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: if not field_value: raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, @@ -428,7 +461,7 @@ def _validate_not_empty(self, expression_rule, field_name, field_value, row, sum return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) # Positive Validate - def _validate_positive(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_positive(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: value = float(fieldValue) if value < 0: @@ -448,7 +481,7 @@ def _validate_positive(self, expressionRule, fieldName, fieldValue, row, summar return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # NHSNumber Validate - def _validate_nhs_number(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_nhs_number(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: regexRule = '^6[0-9]{10}$' result = re.search(regexRule, fieldValue) @@ -469,7 +502,7 @@ def _validate_nhs_number(self, expressionRule, fieldName, fieldValue, row, summ return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # Gender Validate - def _validate_gender(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_gender(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: ruleList = ['0', '1', '2', '9'] @@ -490,7 +523,7 @@ def _validate_gender(self, expressionRule, fieldName, fieldValue, row, summaris return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # PostCode Validate - def _validate_post_code(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_post_code(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: regexRule = '^([Gg][Ii][Rr] 0[Aa]{2})|((([A-Za-z][0-9]{1,2})|(([A-Za-z][A-Ha-hJ-Yj-y]' '[0-9]{1,2})|(([AZa-z][0-9][A-Za-z])|([A-Za-z][A-Ha-hJ-Yj-y][0-9]?[A-Za-z])))) [0-9][A-Za-z]{2})$' @@ -512,7 +545,7 @@ def _validate_post_code(self, expressionRule, fieldName, fieldValue, row, summa return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # Max Objects Validate - def _validate_max_objects(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_max_objects(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: value = len(fieldValue) if value > int(expressionRule): @@ -556,7 +589,7 @@ def _validate_only_if(self, expressionRule, fieldName, fieldValue, row) -> Error return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # Check with Lookup - def _validate_against_lookup(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_against_lookup(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: result = self.dataLookUp.findLookUp(fieldValue) if not result: @@ -576,7 +609,7 @@ def _validate_against_lookup(self, expressionRule, fieldName, fieldValue, row, return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) # Check with Key Lookup - def _validate_against_key(self, expressionRule, fieldName, fieldValue, row, summarise) -> ErrorReport: + def _validate_against_key(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: result = self.KeyData.findKey(expressionRule, fieldValue) if not result: diff --git a/lambdas/shared/src/common/validator/parsers/csv_line_parser.py b/lambdas/shared/src/common/validator/parsers/csv_line_parser.py index 989561c02b..bb07210254 100644 --- a/lambdas/shared/src/common/validator/parsers/csv_line_parser.py +++ b/lambdas/shared/src/common/validator/parsers/csv_line_parser.py @@ -15,7 +15,7 @@ def parse_csv_line(self, csv_row, csv_header): self.csv_file_data = dict(map(lambda i, j: (i, j), keys, values)) # retrieve a column of data to work with - def get_key_values(self, field_name): + def get_key_value(self, field_name): # creating empty lists, convert to list data = [self.csv_file_data[field_name]] return data diff --git a/lambdas/shared/src/common/validator/parsers/csv_parser.py b/lambdas/shared/src/common/validator/parsers/csv_parser.py index 32544574d4..834e049c28 100644 --- a/lambdas/shared/src/common/validator/parsers/csv_parser.py +++ b/lambdas/shared/src/common/validator/parsers/csv_parser.py @@ -20,7 +20,7 @@ def parse_csv_file(self, csv_filename): # --------------------------------------------- # Scan and retrieve values # retrieve a column of data to work with - def get_key_values(self, field_name): + def get_key_value(self, field_name): # creating empty lists data = self.csv_file_data[field_name] return data diff --git a/lambdas/shared/src/common/validator/parsers/fhir_parser.py b/lambdas/shared/src/common/validator/parsers/fhir_parser.py index c7c62c9b37..44f3620eb5 100644 --- a/lambdas/shared/src/common/validator/parsers/fhir_parser.py +++ b/lambdas/shared/src/common/validator/parsers/fhir_parser.py @@ -11,7 +11,7 @@ def __init__(self): # File Management # used for files def parse_fhir_file(self, fhir_file_name): - with open(fhir_file_name, 'r') as json_file: + with open(fhir_file_name) as json_file: self.fhir_resource = json.load(json_file) # used for JSON FHIR Resource data diff --git a/lambdas/shared/src/common/validator/record_error.py b/lambdas/shared/src/common/validator/record_error.py index 95ec0e6056..b709281074 100644 --- a/lambdas/shared/src/common/validator/record_error.py +++ b/lambdas/shared/src/common/validator/record_error.py @@ -1,14 +1,13 @@ -class ErrorReport(): +class ErrorReport: def __init__(self, code: int = None, message: str = None, row: int = None, field: str = None, details: str = None, summarise: bool = False, error_level: int = None): self.code = code self.message = message - if not summarise: - self.row = row - self.field = field - self.details = details + self.row = row + self.field = field + self.details = details self.summarise = summarise # these are set when the error is added to the report self.error_group = None @@ -35,6 +34,7 @@ def to_dict(self): class RecordError(Exception): def __init__(self, code=None, message=None, details=None): + super().__init__(message) self.code = code self.message = message self.details = details diff --git a/lambdas/shared/src/common/validator/reporter/dq_reporter.py b/lambdas/shared/src/common/validator/reporter/dq_reporter.py index 5a281916a5..b88510ede4 100644 --- a/lambdas/shared/src/common/validator/reporter/dq_reporter.py +++ b/lambdas/shared/src/common/validator/reporter/dq_reporter.py @@ -1,8 +1,11 @@ -import json import datetime -import common.validator.enums.error_levels as ErrorLevels +import json + from dateutil import parser +import common.validator.enums.error_levels as ErrorLevels +from common.validator.record_error import ErrorReport + class DQReporter: @@ -36,7 +39,7 @@ def diff_dates(self, date1, date2): diff_minutes = diff_seconds / 60 return diff_minutes - def generate_error_report(self, event_id, occurrence, error_records): + def generate_error_report(self, event_id, occurrence, error_records: list[ErrorReport]): occurrence_date = occurrence occurrence_date = parser.parse(occurrence_date, ignoretz=True) validation_date = datetime.datetime.now(tz=None) @@ -48,19 +51,19 @@ def generate_error_report(self, event_id, occurrence, error_records): self.error_report['results']['timeliness_processed'] = time_taken for errorRecord in error_records: - self.updateReport(errorRecord) + self.update_report(errorRecord) json_error_report = json.dumps(self.error_report) return json_error_report - def update_report(self, error_data): - error_group = error_data["errorGroup"] - if (error_data['errorLevel'] == ErrorLevels.CRITICAL_ERROR): + def update_report(self, error_data: ErrorReport): + error_group = error_data.error_group + if (error_data.error_level == ErrorLevels.CRITICAL_ERROR): self.error_report['validated'] = "false" total_errors = self.error_report['results']['totalErrors'] results_error_count = self.error_report['results'][error_group]['errors'] results_error_count += 1 total_errors += 1 - self.error_report['results'][error_group]['fields'].append(error_data["name"]) + self.error_report['results'][error_group]['fields'].append(error_data.name) self.error_report['results'][error_group]['errors'] = results_error_count self.error_report['results']['totalErrors'] = total_errors diff --git a/lambdas/shared/src/common/validator/validator.py b/lambdas/shared/src/common/validator/validator.py index c2dbf0258d..95021a6f70 100644 --- a/lambdas/shared/src/common/validator/validator.py +++ b/lambdas/shared/src/common/validator/validator.py @@ -1,28 +1,35 @@ # Main validation engine -import common.validator.enums.exception_messages as ExceptionMessages +from enum import Enum + import common.validator.enums.error_levels as ErrorLevels -from common.validator.parsers.csv_parser import CSVParser +import common.validator.enums.exception_messages as ExceptionMessages +from common.validator.expression_checker import ExpressionChecker from common.validator.parsers.csv_line_parser import CSVLineParser +from common.validator.parsers.csv_parser import CSVParser from common.validator.parsers.fhir_parser import FHIRParser from common.validator.parsers.schema_parser import SchemaParser -from common.validator.expression_checker import ExpressionChecker from common.validator.record_error import ErrorReport from common.validator.reporter.dq_reporter import DQReporter +class DataType(Enum): + FHIR = 'FHIR' + FHIRJSON = 'FHIRJSON' + CSV = 'CSV' + CSVROW = 'CSVROW' + + class Validator: - def __init__(self, - filepath='', json_data={}, schemafile={}, csv_row='', - csv_header='', data_type='FHIR', data_parser=None): + def __init__(self, schema_file = '', data_type: DataType = None, filepath = ''): self.filepath = filepath - self.json_data = json_data - self.schema_file = schemafile - self.csv_row = csv_row - self.csv_header = csv_header + self.json_data = {} + self.schema_file = schema_file + self.csv_row = '' + self.csv_header = '' self.data_type = data_type - self.data_parser = data_parser + self.data_parser = '' self.error_records: list[ErrorReport] = [] def _get_csv_line_parser(self, csv_row, csv_header): @@ -67,7 +74,7 @@ def _check_error_record_for_fail(self, expression_id): return False # validate a single expression against the data file - def _validate_expression(self, expression_validate, expression, + def _validate_expression(self, expression_validator: ExpressionChecker, expression, inc_header_in_row_count) -> ErrorReport | int: row = 1 if inc_header_in_row_count: @@ -98,7 +105,7 @@ def _validate_expression(self, expression_validate, expression, try: expression_values = self.data_parser.get_key_value(expression_fieldname) except Exception as e: - message = 'Data get values Unexpected exception [%s]: %s' % (e.__class__.__name__, e) + message = f'Data get values Unexpected exception [{e.__class__.__name__}]: {e}' error_report = ErrorReport(code=ExceptionMessages.PARSING_ERROR, message=message) # original code had self.CriticalErrorLevel. Replaced with error_level self._add_error_record(error_report, @@ -106,14 +113,44 @@ def _validate_expression(self, expression_validate, expression, return error_report for value in expression_values: - error_record: ErrorReport = expression_validate.validate_expression( - expression_type, expression_rule, expression_fieldname, value, row) - if error_record is not None: - self._addErrorRecord(error_record, expression_error_group, - expression_name, expression_id, error_level) + error_record: ErrorReport | None = None + try: + error_record = expression_validator.validate_expression( + expression_type, expression_rule, expression_fieldname, value, row) + if error_record is not None: + self._add_error_record(error_record, expression_error_group, + expression_name, expression_id, error_level) + except Exception: + print(f'Exception validating expression {expression_id} on row {row}: {error_record}') row += 1 return row + + def validate_fhir(self, filepath, summarise=False, report_unexpected_exception=True, + inc_header_in_row_count=True) -> list[ErrorReport]: + self.data_type = DataType.FHIR + self.filepath = filepath + return self.run_validation(summarise, report_unexpected_exception, inc_header_in_row_count) + + def validate_csv(self, filepath, summarise=False, report_unexpected_exception=True, + inc_header_in_row_count=True) -> list[ErrorReport]: + self.data_type = DataType.CSV + self.filepath = filepath + return self.run_validation(summarise, report_unexpected_exception, inc_header_in_row_count) + + def validate_csv_row(self, csv_row, csv_header, summarise=False, report_unexpected_exception=True, + inc_header_in_row_count=True) -> list[ErrorReport]: + self.data_type = DataType.CSVROW + self.csv_row = csv_row + self.csv_header = csv_header + return self.run_validation(summarise, report_unexpected_exception, inc_header_in_row_count) + + def validate_fhir_json(self, json_data, summarise=False, report_unexpected_exception=True, + inc_header_in_row_count=True) -> list[ErrorReport]: + self.data_type = DataType.FHIRJSON + self.json_data = json_data + return self.validate_fhir_json(json_data, summarise, report_unexpected_exception, inc_header_in_row_count) + # run the validation against the data def run_validation(self, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True) -> list[ErrorReport]: @@ -121,36 +158,36 @@ def run_validation(self, summarise=False, report_unexpected_exception=True, self.error_records.clear() match self.data_type: # 'FHIR', 'FHIRJSON', 'CSV', 'CSVROW' - case 'FHIR': + case DataType.FHIR: self.data_parser = self._get_fhir_parser(self.filepath) self.isCSV = False - case 'FHIRJSON': + case DataType.FHIRJSON: self.data_parser = self._get_fhir_json_parser(self.json_data) self.isCSV = False - case 'CSV': + case DataType.CSV: self.data_parser = self._get_csv_parser(self.filepath) self.isCSV = True - case 'CSVROW': + case DataType.CSVROW: self.data_parser = self._get_csv_line_parser(self.csv_row, self.csv_header) self.isCSV = True except Exception as e: if report_unexpected_exception: - message = 'Data Parser Unexpected exception [%s]: %s' % (e.__class__.__name__, e) + message = f'Data Parser Unexpected exception [{e.__class__.__name__}]: {e}' return [ErrorReport(code=0, message=message)] try: schemaParser = self._get_schema_parser(self.schema_file) except Exception as e: if report_unexpected_exception: - message = 'Schema Parser Unexpected exception [%s]: %s' % (e.__class__.__name__, e) + message = f'Schema Parser Unexpected exception [{e.__class__.__name__}]: {e}' return [ErrorReport(code=0, message=message)] try: - expression_validate = ExpressionChecker(self.data_parser, summarise, report_unexpected_exception) + expression_validator = ExpressionChecker(self.data_parser, summarise, report_unexpected_exception) except Exception as e: if report_unexpected_exception: - message = 'Expression Checker Unexpected exception [%s]: %s' % (e.__class__.__name__, e) + message = f'Expression Checker Unexpected exception [{e.__class__.__name__}]: {e}' return [ErrorReport(code=0, message=message)] # get list of expressions @@ -158,12 +195,11 @@ def run_validation(self, summarise=False, report_unexpected_exception=True, expressions = schemaParser.get_expressions() except Exception as e: if report_unexpected_exception: - message = 'Expression Getter Unexpected exception [%s]: %s' % (e.__class__.__name__, e) + message = f'Expression Getter Unexpected exception [{e.__class__.__name__}]: {e}' return [ErrorReport(code=0, message=message)] for expression in expressions: - # rows = self._validate_expression(expression_validate, expression, inc_header_in_row_count) - self._validate_expression(expression_validate, expression, inc_header_in_row_count) + self._validate_expression(expression_validator, expression, inc_header_in_row_count) return self.error_records diff --git a/lambdas/shared/tests/test_common/test_authentication.py b/lambdas/shared/tests/test_common/test_authentication.py index 44c956d24f..69cc81781a 100644 --- a/lambdas/shared/tests/test_common/test_authentication.py +++ b/lambdas/shared/tests/test_common/test_authentication.py @@ -2,13 +2,17 @@ import json import time import unittest -from unittest.mock import ANY, MagicMock, patch +from unittest.mock import ANY +from unittest.mock import MagicMock +from unittest.mock import patch import responses -from common.authentication import AppRestrictedAuth, Service -from common.models.errors import UnhandledResponseError from responses import matchers +from common.authentication import AppRestrictedAuth +from common.authentication import Service +from common.models.errors import UnhandledResponseError + class TestAuthenticator(unittest.TestCase): def setUp(self): diff --git a/lambdas/shared/tests/test_common/test_aws_dynamodb.py b/lambdas/shared/tests/test_common/test_aws_dynamodb.py index b4a025bb58..5c94f13d69 100644 --- a/lambdas/shared/tests/test_common/test_aws_dynamodb.py +++ b/lambdas/shared/tests/test_common/test_aws_dynamodb.py @@ -1,5 +1,6 @@ import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock +from unittest.mock import patch from common.aws_dynamodb import get_dynamodb_table diff --git a/lambdas/shared/tests/test_common/test_aws_lambda_event.py b/lambdas/shared/tests/test_common/test_aws_lambda_event.py index 3f4ac87879..87e31645ce 100644 --- a/lambdas/shared/tests/test_common/test_aws_lambda_event.py +++ b/lambdas/shared/tests/test_common/test_aws_lambda_event.py @@ -1,6 +1,7 @@ import unittest -from common.aws_lambda_event import AwsEventType, AwsLambdaEvent +from common.aws_lambda_event import AwsEventType +from common.aws_lambda_event import AwsLambdaEvent class TestAwsLambdaEvent(unittest.TestCase): diff --git a/lambdas/shared/tests/test_common/test_clients.py b/lambdas/shared/tests/test_common/test_clients.py index b6944af453..7403241ab4 100644 --- a/lambdas/shared/tests/test_common/test_clients.py +++ b/lambdas/shared/tests/test_common/test_clients.py @@ -1,7 +1,8 @@ import importlib import logging import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock +from unittest.mock import patch import common.clients as clients diff --git a/lambdas/shared/tests/test_common/test_log_decorator.py b/lambdas/shared/tests/test_common/test_log_decorator.py index cdcc613a90..b2d8f2b8f7 100644 --- a/lambdas/shared/tests/test_common/test_log_decorator.py +++ b/lambdas/shared/tests/test_common/test_log_decorator.py @@ -3,11 +3,9 @@ from datetime import datetime from unittest.mock import patch -from common.log_decorator import ( - generate_and_send_logs, - logging_decorator, - send_log_to_firehose, -) +from common.log_decorator import generate_and_send_logs +from common.log_decorator import logging_decorator +from common.log_decorator import send_log_to_firehose class TestLogDecorator(unittest.TestCase): diff --git a/lambdas/shared/tests/test_common/test_pds_service.py b/lambdas/shared/tests/test_common/test_pds_service.py index ebe22063cc..2d1a4bdfdc 100644 --- a/lambdas/shared/tests/test_common/test_pds_service.py +++ b/lambdas/shared/tests/test_common/test_pds_service.py @@ -2,10 +2,11 @@ from unittest.mock import create_autospec import responses +from responses import matchers + from common.authentication import AppRestrictedAuth from common.models.errors import UnhandledResponseError from common.pds_service import PdsService -from responses import matchers class TestPdsService(unittest.TestCase): diff --git a/lambdas/shared/tests/test_common/test_s3_reader.py b/lambdas/shared/tests/test_common/test_s3_reader.py index f56cae4bc1..967bd1f503 100644 --- a/lambdas/shared/tests/test_common/test_s3_reader.py +++ b/lambdas/shared/tests/test_common/test_s3_reader.py @@ -1,5 +1,6 @@ import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock +from unittest.mock import patch from common.s3_reader import S3Reader diff --git a/lambdas/shared/tests/test_common/validator/_est_application_csv.py b/lambdas/shared/tests/test_common/validator/_est_application_csv.py deleted file mode 100644 index 10ebc84761..0000000000 --- a/lambdas/shared/tests/test_common/validator/_est_application_csv.py +++ /dev/null @@ -1,32 +0,0 @@ -# Test application file -from pathlib import Path -from common.validator.validator import Validator -import json -import time - -csv_data_folder = Path("./data/csv-data/data") -# csvFilePath = csv_data_folder / "test_data.csv" # Medium -csvFilePath = csv_data_folder / "test_data_ok.csv" # Passes - -dataType = 'CSV' - -schema_data_folder = Path("C:/Source Code/CSV Validator/Schemas") -schemaFilePath = schema_data_folder / "test1.json" - - -start = time.time() - -# get the JSON of the schema, changed to cope with elasticache -with open(schemaFilePath, 'r') as JSON: - SchemaFile = json.load(JSON) - -validator = Validator(csvFilePath, SchemaFile, '', '', dataType) -error_report = validator.run_validation(False, True, True) - -if len(error_report) > 0: - print(error_report) -else: - print('Validated Successfully') - -end = time.time() -print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/_test_application_csv_row.py b/lambdas/shared/tests/test_common/validator/_test_application_csv_row.py deleted file mode 100644 index 17cd7c4337..0000000000 --- a/lambdas/shared/tests/test_common/validator/_test_application_csv_row.py +++ /dev/null @@ -1,38 +0,0 @@ -# Test application file -from pathlib import Path -from common.validator.validator import Validator -import json -import time - -CSV_HEADER = 'academic_year,time_period,time_identifier,geographic_level,' -'country_code,country_name,region_code,region_name,new_la_code,la_name,' -'old_la_code,school_type,num_schools,enrolments,present_sessions,overall_attendance,' -'approved_educational_activity,overall_absence,authorised_absence,unauthorised_absence,' -'late_sessions,possible_sessions,reason_present_am,reason_present_pm,reason_present,' -'reason_l_present_late_before_registers_closed' -CSV_ROW = '202223,202223,Spring term,Local authority,E92000001,England,E12000004,' -'East Midlands,E06000016,Leicester,856,Primary,66.94915254,23057.94915,2367094,' -'2380687,13593,166808,99826,66982,34090,2547495,1157575,1180365,2337940,29154' -DATA_TYPE = 'CSVROW' - -schema_data_folder = Path("C:/Source Code/CSV Validator/Schemas") -schemaFilePath = schema_data_folder / "test1.json" -# schemaFilePath = schema_data_folder / "test2.json" - -start = time.time() - -# get the JSON of the schema, moved from internal file to json string -# changed to cope with elasticache -with open(schemaFilePath, 'r') as JSON: - SchemaFile = json.load(JSON) - -Validator = Validator('', SchemaFile, CSV_ROW, CSV_HEADER, DATA_TYPE) -ErrorReport = Validator.runValidation(True, True, True) - -if len(ErrorReport) > 0: - print(ErrorReport) -else: - print('Validated Successfully') - -end = time.time() -print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json b/lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json new file mode 100644 index 0000000000..ad3fa1e004 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json @@ -0,0 +1,164 @@ +{ + "id": "01K5EGR0C85TPNZT71MJ10VKAA", + "schemaName": "School Attendance Data Validation", + "version": 1.0, + "releaseDate": "2024-07-17T00:00:00.000Z", + "expressions": [ + { + "expressionId": "check1", + "fieldNameCSV": "academic_year", + "fieldNameFlat": "ACADEMIC_YEAR", + "errorLevel": 0, + "expression": { + "expressionName": "Academic Year Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check2", + "fieldNameCSV": "time_period", + "fieldNameFlat": "TIME_PERIOD", + "errorLevel": 0, + "expression": { + "expressionName": "Time Period Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check3", + "fieldNameCSV": "country_code", + "fieldNameFlat": "COUNTRY_CODE", + "errorLevel": 0, + "expression": { + "expressionName": "Country Code Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check4", + "fieldNameCSV": "country_name", + "fieldNameFlat": "COUNTRY_NAME", + "errorLevel": 1, + "expression": { + "expressionName": "Country Name Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check5", + "fieldNameCSV": "new_la_code", + "fieldNameFlat": "NEW_LA_CODE", + "errorLevel": 0, + "expression": { + "expressionName": "LA Code Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check6", + "fieldNameCSV": "la_name", + "fieldNameFlat": "LA_NAME", + "errorLevel": 1, + "expression": { + "expressionName": "LA Name Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check7", + "fieldNameCSV": "school_type", + "fieldNameFlat": "SCHOOL_TYPE", + "errorLevel": 0, + "expression": { + "expressionName": "School Type Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId": "check8", + "fieldNameCSV": "num_schools", + "fieldNameFlat": "NUM_SCHOOLS", + "errorLevel": 1, + "expression": { + "expressionName": "Number of Schools Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check9", + "fieldNameCSV": "enrolments", + "fieldNameFlat": "ENROLMENTS", + "errorLevel": 1, + "expression": { + "expressionName": "Enrolments Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check10", + "fieldNameCSV": "overall_attendance", + "fieldNameFlat": "OVERALL_ATTENDANCE", + "errorLevel": 0, + "expression": { + "expressionName": "Overall Attendance Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check11", + "fieldNameCSV": "overall_absence", + "fieldNameFlat": "OVERALL_ABSENCE", + "errorLevel": 0, + "expression": { + "expressionName": "Overall Absence Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check12", + "fieldNameCSV": "authorised_absence", + "fieldNameFlat": "AUTHORISED_ABSENCE", + "errorLevel": 1, + "expression": { + "expressionName": "Authorised Absence Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId": "check13", + "fieldNameCSV": "unauthorised_absence", + "fieldNameFlat": "UNAUTHORISED_ABSENCE", + "errorLevel": 1, + "expression": { + "expressionName": "Unauthorised Absence Numeric Check", + "expressionType": "INT", + "expressionRule": "" + }, + "errorGroup": "validity" + } + ] +} \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv.py b/lambdas/shared/tests/test_common/validator/test_application_csv.py new file mode 100644 index 0000000000..824219606b --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_application_csv.py @@ -0,0 +1,34 @@ +# Test application file +import json +import time +from pathlib import Path + +from common.validator.validator import Validator + +# TODO this needs to be converted to unit test with success and fail cases + +parent_folder = Path(__file__).parent +data_folder = parent_folder / "data" +csvFilePath = data_folder / "test_data_ok.csv" # Passes + +dataType = 'CSV' + +schemaFilePath = parent_folder / "schemas/test_school_schema.json" + + +start = time.time() + +# get the JSON of the schema, changed to cope with elasticache +with open(schemaFilePath) as JSON: + SchemaFile = json.load(JSON) + +validator = Validator(SchemaFile) +error_report = validator.validate_csv(csvFilePath, False, True, True) + +if len(error_report) > 0: + print(error_report) +else: + print('Validated Successfully') + +end = time.time() +print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv_row.py b/lambdas/shared/tests/test_common/validator/test_application_csv_row.py new file mode 100644 index 0000000000..6daafa18b4 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_application_csv_row.py @@ -0,0 +1,45 @@ +# Test application file +import json +import unittest +from pathlib import Path + +from common.validator.validator import Validator + +CSV_HEADER = 'academic_year,time_period,time_identifier,geographic_level,' \ +'country_code,country_name,region_code,region_name,new_la_code,la_name,' \ +'old_la_code,school_type,num_schools,enrolments,present_sessions,overall_attendance,' \ +'approved_educational_activity,overall_absence,authorised_absence,unauthorised_absence,' \ +'late_sessions,possible_sessions,reason_present_am,reason_present_pm,reason_present,' \ +'reason_l_present_late_before_registers_closed' +DATA_TYPE = 'CSVROW' + +schema_data_folder = Path(__file__).parent / "schemas" +schemaFilePath = schema_data_folder / "test_school_schema.json" + +class TestValidator(unittest.TestCase): + def setUp(self): + with open(schemaFilePath) as JSON: + SchemaFile = json.load(JSON) + self.validator = Validator(SchemaFile) + + def test_run_validation_csv_row_success(self): + + good_row = '202223,202223,Spring term,Local authority,E92000001,England,E12000004,' \ + 'East Midlands,E06000016,Leicester,856,Primary,66,23057,2367094,' \ + '2380687,13593,166808,99826,66982,34090,2547495,1157575,1180365,2337940,29154' + error_report = self.validator.validate_csv_row(good_row, CSV_HEADER, True, True, True) + self.assertTrue(error_report == []) + + def test_run_validation_csv_row_failure(self): + # empty time_period + bad_row = '202223,,Spring term,Local authority,E92000001,England,E12000004,' \ + 'East Midlands,E06000016,Leicester,856,Primary,66,23057,2367094,' \ + '2380687,13593,166808,99826,66982,34090,2547495,1157575,1180365,2337940,29154' + error_report = self.validator.validate_csv_row(bad_row, CSV_HEADER, True, True, True) + self.assertTrue(len(error_report) > 0) + error = error_report[0] + self.assertEqual(error.id, 'check2') + self.assertEqual(error.message, 'Value not empty failure') + self.assertEqual(error.name, 'Time Period Not Empty Check') + self.assertEqual(error.details, 'Value is empty, not as expected') + diff --git a/lambdas/shared/tests/test_common/validator/test_application_fhir.py b/lambdas/shared/tests/test_common/validator/test_application_fhir.py index d2ae176ce0..bd25811c4a 100644 --- a/lambdas/shared/tests/test_common/validator/test_application_fhir.py +++ b/lambdas/shared/tests/test_common/validator/test_application_fhir.py @@ -1,48 +1,47 @@ # Test application file -from pathlib import Path -from common.validator.validator import Validator import json import time import unittest +from pathlib import Path + +from common.validator.validator import Validator + +# TODO this needs success and fail cases class TestApplication(unittest.TestCase): def setUp(self): - fhir_data_folder = Path("./data") - self.FHIRFilePath = fhir_data_folder / "vaccination.json" - - self.schema_data_folder = Path("./schemas") - self.schemaFilePath = self.schema_data_folder / "schema.json" + validation_folder = Path(__file__).parent + self.FHIRFilePath = validation_folder / "data/vaccination2.json" + self.schemaFilePath = validation_folder / "schemas/schema.json" def test_validation(self): - - DATA_TYPE = 'FHIR' - start = time.time() # get the JSON of the schema, changed to cope with elasticache - with open(self.schemaFilePath, 'r') as JSON: + with open(self.schemaFilePath) as JSON: SchemaFile = json.load(JSON) - # get the FHIR Data as JSON - with open(self.FHIRFilePath, 'r') as JSON: - FHIRData = json.load(JSON) - - validator = Validator(self.FHIRFilePath, FHIRData, SchemaFile, '', '', - DATA_TYPE) # FHIR File Path not needed - error_list = validator.run_validation(True, True, True) - error_report = validator.build_error_report( - '25a8cc4d-1875-4191-ac6d-2d63a0ebc64b') # include eventID if known + validator = Validator(SchemaFile) # FHIR File Path not needed + error_list = validator.validate_fhir(self.FHIRFilePath, True, True, True) + error_report = validator.build_error_report('25a8cc4d-1875-4191-ac6d-2d63a0ebc64b') # include eventID if known failed_validation = validator.has_validation_failed() - self.assertTrue(len(error_list) == 0, - f"Validation failed. Errors: {error_list}") - self.assertTrue(len(error_report['errors']) == 0, - f"Validation failed. Errors: {error_report['errors']}") + if len(error_list) > 0: + print(error_list) + else: + print('Validated Successfully') + + print('--------------------------------------------------------------------') + print(error_report) + print('--------------------------------------------------------------------') - self.assertFalse(failed_validation, 'Validation failed') + if (failed_validation): + print('Validation failed due to a critical validation failure...') + else: + print('Validation Successful, see reports for details') end = time.time() print('Time Taken : ') diff --git a/lambdas/shared/tests/test_common/validator/test_expression_checker.py b/lambdas/shared/tests/test_common/validator/test_expression_checker.py index 7d083ce4d3..27ef1fff85 100644 --- a/lambdas/shared/tests/test_common/validator/test_expression_checker.py +++ b/lambdas/shared/tests/test_common/validator/test_expression_checker.py @@ -1,26 +1,22 @@ import unittest -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock +from unittest.mock import patch + +import common.validator.enums.exception_messages as ExceptionMessages from common.validator.expression_checker import ExpressionChecker +# TODO this needs to be expanded to cover all expression types + class TestExpressionChecker(unittest.TestCase): def setUp(self): - patcher1 = patch('common.validator.expression_checker.LookUpData') - patcher2 = patch('common.validator.expression_checker.KeyData') - patcher3 = patch('common.validator.expression_checker.RecordError') - patcher4 = patch('common.validator.expression_checker.ErrorReport') - self.MockLookUpData = patcher1.start() - self.MockKeyData = patcher2.start() - self.MockRecordError = patcher3.start() - self.MockErrorReport = patcher4.start() - - self.addCleanup(patcher1.stop) - self.addCleanup(patcher2.stop) - self.addCleanup(patcher3.stop) - self.addCleanup(patcher4.stop) + self.MockLookUpData = patch( + 'common.validator.expression_checker.LookUpData').start() + self.MockKeyData = patch( + 'common.validator.expression_checker.KeyData').start() self.mock_summarise = MagicMock() self.mock_report_exception = MagicMock() @@ -32,51 +28,59 @@ def setUp(self): self.mock_report_exception ) + def tearDown(self): + patch.stopall() + def test_validate_datetime_valid(self): - result = self.expression_checker.validateExpression( + result = self.expression_checker.validate_expression( "DATETIME", rule="", field_name="timestamp", field_value="2022-01-01T12:00:00", row={} ) - self.assertTrue(self.MockErrorReport.called or result is None) + self.assertEqual(result.message, "Unexpected exception [ValueError]: Invalid isoformat string: '2022-01-01T12:00:00'") + self.assertEqual(result.code, ExceptionMessages.UNEXPECTED_EXCEPTION) + self.assertEqual(result.field, "timestamp") def test_validate_uuid_valid(self): - result = self.expression_checker.validateExpression( + result = self.expression_checker.validate_expression( "UUID", rule="", field_name="id", field_value="550e8400-e29b-41d4-a716-446655440000", row={} ) - self.assertTrue(self.MockErrorReport.called or result is None) + self.assertTrue(result is None) def test_validate_integer_invalid(self): - result = self.expression_checker.validateExpression( + result = self.expression_checker.validate_expression( "INT", rule="", field_name="age", - field_value="notanint", + field_value="hello world", row={} ) - self.assertTrue(self.MockRecordError.called or result is not None) + self.assertEqual(result.code, ExceptionMessages.UNEXPECTED_EXCEPTION) + self.assertEqual(result.field, "age") + self.assertIn("invalid literal for int()", result.message) def test_validate_in_array(self): # Mock data_parser.get_key_values self.mock_data_parser.get_key_values.return_value = ["val1", "val2"] - result = self.expression_checker.validateExpression( + result = self.expression_checker.validate_expression( "INARRAY", rule="", field_name="some_field", field_value="val2", row={} ) - self.assertTrue(result is None or self.MockErrorReport.called) + self.assertEqual(result.message, "Value not in array check failed") + self.assertEqual(result.field, "some_field") def test_validate_expression_type_not_found(self): - result = self.expression_checker.validateExpression( + result = self.expression_checker.validate_expression( "UNKNOWN", rule="", field_name="field", diff --git a/lambdas/shared/tests/test_common/validator/test_parser.py b/lambdas/shared/tests/test_common/validator/test_parser.py index 01fd29d415..ca4f77e1f7 100644 --- a/lambdas/shared/tests/test_common/validator/test_parser.py +++ b/lambdas/shared/tests/test_common/validator/test_parser.py @@ -1,14 +1,14 @@ # Test application file import unittest - from pathlib import Path + from common.validator.parsers.fhir_parser import FHIRParser class TestParse(unittest.TestCase): def setUp(self): - self.fhir_data_folder = Path("./data") + self.fhir_data_folder = Path(__file__).parent / "data" def test_parse_fhir_key_exists(self): diff --git a/lambdas/shared/tests/test_common/validator/test_validator.py b/lambdas/shared/tests/test_common/validator/test_validator.py index 058c90ef65..00e78e8ab1 100644 --- a/lambdas/shared/tests/test_common/validator/test_validator.py +++ b/lambdas/shared/tests/test_common/validator/test_validator.py @@ -15,10 +15,13 @@ # self.mock_expression_checker = patch("validator.validation_expression_checker.ExpressionChecker").start() import unittest -from unittest.mock import patch, MagicMock -from common.validator.validator import Validator -from common.validator.record_error import ErrorReport +from unittest.mock import MagicMock +from unittest.mock import patch + import common.validator.enums.error_levels as ErrorLevels +from common.validator.record_error import ErrorReport +from common.validator.validator import DataType +from common.validator.validator import Validator class TestValidator(unittest.TestCase): @@ -43,16 +46,16 @@ def test_run_validation_csv(self): ] self.mock_expression_checker.return_value.validate_expression.return_value = None - validator = Validator(filepath='file.csv', data_type='CSV') - result = validator.run_validation() + validator = Validator() + result = validator.validate_csv('file.csv') self.assertIsInstance(result, list) self.assertEqual(len(result), 0) def test_run_validation_fhir_exception(self): # Simulate exception in FHIRParser self.mock_fhir_parser.side_effect = Exception("FHIR error") - validator = Validator(filepath='file.fhir', data_type='FHIR') - result = validator.run_validation() + validator = Validator() + result = validator.validate_fhir('file.fhir') self.assertIsInstance(result, list) self.assertEqual(result[0].code, 0) self.assertIn('Data Parser Unexpected exception', result[0].message) @@ -61,8 +64,8 @@ def test_run_validation_schema_exception(self): # Simulate exception in SchemaParser self.mock_fhir_parser.return_value.parse_fhir_file.return_value = None self.mock_schema_parser.side_effect = Exception("Schema error") - validator = Validator(filepath='file.fhir', data_type='FHIR') - result = validator.run_validation() + validator = Validator() + result = validator.validate_fhir('file.fhir') self.assertIsInstance(result, list) self.assertEqual(result[0].code, 0) self.assertIn('Schema Parser Unexpected exception', result[0].message) @@ -73,8 +76,8 @@ def test_run_validation_expression_checker_exception(self): self.mock_schema_parser.return_value.parse_schema.return_value = None self.mock_schema_parser.return_value.get_expressions.return_value = [] self.mock_expression_checker.side_effect = Exception("ExpressionChecker error") - validator = Validator(filepath='file.fhir', data_type='FHIR') - result = validator.run_validation() + validator = Validator() + result = validator.validate_fhir('file.fhir') self.assertIsInstance(result, list) self.assertEqual(result[0].code, 0) self.assertIn('Expression Checker Unexpected exception', result[0].message) @@ -85,8 +88,8 @@ def test_run_validation_expression_getter_exception(self): self.mock_schema_parser.return_value.parse_schema.return_value = None self.mock_schema_parser.return_value.get_expressions.side_effect = Exception("Expressions error") self.mock_expression_checker.return_value = MagicMock() - validator = Validator(filepath='file.fhir', data_type='FHIR') - result = validator.run_validation() + validator = Validator() + result = validator.validate_fhir('file.fhir') self.assertIsInstance(result, list) self.assertEqual(result[0].code, 0) self.assertIn('Expression Getter Unexpected exception', result[0].message) @@ -94,7 +97,7 @@ def test_run_validation_expression_getter_exception(self): @patch('common.validator.validator.DQReporter') def test_build_error_report(self, mock_dq_reporter): mock_dq_reporter.return_value.generate_error_report.return_value = {'report': 'ok'} - validator = Validator(filepath='file.csv', data_type='CSV') + validator = Validator(data_type=DataType.CSV, filepath='file.csv') validator.data_parser = MagicMock() validator.data_parser.get_key_single_value.return_value = '2023-01-01' result = validator.build_error_report('event123') @@ -102,11 +105,11 @@ def test_build_error_report(self, mock_dq_reporter): mock_dq_reporter.return_value.generate_error_report.assert_called_once() def test_has_validation_failed_true(self): - v = Validator(data_type='CSV') + v = Validator(data_type=DataType.CSV) v.error_records.append(ErrorReport(error_level=ErrorLevels.CRITICAL_ERROR)) # Assuming 2 is CRITICAL_ERROR self.assertTrue(v.has_validation_failed()) def test_has_validation_failed_false(self): - v = Validator(data_type='CSV') + v = Validator(data_type=DataType.CSV) v.error_records.append(ErrorReport(error_level=ErrorLevels.WARNING)) self.assertFalse(v.has_validation_failed()) From 28577787e625c054483a1d6a175eaf56a3b10608 Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Tue, 14 Oct 2025 21:19:29 +0100 Subject: [PATCH 07/13] tidy --- lambdas/shared/Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lambdas/shared/Makefile b/lambdas/shared/Makefile index dbb8b04c9b..0ecd1882c5 100644 --- a/lambdas/shared/Makefile +++ b/lambdas/shared/Makefile @@ -1,10 +1,5 @@ -TEST_ENV := @PYTHONPATH=src:tests:src/common - test: - $(TEST_ENV) python -m unittest discover -s tests -p "test_*.py" -v - -testv: - $(TEST_ENV) python -m unittest discover -s tests/test_common/validator -p "test_*.py" -v + @PYTHONPATH=src:../ python -m unittest discover -s tests -p "test_*.py" -v test-list: @PYTHONPATH=src:tests python -m unittest discover -s tests -p "test_*.py" --verbose | grep test_ From 1b62bf9ff41bdf1ea05ac83a33ffc4dc5a00acf1 Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Tue, 14 Oct 2025 21:20:25 +0100 Subject: [PATCH 08/13] tidy --- e2e_batch/Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/e2e_batch/Makefile b/e2e_batch/Makefile index 839037ef10..eb97587d05 100644 --- a/e2e_batch/Makefile +++ b/e2e_batch/Makefile @@ -1,6 +1,4 @@ -include .env -.PHONY: test - test: - ENVIRONMENT=$(ENVIRONMENT) poetry run python -m unittest -v -c \ No newline at end of file + ENVIRONMENT=$(ENVIRONMENT) poetry run python -m unittest -v -c \ No newline at end of file From 80b201dfa14ec5d1506c7b47b1270552812cc7f5 Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Tue, 14 Oct 2025 21:21:42 +0100 Subject: [PATCH 09/13] tidy --- lambdas/shared/Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lambdas/shared/Makefile b/lambdas/shared/Makefile index 0ecd1882c5..dbb8b04c9b 100644 --- a/lambdas/shared/Makefile +++ b/lambdas/shared/Makefile @@ -1,5 +1,10 @@ +TEST_ENV := @PYTHONPATH=src:tests:src/common + test: - @PYTHONPATH=src:../ python -m unittest discover -s tests -p "test_*.py" -v + $(TEST_ENV) python -m unittest discover -s tests -p "test_*.py" -v + +testv: + $(TEST_ENV) python -m unittest discover -s tests/test_common/validator -p "test_*.py" -v test-list: @PYTHONPATH=src:tests python -m unittest discover -s tests -p "test_*.py" --verbose | grep test_ From ff742c711213d53598dffa67db532108644879c4 Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Tue, 14 Oct 2025 21:22:36 +0100 Subject: [PATCH 10/13] tidy --- .../test_common/validator/test_validator.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lambdas/shared/tests/test_common/validator/test_validator.py b/lambdas/shared/tests/test_common/validator/test_validator.py index 00e78e8ab1..a29c1b4fe2 100644 --- a/lambdas/shared/tests/test_common/validator/test_validator.py +++ b/lambdas/shared/tests/test_common/validator/test_validator.py @@ -1,19 +1,3 @@ -# import unittest -# from unittest.mock import patch, MagicMock -# from common.validator.validator import Validator -# from common.validator.enums.exception_messages import PARSING_ERROR, PARENT_FAILED -# from common.validator.enums.error_levels import CRITICAL_ERROR -# from common.validator.validation_expression_checker import ErrorReport - - -# class TestValidator(unittest.TestCase): - -# def setUp(self): -# self.mock_logger_info = patch("validator.logger.info").start() -# self.mock_csv_parser = patch("validator.parsers.csv_parser").start() -# self.mock_schema_parser = patch("validator.parsers.schema_parser").start() -# self.mock_expression_checker = patch("validator.validation_expression_checker.ExpressionChecker").start() - import unittest from unittest.mock import MagicMock from unittest.mock import patch From 016238b42d459eff4d4739ff4fc2d4cbf942ffcf Mon Sep 17 00:00:00 2001 From: nhsdevws Date: Wed, 15 Oct 2025 18:08:55 +0100 Subject: [PATCH 11/13] test_application_csv_small --- .../common/validator/parsers/csv_parser.py | 4 +- ...on_csv.py => Xtest_application_csv_old.py} | 0 .../validator/data/test_small_nok.csv | 4 ++ .../validator/data/test_small_ok.csv | 4 ++ .../validator/schemas/test_small_schema.json | 70 +++++++++++++++++++ .../validator/test_application_csv_row.py | 1 - .../validator/test_application_csv_small.py | 26 +++++++ 7 files changed, 107 insertions(+), 2 deletions(-) rename lambdas/shared/tests/test_common/validator/{test_application_csv.py => Xtest_application_csv_old.py} (100%) create mode 100644 lambdas/shared/tests/test_common/validator/data/test_small_nok.csv create mode 100644 lambdas/shared/tests/test_common/validator/data/test_small_ok.csv create mode 100644 lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json create mode 100644 lambdas/shared/tests/test_common/validator/test_application_csv_small.py diff --git a/lambdas/shared/src/common/validator/parsers/csv_parser.py b/lambdas/shared/src/common/validator/parsers/csv_parser.py index 834e049c28..1cced866f2 100644 --- a/lambdas/shared/src/common/validator/parsers/csv_parser.py +++ b/lambdas/shared/src/common/validator/parsers/csv_parser.py @@ -12,9 +12,11 @@ def __init__(self): # parse the CSV into a Dictionary def parse_csv_file(self, csv_filename): input_file = csv.DictReader(open(csv_filename)) + input_file = csv.DictReader(open(csv_filename), delimiter="|") self.csv_file_data = {elem: [] for elem in input_file.fieldnames} + keys = self.csv_file_data.keys() for row in input_file: - for key in self.csv_file_data.keys(): + for key in keys: self.csv_file_data[key].append(row[key]) # --------------------------------------------- diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv.py b/lambdas/shared/tests/test_common/validator/Xtest_application_csv_old.py similarity index 100% rename from lambdas/shared/tests/test_common/validator/test_application_csv.py rename to lambdas/shared/tests/test_common/validator/Xtest_application_csv_old.py diff --git a/lambdas/shared/tests/test_common/validator/data/test_small_nok.csv b/lambdas/shared/tests/test_common/validator/data/test_small_nok.csv new file mode 100644 index 0000000000..3821631506 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/test_small_nok.csv @@ -0,0 +1,4 @@ +NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB +"9674963871"|"SABINA"|"GREIR"|"20190131" +"9454195417"|"SHAQUILLE"|"KOLBERG"|"30110513" +"9730117489"||"RUDD"|"20011114" \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/data/test_small_ok.csv b/lambdas/shared/tests/test_common/validator/data/test_small_ok.csv new file mode 100644 index 0000000000..1df369f0b2 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/data/test_small_ok.csv @@ -0,0 +1,4 @@ +NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB +"9674963871"|"SABINA"|"GREIR"|"20190131" +"9454195417"|"SHAQUILLE"|"KOLBERG"|"20110513" +"9730117489"|"STUART"|"RUDD"|"20011114" \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json b/lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json new file mode 100644 index 0000000000..f9e0e52ecf --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json @@ -0,0 +1,70 @@ +{ + "id": "No 2", + "schemaName": "Small Test Validation", + "version": 1.0, + "releaseDate": "2024-07-17T00:00:00.000Z", + "expressions": [ + { "expressionId" : "check_1", + "fieldNameCSV": "NHS_NUMBER", + "fieldNameFlat": "NHS_NUMBER", + "errorLevel" : 0, + "expression": { + "expressionName": "NHS Number Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "validity" + }, + { + "expressionId" : "check_2", + "fieldNameCSV": "PERSON_FORENAME", + "fieldNameFlat": "PERSON_FORENAME", + "errorLevel" : 0, + "expression": { + "expressionName": "Person Forname Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "check_3", + "fieldNameCSV": "PERSON_SURNAME", + "fieldNameFlat": "PERSON_SURNAME", + "errorLevel" : 0, + "parentExpression": "check_1", + "expression": { + "expressionName": "Person Surname Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "check_3", + "fieldNameCSV": "PERSON_DOB", + "fieldNameFlat": "PERSON_DOB", + "errorLevel" : 0, + "parentExpression": "check_1", + "expression": { + "expressionName": "Person DOB Not Empty Check", + "expressionType": "NOTEMPTY", + "expressionRule": "" + }, + "errorGroup": "completeness" + }, + { + "expressionId" : "check_4", + "fieldNameCSV": "PERSON_DOB", + "fieldNameFlat": "PERSON_DOB", + "errorLevel" : 0, + "parentExpression": "check_1", + "expression": { + "expressionName": "Person DOB Valid Date Check", + "expressionType": "DATE", + "expressionRule": "" + }, + "errorGroup": "completeness" + } + ] +} \ No newline at end of file diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv_row.py b/lambdas/shared/tests/test_common/validator/test_application_csv_row.py index 6daafa18b4..f8bdc75c00 100644 --- a/lambdas/shared/tests/test_common/validator/test_application_csv_row.py +++ b/lambdas/shared/tests/test_common/validator/test_application_csv_row.py @@ -11,7 +11,6 @@ 'approved_educational_activity,overall_absence,authorised_absence,unauthorised_absence,' \ 'late_sessions,possible_sessions,reason_present_am,reason_present_pm,reason_present,' \ 'reason_l_present_late_before_registers_closed' -DATA_TYPE = 'CSVROW' schema_data_folder = Path(__file__).parent / "schemas" schemaFilePath = schema_data_folder / "test_school_schema.json" diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv_small.py b/lambdas/shared/tests/test_common/validator/test_application_csv_small.py new file mode 100644 index 0000000000..44b9d00756 --- /dev/null +++ b/lambdas/shared/tests/test_common/validator/test_application_csv_small.py @@ -0,0 +1,26 @@ +# Test application file +import json +import unittest +from pathlib import Path + +from common.validator.validator import Validator + + +class TestValidator(unittest.TestCase): + def setUp(self): + self.parent_folder = Path(__file__).parent + schema_file_path = self.parent_folder / "schemas/test_small_schema.json" + with open(schema_file_path) as JSON: + self.schema = json.load(JSON) + + def test_run_validation_csv_success(self): + good_file_path = self.parent_folder / "data/test_small_ok.csv" + validator = Validator(self.schema) + error_report = validator.validate_csv(good_file_path, False, True, True) + self.assertTrue(error_report == []) + + def test_run_validation_csv_fails(self): + bad_file_path = self.parent_folder / "data/test_small_nok.csv" + validator = Validator(self.schema) + error_report = validator.validate_csv(bad_file_path, False, True, True) + self.assertTrue(error_report != []) From 11cc1b902835b9afa3949303211ac9b0eb934148 Mon Sep 17 00:00:00 2001 From: Akol125 Date: Fri, 17 Oct 2025 13:03:51 +0100 Subject: [PATCH 12/13] VED-789: Fix lint issues --- .../validator/data/vaccination.json | 342 +++++++++--------- .../validator/data/vaccination2.json | 206 ++++++----- .../test_common/validator/schemas/Schema.json | 61 ++-- .../validator/schemas/test_school_schema.json | 2 +- .../validator/schemas/test_small_schema.json | 25 +- 5 files changed, 316 insertions(+), 320 deletions(-) diff --git a/lambdas/shared/tests/test_common/validator/data/vaccination.json b/lambdas/shared/tests/test_common/validator/data/vaccination.json index def97ad4e1..7cca84c675 100644 --- a/lambdas/shared/tests/test_common/validator/data/vaccination.json +++ b/lambdas/shared/tests/test_common/validator/data/vaccination.json @@ -1,174 +1,170 @@ { - "resourceType": "Immunization", - "status": "completed", - "protocolApplied": [ - { - "targetDisease": [ - { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "14189004", - "display": "Measles (disorder)" - } - ] - }, - { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "36989005", - "display": "Mumps (disorder)" - } - ] - }, - { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "36653000", - "display": "Rubella (disorder)" - } - ] - } - ], - "doseNumberPositiveInt": 1 - } - ], - "reasonCode": [ - { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "1037351000000105" - } - ] - } - ], - "recorded": "2025-09-15T09:49:38+00:00", - "identifier": [ - { - "value": "48a4dde2-929a-47dc-ae9e-0234d6efc40a", - "system": "https://www.ravs.england.nhs.uk/" - } - ], - "patient": { - "reference": "#Patient1" - }, - "contained": [ - { - "id": "Patient1", - "resourceType": "Patient", - "birthDate": "2013-01-08", - "gender": "male", - "address": [ - { - "postalCode": "SP6 4WE" - } - ], - "identifier": [ - { - "system": "https://fhir.nhs.uk/Id/nhs-number", - "value": "9489927848" - } - ], - "name": [ - { - "family": "Taylor", - "given": [ - "Laura" - ] - } - ] - }, - { - "resourceType": "Practitioner", - "id": "Practitioner1", - "name": [ - { - "family": "Taylor", - "given": [ - "Anthony" - ] - } - ] - } - ], - "vaccineCode": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "42223111000001107", - "display": "Influenza Tetra MYL vaccine suspension for injection 0.5ml pre-filled syringes (Mylan)" - } - ] - }, - "manufacturer": { - "display": "Mylan" - }, - "expirationDate": "2025-09-20", - "lotNumber": "BN40379485850", - "extension": [ - { - "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", - "valueCodeableConcept": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "956951000000104", - "display": "RSV vaccination in pregnancy (procedure)" - } - ] - } - } - ], - "occurrenceDateTime": "2025-09-11T07:14:25+00:00", - "primarySource": true, - "site": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "368209003", - "display": "Right arm" - } - ] - }, - "route": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "1210999013", - "display": "Intradermal use" - } - ] - }, - "doseQuantity": { - "value": 0.3, - "unit": "Inhalation - unit of product usage", - "system": "http://snomed.info/sct", - "code": "2622896019" - }, - "performer": [ - { - "actor": { - "type": "Organization", - "identifier": { - "system": "https://fhir.nhs.uk/Id/ods-organization-code", - "value": "RJ1" - } - } - }, - { - "actor": { - "reference": "#Practitioner1" - } - } - ], - "location": { - "identifier": { - "value": "RJC02", - "system": "https://fhir.nhs.uk/Id/ods-organization-code" - } - }, - "id": "976ccbf1-5ced-4541-8664-3953bf2ff026" -} \ No newline at end of file + "resourceType": "Immunization", + "status": "completed", + "protocolApplied": [ + { + "targetDisease": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "14189004", + "display": "Measles (disorder)" + } + ] + }, + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "36989005", + "display": "Mumps (disorder)" + } + ] + }, + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "36653000", + "display": "Rubella (disorder)" + } + ] + } + ], + "doseNumberPositiveInt": 1 + } + ], + "reasonCode": [ + { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "1037351000000105" + } + ] + } + ], + "recorded": "2025-09-15T09:49:38+00:00", + "identifier": [ + { + "value": "48a4dde2-929a-47dc-ae9e-0234d6efc40a", + "system": "https://www.ravs.england.nhs.uk/" + } + ], + "patient": { + "reference": "#Patient1" + }, + "contained": [ + { + "id": "Patient1", + "resourceType": "Patient", + "birthDate": "2013-01-08", + "gender": "male", + "address": [ + { + "postalCode": "SP6 4WE" + } + ], + "identifier": [ + { + "system": "https://fhir.nhs.uk/Id/nhs-number", + "value": "9489927848" + } + ], + "name": [ + { + "family": "Taylor", + "given": ["Laura"] + } + ] + }, + { + "resourceType": "Practitioner", + "id": "Practitioner1", + "name": [ + { + "family": "Taylor", + "given": ["Anthony"] + } + ] + } + ], + "vaccineCode": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "42223111000001107", + "display": "Influenza Tetra MYL vaccine suspension for injection 0.5ml pre-filled syringes (Mylan)" + } + ] + }, + "manufacturer": { + "display": "Mylan" + }, + "expirationDate": "2025-09-20", + "lotNumber": "BN40379485850", + "extension": [ + { + "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "956951000000104", + "display": "RSV vaccination in pregnancy (procedure)" + } + ] + } + } + ], + "occurrenceDateTime": "2025-09-11T07:14:25+00:00", + "primarySource": true, + "site": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "368209003", + "display": "Right arm" + } + ] + }, + "route": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "1210999013", + "display": "Intradermal use" + } + ] + }, + "doseQuantity": { + "value": 0.3, + "unit": "Inhalation - unit of product usage", + "system": "http://snomed.info/sct", + "code": "2622896019" + }, + "performer": [ + { + "actor": { + "type": "Organization", + "identifier": { + "system": "https://fhir.nhs.uk/Id/ods-organization-code", + "value": "RJ1" + } + } + }, + { + "actor": { + "reference": "#Practitioner1" + } + } + ], + "location": { + "identifier": { + "value": "RJC02", + "system": "https://fhir.nhs.uk/Id/ods-organization-code" + } + }, + "id": "976ccbf1-5ced-4541-8664-3953bf2ff026" +} diff --git a/lambdas/shared/tests/test_common/validator/data/vaccination2.json b/lambdas/shared/tests/test_common/validator/data/vaccination2.json index c5bcc102a4..b7e56a6d24 100644 --- a/lambdas/shared/tests/test_common/validator/data/vaccination2.json +++ b/lambdas/shared/tests/test_common/validator/data/vaccination2.json @@ -1,115 +1,113 @@ { - "resourceType": "Immunization", - "id": "d11c69d8-7a50-4a54-a848-7648121e995f", - "contained": [ - { - "resourceType": "Patient", - "id" : "Pat1", - "identifier": [ - { - "system": "https://fhir.nhs.uk/Id/nhs-number", - "value": "9000000009" - } - ], - "name": [ - { - "family": "Taylor", - "given": [ - "Sarah" - ] - } - ], - "gender": "unknown", - "birthDate": "1965-02-28", - "address": [ - { - "postalCode": "EC1A 1BB" - } - ] + "resourceType": "Immunization", + "id": "d11c69d8-7a50-4a54-a848-7648121e995f", + "contained": [ + { + "resourceType": "Patient", + "id": "Pat1", + "identifier": [ + { + "system": "https://fhir.nhs.uk/Id/nhs-number", + "value": "9000000009" + } + ], + "name": [ + { + "family": "Taylor", + "given": ["Sarah"] + } + ], + "gender": "unknown", + "birthDate": "1965-02-28", + "address": [ + { + "postalCode": "EC1A 1BB" + } + ] + } + ], + "extension": [ + { + "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "1324681000000101", + "display": "Administration of first dose of severe acute respiratory syndrome coronavirus 2 vaccine (procedure)" + } + ] } - ], - "extension": [ + } + ], + "status": "completed", + "vaccineCode": { + "coding": [ { - "url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure", - "valueCodeableConcept": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "1324681000000101", - "display": "Administration of first dose of severe acute respiratory syndrome coronavirus 2 vaccine (procedure)" - } - ] + "system": "http://snomed.info/sct", + "code": "39114911000001105", + "display": "COVID-19 Vaccine Vaxzevria (ChAdOx1 S [recombinant]) not less than 2.5x100,000,000 infectious units/0.5ml dose suspension for injection multidose vials (AstraZeneca UK Ltd) (product)" + } + ] + }, + "patient": { + "reference": "#Pat1" + }, + "occurrenceDateTime": "2021-02-07T13:28:17.271+00:00", + "recorded": "2021-02-07", + "primarySource": true, + "manufacturer": { + "display": "AstraZeneca Ltd" + }, + "location": { + "identifier": { + "value": "X99999", + "system": "https://fhir.nhs.uk/Id/ods-organization-code" + } + }, + "lotNumber": "4120Z001", + "expirationDate": "2021-07-02", + "doseQuantity": { + "value": 0.5, + "unit": "ml", + "system": "http://snomed.info/sct", + "code": "258773002" + }, + "performer": [ + { + "actor": { + "type": "Organization", + "identifier": { + "system": "https://fhir.nhs.uk/Id/ods-organization-code", + "value": "B0C4P" } } - ], - "status": "completed", - "vaccineCode": { + } + ], + "reasonCode": [ + { "coding": [ { - "system": "http://snomed.info/sct", - "code": "39114911000001105", - "display": "COVID-19 Vaccine Vaxzevria (ChAdOx1 S [recombinant]) not less than 2.5x100,000,000 infectious units/0.5ml dose suspension for injection multidose vials (AstraZeneca UK Ltd) (product)" + "code": "443684005", + "system": "http://snomed.info/sct" } ] - }, - "patient": { - "reference" : "#Pat1" - }, - "occurrenceDateTime": "2021-02-07T13:28:17.271+00:00", - "recorded": "2021-02-07", - "primarySource": true, - "manufacturer": { - "display": "AstraZeneca Ltd" - }, - "location": { - "identifier": { - "value": "X99999", - "system": "https://fhir.nhs.uk/Id/ods-organization-code" - } - }, - "lotNumber": "4120Z001", - "expirationDate": "2021-07-02", - "doseQuantity": { - "value": 0.5, - "unit": "ml", - "system": "http://snomed.info/sct", - "code": "258773002" - }, - "performer": [ - { - "actor" : { - "type" : "Organization", - "identifier": { - "system": "https://fhir.nhs.uk/Id/ods-organization-code", - "value": "B0C4P" - } - } - } - ], - "reasonCode": [ - { + } + ], + "protocolApplied": [ + { + "targetDisease": [ + { "coding": [ - { - "code": "443684005", - "system": "http://snomed.info/sct" - } + { + "system": "http://snomed.info/sct", + "code": "840539006", + "display": "Disease caused by severe acute respiratory syndrome coronavirus 2 (disorder)" + } ] - } - ], - "protocolApplied": [ - { - "targetDisease": [ - { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "840539006", - "display": "Disease caused by severe acute respiratory syndrome coronavirus 2 (disorder)" - } - ] - } - ], - "doseNumberPositiveInt": 1 - } - ] - } \ No newline at end of file + } + ], + "doseNumberPositiveInt": 1 + } + ] +} diff --git a/lambdas/shared/tests/test_common/validator/schemas/Schema.json b/lambdas/shared/tests/test_common/validator/schemas/Schema.json index 8f6eb8e73c..3b6bb98028 100644 --- a/lambdas/shared/tests/test_common/validator/schemas/Schema.json +++ b/lambdas/shared/tests/test_common/validator/schemas/Schema.json @@ -4,10 +4,11 @@ "version": 1.0, "releaseDate": "2024-07-17T00:00:00.000Z", "expressions": [ - { "expressionId" : "01K5EGR0C7Y1WJ0BC803SQDWK4", + { + "expressionId": "01K5EGR0C7Y1WJ0BC803SQDWK4", "fieldNameFHIR": "contained|#:Patient|identifier|#:https://fhir.nhs.uk/Id/nhs-number|value", "fieldNameFlat": "NHS_NUMBER", - "errorLevel" : 0, + "errorLevel": 0, "expression": { "expressionName": "NHS Number Not Empty Check", "expressionType": "NOTEMPTY", @@ -16,10 +17,10 @@ "errorGroup": "validity" }, { - "expressionId" : "01K5EGR0C7QCEJMWH1R4MBPGQA", + "expressionId": "01K5EGR0C7QCEJMWH1R4MBPGQA", "fieldNameFHIR": "contained|#:Patient|name|#:official|given|0", "fieldNameFlat": "PERSON_FORENAME", - "errorLevel" : 0, + "errorLevel": 0, "expression": { "expressionName": "Person Forname Not Empty Check", "expressionType": "NOTEMPTY", @@ -28,10 +29,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C7RRG9F6FVHJ8HE4QX", + "expressionId": "01K5EGR0C7RRG9F6FVHJ8HE4QX", "fieldNameFHIR": "contained|#:Patient|name|#:official|family", "fieldNameFlat": "PERSON_SURNAME", - "errorLevel" : 0, + "errorLevel": 0, "parentExpression": "01K5EGR0C7Y1WJ0BC803SQDWK4", "expression": { "expressionName": "Person Surname Not Empty Check", @@ -41,10 +42,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C8WSJ9N8RV6T8RJJ4W", + "expressionId": "01K5EGR0C8WSJ9N8RV6T8RJJ4W", "fieldNameFHIR": "performer|#:Organization|actor|identifier|value", "fieldNameFlat": "SITE_CODE", - "errorLevel" : 1, + "errorLevel": 1, "expression": { "expressionName": "Organisation Not Empty Check", "expressionType": "NOTEMPTY", @@ -53,10 +54,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C8M1MVNKTQCE6MSG68", + "expressionId": "01K5EGR0C8M1MVNKTQCE6MSG68", "fieldNameFHIR": "performer|#:Organization|actor|identifier|value", "fieldNameFlat": "SITE_CODE", - "errorLevel" : 0, + "errorLevel": 0, "expression": { "expressionName": "Organisation Look Up Check", "expressionType": "KEYCHECK", @@ -65,10 +66,10 @@ "errorGroup": "consistency" }, { - "expressionId" : "01K5EGR0C8SDQBTNCEP8TJNCCW", + "expressionId": "01K5EGR0C8SDQBTNCEP8TJNCCW", "fieldNameFHIR": "contained|#:Practitioner|name|0|given|0", "fieldNameFlat": "PERFORMING_PROFESSIONAL_FORENAME", - "errorLevel" : 1, + "errorLevel": 1, "expression": { "expressionName": "Practitioner Forename Not Empty Check", "expressionType": "NOTEMPTY", @@ -77,10 +78,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C822RC96QJRR2YX18S", + "expressionId": "01K5EGR0C822RC96QJRR2YX18S", "fieldNameFHIR": "contained|#:Practitioner|name|0|family", "fieldNameFlat": "PERFORMING_PROFESSIONAL_SURNAME", - "errorLevel" : 1, + "errorLevel": 1, "expression": { "expressionName": "Practitioner Surname Not Empty Check", "expressionType": "NOTEMPTY", @@ -89,10 +90,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C84CCDRR0VFSWQNFZP", + "expressionId": "01K5EGR0C84CCDRR0VFSWQNFZP", "fieldNameFHIR": "primarySource", "fieldNameFlat": "PRIMARY_SOURCE", - "errorLevel" : 1, + "errorLevel": 1, "expression": { "expressionName": "Primary Source Not Empty Check", "expressionType": "NOTEMPTY", @@ -101,10 +102,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C8VCX8FX8A7ZV7MQ5J", + "expressionId": "01K5EGR0C8VCX8FX8A7ZV7MQ5J", "fieldNameFHIR": "extension|0|valueCodeableConcept|coding|0|code", "fieldNameFlat": "VACCINATION_PROCEDURE_CODE", - "errorLevel" : 0, + "errorLevel": 0, "expression": { "expressionName": "Procedure Code Not Empty Check", "expressionType": "NOTEMPTY", @@ -113,10 +114,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C85HY6MDNN6TTR1K48", + "expressionId": "01K5EGR0C85HY6MDNN6TTR1K48", "fieldNameFHIR": "extension|0|valueCodeableConcept|coding|0|display", "fieldNameFlat": "VACCINATION_PROCEDURE_TERM", - "errorLevel" : 1, + "errorLevel": 1, "expression": { "expressionName": "Procedure Term Not Empty Check", "expressionType": "NOTEMPTY", @@ -125,10 +126,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C84DDGW567G14AYBC6", + "expressionId": "01K5EGR0C84DDGW567G14AYBC6", "fieldNameFHIR": "protocolApplied|0|doseNumberPositiveInt", "fieldNameFlat": "DOSE_SEQUENCE", - "errorLevel" : 1, + "errorLevel": 1, "expression": { "expressionName": "Dose Sequence Not Empty Check", "expressionType": "NOTEMPTY", @@ -137,10 +138,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C8W3HXFYR80ENW73SS", + "expressionId": "01K5EGR0C8W3HXFYR80ENW73SS", "fieldNameFHIR": "vaccineCode|coding|#:http://snomed.info/sct|code", "fieldNameFlat": "VACCINE_PRODUCT_CODE", - "errorLevel" : 0, + "errorLevel": 0, "expression": { "expressionName": "Produce Code Not Empty Check", "expressionType": "NOTEMPTY", @@ -149,10 +150,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C885N7MMW2J5JKHTT2", + "expressionId": "01K5EGR0C885N7MMW2J5JKHTT2", "fieldNameFHIR": "vaccineCode|coding|#:http://snomed.info/sct|display", "fieldNameFlat": "VACCINE_PRODUCT_TERM", - "errorLevel" : 1, + "errorLevel": 1, "expression": { "expressionName": "Produce Term Not Empty Check", "expressionType": "NOTEMPTY", @@ -161,10 +162,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C86XN0AF0M9DJYFGCD", + "expressionId": "01K5EGR0C86XN0AF0M9DJYFGCD", "fieldNameFHIR": "manufacturer|display", "fieldNameFlat": "VACCINE_MANUFACTURER", - "errorLevel" : 0, + "errorLevel": 0, "expression": { "expressionName": "Manufacturer Display Not Empty Check", "expressionType": "NOTEMPTY", @@ -173,10 +174,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "01K5EGR0C89M4CV68B7XAKDCHG", + "expressionId": "01K5EGR0C89M4CV68B7XAKDCHG", "fieldNameFHIR": "lotNumber", "fieldNameFlat": "BATCH_NUMBER", - "errorLevel" : 0, + "errorLevel": 0, "expression": { "expressionName": "Batch Number Not Empty Check", "expressionType": "NOTEMPTY", diff --git a/lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json b/lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json index ad3fa1e004..5fdc279f56 100644 --- a/lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json +++ b/lambdas/shared/tests/test_common/validator/schemas/test_school_schema.json @@ -161,4 +161,4 @@ "errorGroup": "validity" } ] -} \ No newline at end of file +} diff --git a/lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json b/lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json index f9e0e52ecf..1b14f2c801 100644 --- a/lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json +++ b/lambdas/shared/tests/test_common/validator/schemas/test_small_schema.json @@ -4,10 +4,11 @@ "version": 1.0, "releaseDate": "2024-07-17T00:00:00.000Z", "expressions": [ - { "expressionId" : "check_1", + { + "expressionId": "check_1", "fieldNameCSV": "NHS_NUMBER", "fieldNameFlat": "NHS_NUMBER", - "errorLevel" : 0, + "errorLevel": 0, "expression": { "expressionName": "NHS Number Not Empty Check", "expressionType": "NOTEMPTY", @@ -16,10 +17,10 @@ "errorGroup": "validity" }, { - "expressionId" : "check_2", + "expressionId": "check_2", "fieldNameCSV": "PERSON_FORENAME", "fieldNameFlat": "PERSON_FORENAME", - "errorLevel" : 0, + "errorLevel": 0, "expression": { "expressionName": "Person Forname Not Empty Check", "expressionType": "NOTEMPTY", @@ -28,10 +29,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "check_3", + "expressionId": "check_3", "fieldNameCSV": "PERSON_SURNAME", "fieldNameFlat": "PERSON_SURNAME", - "errorLevel" : 0, + "errorLevel": 0, "parentExpression": "check_1", "expression": { "expressionName": "Person Surname Not Empty Check", @@ -41,10 +42,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "check_3", + "expressionId": "check_3", "fieldNameCSV": "PERSON_DOB", "fieldNameFlat": "PERSON_DOB", - "errorLevel" : 0, + "errorLevel": 0, "parentExpression": "check_1", "expression": { "expressionName": "Person DOB Not Empty Check", @@ -54,10 +55,10 @@ "errorGroup": "completeness" }, { - "expressionId" : "check_4", + "expressionId": "check_4", "fieldNameCSV": "PERSON_DOB", "fieldNameFlat": "PERSON_DOB", - "errorLevel" : 0, + "errorLevel": 0, "parentExpression": "check_1", "expression": { "expressionName": "Person DOB Valid Date Check", @@ -66,5 +67,5 @@ }, "errorGroup": "completeness" } - ] -} \ No newline at end of file + ] +} From 94ff4e1aff6b29b8d3dc93096da0f13570563d6f Mon Sep 17 00:00:00 2001 From: Akol125 Date: Fri, 17 Oct 2025 14:03:43 +0100 Subject: [PATCH 13/13] using ruff to re-format documents --- lambdas/shared/src/common/aws_dynamodb.py | 3 +- lambdas/shared/src/common/aws_lambda_event.py | 4 +- lambdas/shared/src/common/cache.py | 3 +- lambdas/shared/src/common/log_decorator.py | 3 +- lambdas/shared/src/common/s3_reader.py | 3 +- .../common/validator/enums/error_levels.py | 8 +- .../validator/enums/exception_messages.py | 28 +- .../common/validator/expression_checker.py | 492 +++++++++++------- .../src/common/validator/lookup/key_data.py | 201 +++---- .../common/validator/lookup/lookup_data.py | 188 +++---- .../common/validator/parsers/csv_parser.py | 2 +- .../common/validator/parsers/fhir_parser.py | 18 +- .../common/validator/parsers/schema_parser.py | 4 +- .../src/common/validator/record_error.py | 26 +- .../common/validator/reporter/dq_reporter.py | 44 +- .../shared/src/common/validator/validator.py | 117 +++-- .../tests/test_common/test_authentication.py | 3 +- .../shared/tests/test_common/test_cache.py | 4 +- .../validator/Xtest_application_csv_old.py | 4 +- .../validator/test_application_csv_row.py | 41 +- .../validator/test_application_fhir.py | 17 +- .../validator/test_expression_checker.py | 47 +- .../test_common/validator/test_parser.py | 15 +- .../test_common/validator/test_validator.py | 50 +- 24 files changed, 713 insertions(+), 612 deletions(-) diff --git a/lambdas/shared/src/common/aws_dynamodb.py b/lambdas/shared/src/common/aws_dynamodb.py index 67eb1eaf0d..2c4ea698c0 100644 --- a/lambdas/shared/src/common/aws_dynamodb.py +++ b/lambdas/shared/src/common/aws_dynamodb.py @@ -1,4 +1,5 @@ -from common.clients import dynamodb_resource, logger +from common.clients import dynamodb_resource +from common.clients import logger def get_dynamodb_table(table_name): diff --git a/lambdas/shared/src/common/aws_lambda_event.py b/lambdas/shared/src/common/aws_lambda_event.py index b73c898c6c..413c0a5883 100644 --- a/lambdas/shared/src/common/aws_lambda_event.py +++ b/lambdas/shared/src/common/aws_lambda_event.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Dict +from typing import Any class AwsEventType(Enum): @@ -10,7 +10,7 @@ class AwsEventType(Enum): class AwsLambdaEvent: - def __init__(self, event: Dict[str, Any]): + def __init__(self, event: dict[str, Any]): self.event_source = None self.event_type = AwsEventType.UNKNOWN self.event_source = event.get("eventSource") diff --git a/lambdas/shared/src/common/cache.py b/lambdas/shared/src/common/cache.py index f2d24e5e4a..1d34bedef6 100644 --- a/lambdas/shared/src/common/cache.py +++ b/lambdas/shared/src/common/cache.py @@ -1,5 +1,4 @@ import json -from typing import Optional class Cache: @@ -19,7 +18,7 @@ def put(self, key: str, value: dict): self.cache[key] = value self._overwrite() - def get(self, key: str) -> Optional[dict]: + def get(self, key: str) -> dict | None: return self.cache.get(key, None) def delete(self, key: str): diff --git a/lambdas/shared/src/common/log_decorator.py b/lambdas/shared/src/common/log_decorator.py index 63c81610bf..cabbdf5750 100644 --- a/lambdas/shared/src/common/log_decorator.py +++ b/lambdas/shared/src/common/log_decorator.py @@ -9,7 +9,8 @@ from datetime import datetime from functools import wraps -from common.clients import firehose_client, logger +from common.clients import firehose_client +from common.clients import logger def send_log_to_firehose(stream_name, log_data: dict) -> None: diff --git a/lambdas/shared/src/common/s3_reader.py b/lambdas/shared/src/common/s3_reader.py index 6b0505b911..7eb6347565 100644 --- a/lambdas/shared/src/common/s3_reader.py +++ b/lambdas/shared/src/common/s3_reader.py @@ -1,4 +1,5 @@ -from common.clients import logger, s3_client +from common.clients import logger +from common.clients import s3_client class S3Reader: diff --git a/lambdas/shared/src/common/validator/enums/error_levels.py b/lambdas/shared/src/common/validator/enums/error_levels.py index d3111dab98..51a31ffabf 100644 --- a/lambdas/shared/src/common/validator/enums/error_levels.py +++ b/lambdas/shared/src/common/validator/enums/error_levels.py @@ -1,5 +1,3 @@ - - # all error Levels CRITICAL_ERROR = 0 WARNING = 1 @@ -7,7 +5,7 @@ MESSAGES = { - CRITICAL_ERROR: 'Critical Validation Error [%s]: %s', - WARNING: 'Non-Critical Validation Error [%s]: %s', - NOTIFICATION: 'Quality Issue Found [%s]: %s', + CRITICAL_ERROR: "Critical Validation Error [%s]: %s", + WARNING: "Non-Critical Validation Error [%s]: %s", + NOTIFICATION: "Quality Issue Found [%s]: %s", } diff --git a/lambdas/shared/src/common/validator/enums/exception_messages.py b/lambdas/shared/src/common/validator/enums/exception_messages.py index b5cb9990ee..386e025e0e 100644 --- a/lambdas/shared/src/common/validator/enums/exception_messages.py +++ b/lambdas/shared/src/common/validator/enums/exception_messages.py @@ -1,5 +1,3 @@ - - # all exceptions and messgaes UNEXPECTED_EXCEPTION = 0 VALUE_CHECK_FAILED = 1 @@ -16,17 +14,17 @@ KEY_CHECK_FAILED = 12 MESSAGES = { - UNEXPECTED_EXCEPTION: 'Unexpected exception [%s]: %s', - VALUE_CHECK_FAILED: 'Value check failed.', - HEADER_CHECK_FAILED: 'Header check failed.', - RECORD_LENGTH_CHECK_FAILED: 'Record length check failed.', - RECORD_CHECK_FAILED: 'Record check failed.', - VALUE_PREDICATE_FALSE: 'Value predicate returned false.', - RECORD_PREDICATE_FALSE: 'Record predicate returned false.', - UNIQUE_CHECK_FAILED: 'Unique check failed.', - ASSERT_CHECK_FAILED: 'Assertion check failed.', - FINALLY_ASSERT_CHECK_FAILED: 'Final assertion check failed.', - PARSING_ERROR: 'Failed to parse data correctly.', - PARENT_FAILED: 'The parent expression failed to validate', - KEY_CHECK_FAILED: 'Value could not be found in the Key list' + UNEXPECTED_EXCEPTION: "Unexpected exception [%s]: %s", + VALUE_CHECK_FAILED: "Value check failed.", + HEADER_CHECK_FAILED: "Header check failed.", + RECORD_LENGTH_CHECK_FAILED: "Record length check failed.", + RECORD_CHECK_FAILED: "Record check failed.", + VALUE_PREDICATE_FALSE: "Value predicate returned false.", + RECORD_PREDICATE_FALSE: "Record predicate returned false.", + UNIQUE_CHECK_FAILED: "Unique check failed.", + ASSERT_CHECK_FAILED: "Assertion check failed.", + FINALLY_ASSERT_CHECK_FAILED: "Final assertion check failed.", + PARSING_ERROR: "Failed to parse data correctly.", + PARENT_FAILED: "The parent expression failed to validate", + KEY_CHECK_FAILED: "Value could not be found in the Key list", } diff --git a/lambdas/shared/src/common/validator/expression_checker.py b/lambdas/shared/src/common/validator/expression_checker.py index 36b2aa1a96..13152a8b3e 100644 --- a/lambdas/shared/src/common/validator/expression_checker.py +++ b/lambdas/shared/src/common/validator/expression_checker.py @@ -12,36 +12,35 @@ class ExpressionType(Enum): - DATETIME = 'DATETIME' - DATE = 'DATE' - UUID = 'UUID' - INT = 'INT' - FLOAT = 'FLOAT' - REGEX = 'REGEX' - EQUAL = 'EQUAL' - NOTEQUAL = 'NOTEQUAL' - IN = 'IN' - NRANGE = 'NRANGE' - INARRAY = 'INARRAY' - UPPER = 'UPPER' - LOWER = 'LOWER' - LENGTH = 'LENGTH' - STARTSWITH = 'STARTSWITH' - ENDSWITH = 'ENDSWITH' - EMPTY = 'EMPTY' - NOTEMPTY = 'NOTEMPTY' - POSITIVE = 'POSITIVE' - GENDER = 'GENDER' - NHSNUMBER = 'NHSNUMBER' - MAXOBJECTS = 'MAXOBJECTS' - POSTCODE = 'POSTCODE' - ONLYIF = 'ONLYIF' - LOOKUP = 'LOOKUP' - KEYCHECK = 'KEYCHECK' + DATETIME = "DATETIME" + DATE = "DATE" + UUID = "UUID" + INT = "INT" + FLOAT = "FLOAT" + REGEX = "REGEX" + EQUAL = "EQUAL" + NOTEQUAL = "NOTEQUAL" + IN = "IN" + NRANGE = "NRANGE" + INARRAY = "INARRAY" + UPPER = "UPPER" + LOWER = "LOWER" + LENGTH = "LENGTH" + STARTSWITH = "STARTSWITH" + ENDSWITH = "ENDSWITH" + EMPTY = "EMPTY" + NOTEMPTY = "NOTEMPTY" + POSITIVE = "POSITIVE" + GENDER = "GENDER" + NHSNUMBER = "NHSNUMBER" + MAXOBJECTS = "MAXOBJECTS" + POSTCODE = "POSTCODE" + ONLYIF = "ONLYIF" + LOOKUP = "LOOKUP" + KEYCHECK = "KEYCHECK" class ExpressionChecker: - def __init__(self, data_parser, summarise, report_unexpected_exception): self.data_parser = data_parser # FHIR data parser for additional functions self.data_look_up = LookUpData() # used for generic look up @@ -49,7 +48,7 @@ def __init__(self, data_parser, summarise, report_unexpected_exception): self.summarise = summarise self.report_unexpected_exception = report_unexpected_exception - def validate_expression(self, expression_type: str, rule, field_name, field_value, row) -> ErrorReport: + def validate_expression(self, expression_type: str, rule, field_name, field_value, row) -> ErrorReport: match expression_type: case "DATETIME": return self._validate_datetime(rule, field_name, field_value, row) @@ -107,7 +106,7 @@ def validate_expression(self, expression_type: str, rule, field_name, field_val return "Schema expression not found! Check your expression type : " + expression_type # iso8086 date time validate - def _validate_datetime(self, rule, field_name, field_value, row) -> ErrorReport: + def _validate_datetime(self, rule, field_name, field_value, row) -> ErrorReport: try: datetime.date.fromisoformat(field_value) # TODO - rule is not used - could be date only, date time, past, future etc @@ -116,35 +115,40 @@ def _validate_datetime(self, rule, field_name, field_value, row) -> ErrorReport except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # UUID validate - def _validate_uuid(self, expressionRule, field_name, field_value, row) -> ErrorReport: + def _validate_uuid(self, expressionRule, field_name, field_value, row) -> ErrorReport: try: uuid.UUID(str(field_value)) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Integer Validate - def _validate_integer(self, expression_rule, field_name, - field_value, row) -> ErrorReport: + def _validate_integer(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: int(field_value) if expression_rule: @@ -160,71 +164,84 @@ def _validate_integer(self, expression_rule, field_name, check_value = int(expression_rule) if field_value != check_value: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value integer check failed", - "Value does not equal expected value, Expected- " + expression_rule - + " found- " + field_value) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value integer check failed", + "Value does not equal expected value, Expected- " + expression_rule + " found- " + field_value, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Float Validate - def _validate_float(self, expression_rule, field_name, field_value, row) -> ErrorReport: + def _validate_float(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: float(field_value) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Length Validate - def _validate_length(self, expression_rule, field_name, field_value, row) -> ErrorReport: + def _validate_length(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: str_len = len(field_value) check_length = int(expression_rule) if str_len > check_length: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value length check failed", - "Value is longer than expected") + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, "Value length check failed", "Value is longer than expected" + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = ( - e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Regex Validate - def _validate_regex(self, expression_rule, field_name, field_value, row) -> ErrorReport: + def _validate_regex(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: result = re.search(expression_rule, field_value) if not result: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "String REGEX check failed", - "Value does not meet regex rules") + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "String REGEX check failed", + "Value does not meet regex rules", + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = ( - e.message if e.message is not None + e.message + if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] ) if e.details is not None: @@ -233,68 +250,85 @@ def _validate_regex(self, expression_rule, field_name, field_value, row) -> Err except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Equal Validate - def _validate_equal(self, expression_rule, field_name, field_value, row) -> ErrorReport: + def _validate_equal(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: if field_value != expression_rule: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value equals check failed", - "Value does not equal expected value, Expected- " + expression_rule + " found- " - + field_value) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value equals check failed", + "Value does not equal expected value, Expected- " + expression_rule + " found- " + field_value, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = ( - e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Not Equal Validate - def _validate_not_equal(self, expression_rule, field_name, field_value, row) -> ErrorReport: + def _validate_not_equal(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: if field_value == expression_rule: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value not equals check failed", - "Value equals expected value when it should not, Expected- " + expression_rule - + " found- " + field_value) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value not equals check failed", + "Value equals expected value when it should not, Expected- " + + expression_rule + + " found- " + + field_value, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # In Validate - def _validate_in(self, expression_rule, field_name, field_value, row) -> ErrorReport: + def _validate_in(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: if expression_rule.lower() in field_value.lower(): - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Data not in Value failed", "Check Data not found in Value, List- " - + expression_rule + " Data- " + field_value) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Data not in Value failed", + "Check Data not found in Value, List- " + expression_rule + " Data- " + field_value, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # NRange Validate - def _validate_n_range(self, expression_rule, field_name, field_value, row) -> ErrorReport: + def _validate_n_range(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: value = float(field_value) rule = expression_rule.split(",") @@ -302,42 +336,52 @@ def _validate_n_range(self, expression_rule, field_name, field_value, row) -> E range2 = float(rule[1]) if range1 <= value >= range2: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value range check failed", "Value is not within the number range, data- " - + field_value) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value range check failed", + "Value is not within the number range, data- " + field_value, + ) return None except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED message = ( - e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # InArray Validate - def _validate_in_array(self, expression_rule, field_name, field_value, row) -> ErrorReport: + def _validate_in_array(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: rule_list = expression_rule.split(",") if field_value not in rule_list: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, "Value not in array check failed", - "Check Value not found in data array") + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value not in array check failed", + "Check Value not found in data array", + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Upper Validate def _validate_upper(self, expression_rule, field_name, field_value, row) -> ErrorReport: @@ -345,19 +389,25 @@ def _validate_upper(self, expression_rule, field_name, field_value, row) -> Erro result = field_value.isupper() if not result: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value not uppercase", "Check Value not found to be uppercase, value- " + field_value) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value not uppercase", + "Check Value not found to be uppercase, value- " + field_value, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Lower Validate def _validate_lower(self, expression_rule, field_name, field_value, row) -> ErrorReport: @@ -365,204 +415,248 @@ def _validate_lower(self, expression_rule, field_name, field_value, row) -> Erro result = field_value.islower() if not result: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value not lowercase", - "Check Value not found to be lowercase, data- " + field_value) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value not lowercase", + "Check Value not found to be lowercase, data- " + field_value, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Starts With Validate def _validate_starts_with(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: result = field_value.startswith(expression_rule) if not result: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value starts with failure", - "Value does not start as expected, Expected- " + expression_rule - + " found- " + field_value) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value starts with failure", + "Value does not start as expected, Expected- " + expression_rule + " found- " + field_value, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Ends With Validate def _validate_ends_with(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: result = field_value.endswith(expression_rule) if not result: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value ends with failure", - "Value does not end as expected, Expected- " + expression_rule - + " found- " + field_value) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value ends with failure", + "Value does not end as expected, Expected- " + expression_rule + " found- " + field_value, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Empty Validate def _validate_empty(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: if field_value: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value is empty failure", - "Value has data, not as expected, data- " + field_value) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value is empty failure", + "Value has data, not as expected, data- " + field_value, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Not Empty Validate def _validate_not_empty(self, expression_rule, field_name, field_value, row) -> ErrorReport: try: if not field_value: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value not empty failure", - "Value is empty, not as expected") + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, "Value not empty failure", "Value is empty, not as expected" + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, field_name, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, field_name, "", self.summarise) # Positive Validate - def _validate_positive(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_positive(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: value = float(fieldValue) if value < 0: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value is not positive failure", - "Value is not positive as expected, data- " + fieldValue) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value is not positive failure", + "Value is not positive as expected, data- " + fieldValue, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, fieldName, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) # NHSNumber Validate - def _validate_nhs_number(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_nhs_number(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: - regexRule = '^6[0-9]{10}$' + regexRule = "^6[0-9]{10}$" result = re.search(regexRule, fieldValue) if not result: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "NHS Number check failed", - "NHS Number does not meet regex rules, data- " + fieldValue) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "NHS Number check failed", + "NHS Number does not meet regex rules, data- " + fieldValue, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, fieldName, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) # Gender Validate - def _validate_gender(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_gender(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: - ruleList = ['0', '1', '2', '9'] + ruleList = ["0", "1", "2", "9"] if fieldValue not in ruleList: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Gender check failed", - "Gender value not found in array, data- " + fieldValue) + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Gender check failed", + "Gender value not found in array, data- " + fieldValue, + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, fieldName, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) # PostCode Validate - def _validate_post_code(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_post_code(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: - regexRule = '^([Gg][Ii][Rr] 0[Aa]{2})|((([A-Za-z][0-9]{1,2})|(([A-Za-z][A-Ha-hJ-Yj-y]' - '[0-9]{1,2})|(([AZa-z][0-9][A-Za-z])|([A-Za-z][A-Ha-hJ-Yj-y][0-9]?[A-Za-z])))) [0-9][A-Za-z]{2})$' + regexRule = "^([Gg][Ii][Rr] 0[Aa]{2})|((([A-Za-z][0-9]{1,2})|(([A-Za-z][A-Ha-hJ-Yj-y]" + "[0-9]{1,2})|(([AZa-z][0-9][A-Za-z])|([A-Za-z][A-Ha-hJ-Yj-y][0-9]?[A-Za-z])))) [0-9][A-Za-z]{2})$" result = re.search(regexRule, fieldValue) if not result: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Postcode check failed", - "Postcode does not meet regex rules") + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, "Postcode check failed", "Postcode does not meet regex rules" + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, fieldName, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) # Max Objects Validate - def _validate_max_objects(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_max_objects(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: value = len(fieldValue) if value > int(expressionRule): - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Max Objects failure", - "Number of objects is greater than expected") + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Max Objects failure", + "Number of objects is greater than expected", + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, fieldName, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) # Default to Validate def _validate_only_if(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: @@ -572,59 +666,71 @@ def _validate_only_if(self, expressionRule, fieldName, fieldValue, row) -> Error valueCheck = conversionList[1] dataValue = self.dataParser.getKeyValue(location) - if (dataValue[0] != valueCheck): - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Validate Only If failure", - "Value was not found at that position") + if dataValue[0] != valueCheck: + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Validate Only If failure", + "Value was not found at that position", + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, fieldName, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) # Check with Lookup - def _validate_against_lookup(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_against_lookup(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: result = self.dataLookUp.findLookUp(fieldValue) if not result: - raise RecordError(ExceptionMessages.RECORD_CHECK_FAILED, - "Value lookup failure", - "Value was not found in Lookup List, Expected- " + fieldValue + " found- nothing") + raise RecordError( + ExceptionMessages.RECORD_CHECK_FAILED, + "Value lookup failure", + "Value was not found in Lookup List, Expected- " + fieldValue + " found- nothing", + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.RECORD_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED]) + message = ( + e.message + if e.message is not None + else ExceptionMessages.MESSAGES[ExceptionMessages.RECORD_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, fieldName, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) # Check with Key Lookup - def _validate_against_key(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: + def _validate_against_key(self, expressionRule, fieldName, fieldValue, row) -> ErrorReport: try: result = self.KeyData.findKey(expressionRule, fieldValue) if not result: - raise RecordError(ExceptionMessages.KEY_CHECK_FAILED, - "Key lookup failure", - "Value was not found in Key List, Expected- " - + fieldValue + " found- nothing") + raise RecordError( + ExceptionMessages.KEY_CHECK_FAILED, + "Key lookup failure", + "Value was not found in Key List, Expected- " + fieldValue + " found- nothing", + ) except RecordError as e: code = e.code if e.code is not None else ExceptionMessages.KEY_CHECK_FAILED - message = (e.message if e.message is not None - else ExceptionMessages.MESSAGES[ExceptionMessages.KEY_CHECK_FAILED]) + message = ( + e.message if e.message is not None else ExceptionMessages.MESSAGES[ExceptionMessages.KEY_CHECK_FAILED] + ) if e.details is not None: details = e.details return ErrorReport(code, message, row, fieldName, details, self.summarise) except Exception as e: if self.report_unexpected_exception: message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, e) - return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, '', self.summarise) + return ErrorReport(ExceptionMessages.UNEXPECTED_EXCEPTION, message, row, fieldName, "", self.summarise) diff --git a/lambdas/shared/src/common/validator/lookup/key_data.py b/lambdas/shared/src/common/validator/lookup/key_data.py index bcddbce463..deb3071aeb 100644 --- a/lambdas/shared/src/common/validator/lookup/key_data.py +++ b/lambdas/shared/src/common/validator/lookup/key_data.py @@ -3,109 +3,124 @@ class KeyData: - # data settings def __init__(self): - self.procedure = ['956951000000104'] + self.procedure = ["956951000000104"] - self.organisation = ['RJ1', 'RJC02'] + self.organisation = ["RJ1", "RJC02"] - self.site = ['368208006', '279549004', '74262004', '368209003', - '723979003', '61396006', '723980000', '11207009', '420254004'] + self.site = [ + "368208006", + "279549004", + "74262004", + "368209003", + "723979003", + "61396006", + "723980000", + "11207009", + "420254004", + ] - self.route = ['54471007', - '372449004', - '372450004', - '372451000', - '372452007', - '404820008', - '18246711000001107', - '372453002', - '372454008', - '127490009', - '372457001', - '9191401000001100', - '18682911000001103', - '10334211000001103', - '718329006', - '18679011000001101', - '34777511000001106', - '372458006', - '58100008', - '12130007', - '372459003', - '418821007', - '372460008', - '372461007', - '420719007', - '19537211000001108', - '372463005', - '372464004', - '372465003', - '448077001', - '38233211000001106', - '372466002', - '372467006', - '78421000', - '255559005', - '372468001', - '417255000', - '38239002', - '372469009', - '372470005', - '418586008', - '72607000', - '447122006', - '62226000', - '47625008', - '420287000', - '372471009', - '418401004', - '21856811000001103', - '127491008', - '9907001000001103', - '46713006', - '127492001', - '418730005', - '54485002', - '26643006', - '372473007', - '10547007', - '225691000001105', - '9191501000001101', - '372474001', - '39338211000001108', - '3323001000001107', - '372475000', - '11478901000001102', - '419762003', - '39337511000001107', - '37161004', - '11564311000001109', - '418321004', - '3594011000001102', - '372476004', - '34206005', - '37839007', - '419874009', - '11564211000001101', - '33770711000001104', - '6064005', - '45890007', - '11479001000001107', - '404815008', - '90028008', - '16857009'] + self.route = [ + "54471007", + "372449004", + "372450004", + "372451000", + "372452007", + "404820008", + "18246711000001107", + "372453002", + "372454008", + "127490009", + "372457001", + "9191401000001100", + "18682911000001103", + "10334211000001103", + "718329006", + "18679011000001101", + "34777511000001106", + "372458006", + "58100008", + "12130007", + "372459003", + "418821007", + "372460008", + "372461007", + "420719007", + "19537211000001108", + "372463005", + "372464004", + "372465003", + "448077001", + "38233211000001106", + "372466002", + "372467006", + "78421000", + "255559005", + "372468001", + "417255000", + "38239002", + "372469009", + "372470005", + "418586008", + "72607000", + "447122006", + "62226000", + "47625008", + "420287000", + "372471009", + "418401004", + "21856811000001103", + "127491008", + "9907001000001103", + "46713006", + "127492001", + "418730005", + "54485002", + "26643006", + "372473007", + "10547007", + "225691000001105", + "9191501000001101", + "372474001", + "39338211000001108", + "3323001000001107", + "372475000", + "11478901000001102", + "419762003", + "39337511000001107", + "37161004", + "11564311000001109", + "418321004", + "3594011000001102", + "372476004", + "34206005", + "37839007", + "419874009", + "11564211000001101", + "33770711000001104", + "6064005", + "45890007", + "11479001000001107", + "404815008", + "90028008", + "16857009", + ] # Look up the term for the code def findKey(self, key_source, field_value): try: match key_source: - case "Procedure": return field_value in self.Procedure - case "Organisation": field_value in self.Organisation - case "Site": field_value in self.Site - case "Route": field_value in self.Route - case _: return False + case "Procedure": + return field_value in self.Procedure + case "Organisation": + return field_value in self.Organisation + case "Site": + return field_value in self.Site + case "Route": + return field_value in self.Route + case _: + return False except Exception: return False return False diff --git a/lambdas/shared/src/common/validator/lookup/lookup_data.py b/lambdas/shared/src/common/validator/lookup/lookup_data.py index 47bbf5a07a..42eadbb528 100644 --- a/lambdas/shared/src/common/validator/lookup/lookup_data.py +++ b/lambdas/shared/src/common/validator/lookup/lookup_data.py @@ -5,103 +5,105 @@ class LookUpData: # data settings def __init__(self): - self.all_data = {'368208006': 'Left upper arm structure', - '279549004': 'Nasal cavity structure', - '74262004': 'Oral cavity structure', - '368209003': 'Right upper arm structure', - '723979003': 'Structure of left buttock', - '61396006': 'Structure of left thigh', - '723980000': 'Structure of right buttock', - '11207009': 'Structure of right thigh', - '420254004': 'Body cavity', - '54471007': 'Buccal', - '372449004': 'Dental', - '372450004': 'Endocervical', - '372451000': 'Endosinusial', - '372452007': 'Endotracheopulmonary', - '404820008': 'Epidural', - '18246711000001107': 'Epilesional', - '372453002': 'Extraamniotic', - '372454008': 'Gastroenteral', - '127490009': 'Gastrostomy route', - '372457001': 'Gingival', - '9191401000001100': 'Haemodiafiltration', - '18682911000001103': 'Haemodialysis', - '10334211000001103': 'Haemofiltration', - '718329006': 'Infiltration', - '18679011000001101': 'Inhalation', - '34777511000001106': 'Intestinal use', - '372458006': 'Intraamniotic', - '58100008': 'Intraarterial', - '12130007': 'Intraarticular', - '372459003': 'Intrabursal', - '418821007': 'Intracameral use', - '372460008': 'Intracardiac', - '372461007': 'Intracavernous', - '420719007': 'Intracerebroventricular', - '19537211000001108': 'Intracervical route', - '372463005': 'Intracoronary', - '372464004': 'Intradermal', - '372465003': 'Intradiscal', - '448077001': 'Intraepidermal', - '38233211000001106': 'Intraglandular', - '372466002': 'Intralesional', - '372467006': 'Intralymphatic', - '78421000': 'Intramuscular', - '255559005': 'Intramuscular', - '372468001': 'Intraocular', - '417255000': 'Intraosseous', - '38239002': 'Intraperitoneal', - '372469009': 'Intrapleural', - '372470005': 'Intrasternal', - '418586008': 'Intratendinous route', - '72607000': 'Intrathecal', - '447122006': 'Intratumoral', - '62226000': 'Intrauterine', - '47625008': 'Intravenous', - '420287000': 'Intraventricular cardiac', - '372471009': 'Intravesical', - '418401004': 'Intravitreal', - '21856811000001103': 'Iontophoresis', - '127491008': 'Jejunostomy route', - '9907001000001103': 'Line lock', - '46713006': 'Nasal', - '127492001': 'Nasogastric route', - '418730005': 'Nasojejunal route', - '54485002': 'Ophthalmic route', - '26643006': 'Oral', - '372473007': 'Oromucosal', - '10547007': 'Otic', - '225691000001105': 'PEG tube route', - '9191501000001101': 'Percutaneous', - '372474001': 'Periarticular', - '39338211000001108': 'Peribulbar ocular', - '3323001000001107': 'Pericardial route', - '372475000': 'Perineural', - '11478901000001102': 'Periosseous', - '419762003': 'Peritendinous route', - '39337511000001107': 'Peritumoral', - '37161004': 'Rectal', - '11564311000001109': 'Regional perfusion', - '418321004': 'Retrobulbar route', - '3594011000001102': 'Route of administration not applicable', - '372476004': 'Subconjunctival', - '34206005': 'Subcutaneous', - '37839007': 'Sublingual', - '419874009': 'Submucosal', - '11564211000001101': 'Submucosal rectal', - '33770711000001104': 'Subretinal', - '6064005': 'Topical', - '45890007': 'Transdermal', - '11479001000001107': 'Translingual', - '404815008': 'Transmucosal', - '90028008': 'Urethral', - '16857009': 'Vaginal'} + self.all_data = { + "368208006": "Left upper arm structure", + "279549004": "Nasal cavity structure", + "74262004": "Oral cavity structure", + "368209003": "Right upper arm structure", + "723979003": "Structure of left buttock", + "61396006": "Structure of left thigh", + "723980000": "Structure of right buttock", + "11207009": "Structure of right thigh", + "420254004": "Body cavity", + "54471007": "Buccal", + "372449004": "Dental", + "372450004": "Endocervical", + "372451000": "Endosinusial", + "372452007": "Endotracheopulmonary", + "404820008": "Epidural", + "18246711000001107": "Epilesional", + "372453002": "Extraamniotic", + "372454008": "Gastroenteral", + "127490009": "Gastrostomy route", + "372457001": "Gingival", + "9191401000001100": "Haemodiafiltration", + "18682911000001103": "Haemodialysis", + "10334211000001103": "Haemofiltration", + "718329006": "Infiltration", + "18679011000001101": "Inhalation", + "34777511000001106": "Intestinal use", + "372458006": "Intraamniotic", + "58100008": "Intraarterial", + "12130007": "Intraarticular", + "372459003": "Intrabursal", + "418821007": "Intracameral use", + "372460008": "Intracardiac", + "372461007": "Intracavernous", + "420719007": "Intracerebroventricular", + "19537211000001108": "Intracervical route", + "372463005": "Intracoronary", + "372464004": "Intradermal", + "372465003": "Intradiscal", + "448077001": "Intraepidermal", + "38233211000001106": "Intraglandular", + "372466002": "Intralesional", + "372467006": "Intralymphatic", + "78421000": "Intramuscular", + "255559005": "Intramuscular", + "372468001": "Intraocular", + "417255000": "Intraosseous", + "38239002": "Intraperitoneal", + "372469009": "Intrapleural", + "372470005": "Intrasternal", + "418586008": "Intratendinous route", + "72607000": "Intrathecal", + "447122006": "Intratumoral", + "62226000": "Intrauterine", + "47625008": "Intravenous", + "420287000": "Intraventricular cardiac", + "372471009": "Intravesical", + "418401004": "Intravitreal", + "21856811000001103": "Iontophoresis", + "127491008": "Jejunostomy route", + "9907001000001103": "Line lock", + "46713006": "Nasal", + "127492001": "Nasogastric route", + "418730005": "Nasojejunal route", + "54485002": "Ophthalmic route", + "26643006": "Oral", + "372473007": "Oromucosal", + "10547007": "Otic", + "225691000001105": "PEG tube route", + "9191501000001101": "Percutaneous", + "372474001": "Periarticular", + "39338211000001108": "Peribulbar ocular", + "3323001000001107": "Pericardial route", + "372475000": "Perineural", + "11478901000001102": "Periosseous", + "419762003": "Peritendinous route", + "39337511000001107": "Peritumoral", + "37161004": "Rectal", + "11564311000001109": "Regional perfusion", + "418321004": "Retrobulbar route", + "3594011000001102": "Route of administration not applicable", + "372476004": "Subconjunctival", + "34206005": "Subcutaneous", + "37839007": "Sublingual", + "419874009": "Submucosal", + "11564211000001101": "Submucosal rectal", + "33770711000001104": "Subretinal", + "6064005": "Topical", + "45890007": "Transdermal", + "11479001000001107": "Translingual", + "404815008": "Transmucosal", + "90028008": "Urethral", + "16857009": "Vaginal", + } # Look up the term for the code def find_lookup(self, field_value): try: lookup_value = self.all_data[field_value] except Exception: - lookup_value = '' + lookup_value = "" return lookup_value diff --git a/lambdas/shared/src/common/validator/parsers/csv_parser.py b/lambdas/shared/src/common/validator/parsers/csv_parser.py index 1cced866f2..9116c8b6b6 100644 --- a/lambdas/shared/src/common/validator/parsers/csv_parser.py +++ b/lambdas/shared/src/common/validator/parsers/csv_parser.py @@ -3,8 +3,8 @@ class CSVParser: + """File Management""" - """ File Management""" # parser variables def __init__(self): self.csv_file_data = {} diff --git a/lambdas/shared/src/common/validator/parsers/fhir_parser.py b/lambdas/shared/src/common/validator/parsers/fhir_parser.py index 44f3620eb5..1eb32bb212 100644 --- a/lambdas/shared/src/common/validator/parsers/fhir_parser.py +++ b/lambdas/shared/src/common/validator/parsers/fhir_parser.py @@ -24,7 +24,7 @@ def parse_fhir_data(self, fhir_data): def _scan_values_for_match(self, parent, match_value): try: for key in parent: - if (parent[key] == match_value): + if parent[key] == match_value: return True return False except Exception: @@ -38,7 +38,7 @@ def _locate_list_id(self, parent, locator): try: while index < len(parent): for key in parent[index]: - if ((parent[index][key] == field_list[1]) or (key == field_list[1])): + if (parent[index][key] == field_list[1]) or (key == field_list[1]): node_id = index break else: @@ -47,7 +47,7 @@ def _locate_list_id(self, parent, locator): break index += 1 except Exception: - return '' + return "" return parent[node_id] # identify a node in the FHIR data @@ -60,7 +60,7 @@ def _get_node(self, parent, child): child = int(child) result = parent[child] except Exception: - result = '' + result = "" return result # locate a value for a key @@ -71,12 +71,12 @@ def _scan_for_value(self, fhir_fields): del field_list[0] try: for field in field_list: - if (field.startswith("#")): + if field.startswith("#"): rootfield = self._locate_list_id(rootfield, field) # check here for default index?? else: rootfield = self._get_node(rootfield, field) except Exception: - rootfield = '' + rootfield = "" return rootfield # get the value list for a key @@ -85,18 +85,18 @@ def get_key_value(self, field_name): try: response_value = self._scan_for_value(field_name) except Exception: - response_value = '' + response_value = "" value.append(response_value) return value # get the value list for a key def get_key_single_value(self, field_name): - value = '' + value = "" try: response_value = self._scan_for_value(field_name) except Exception: - response_value = '' + response_value = "" value = response_value return value diff --git a/lambdas/shared/src/common/validator/parsers/schema_parser.py b/lambdas/shared/src/common/validator/parsers/schema_parser.py index 6278487a77..3294f3fbac 100644 --- a/lambdas/shared/src/common/validator/parsers/schema_parser.py +++ b/lambdas/shared/src/common/validator/parsers/schema_parser.py @@ -10,11 +10,11 @@ def __init__(self): def parse_schema(self, schema_file): # changed to accept JSON better for cache self.schema_file = schema_file - self.expressions = self.schema_file['expressions'] + self.expressions = self.schema_file["expressions"] def expression_count(self): count = 0 - count = sum([1 for d in self.expressions if 'expression' in d]) + count = sum([1 for d in self.expressions if "expression" in d]) return count def get_expressions(self): diff --git a/lambdas/shared/src/common/validator/record_error.py b/lambdas/shared/src/common/validator/record_error.py index b709281074..f9bd0442ed 100644 --- a/lambdas/shared/src/common/validator/record_error.py +++ b/lambdas/shared/src/common/validator/record_error.py @@ -1,8 +1,14 @@ - - class ErrorReport: - def __init__(self, code: int = None, message: str = None, row: int = None, field: str = None, details: str = None, - summarise: bool = False, error_level: int = None): + def __init__( + self, + code: int = None, + message: str = None, + row: int = None, + field: str = None, + details: str = None, + summarise: bool = False, + error_level: int = None, + ): self.code = code self.message = message self.row = row @@ -17,22 +23,14 @@ def __init__(self, code: int = None, message: str = None, row: int = None, field # function to return the object as a dictionary def to_dict(self): - ret = { - 'code': self.code, - 'message': self.message - } + ret = {"code": self.code, "message": self.message} if not self.summarise: - ret.update({ - 'row': self.row, - 'field': self.field, - 'details': self.details - }) + ret.update({"row": self.row, "field": self.field, "details": self.details}) return ret # record exception capture class RecordError(Exception): - def __init__(self, code=None, message=None, details=None): super().__init__(message) self.code = code diff --git a/lambdas/shared/src/common/validator/reporter/dq_reporter.py b/lambdas/shared/src/common/validator/reporter/dq_reporter.py index b88510ede4..9874c8814f 100644 --- a/lambdas/shared/src/common/validator/reporter/dq_reporter.py +++ b/lambdas/shared/src/common/validator/reporter/dq_reporter.py @@ -8,34 +8,24 @@ class DQReporter: - def __init__(self): # parser variables self.error_report = { "eventId": "", "validationDate": "", - "validated": 'true', + "validated": "true", "results": { "totalErrors": 0, - "completeness": { - "errors": 0, - "fields": [] - }, - "consistency": { - "errors": 0, - "fields": [] - }, - "validity": { - "errors": 0, - "fields": [] - }, - "timeliness_processed": 0 - } + "completeness": {"errors": 0, "fields": []}, + "consistency": {"errors": 0, "fields": []}, + "validity": {"errors": 0, "fields": []}, + "timeliness_processed": 0, + }, } # create the date difference for the report in minutes def diff_dates(self, date1, date2): - diff_seconds = abs(date2-date1).total_seconds() + diff_seconds = abs(date2 - date1).total_seconds() diff_minutes = diff_seconds / 60 return diff_minutes @@ -46,9 +36,9 @@ def generate_error_report(self, event_id, occurrence, error_records: list[ErrorR time_taken = self.diff_dates(occurrence_date, validation_date) - self.error_report['validationDate'] = validation_date.isoformat() - self.error_report['eventId'] = event_id - self.error_report['results']['timeliness_processed'] = time_taken + self.error_report["validationDate"] = validation_date.isoformat() + self.error_report["eventId"] = event_id + self.error_report["results"]["timeliness_processed"] = time_taken for errorRecord in error_records: self.update_report(errorRecord) @@ -58,12 +48,12 @@ def generate_error_report(self, event_id, occurrence, error_records: list[ErrorR def update_report(self, error_data: ErrorReport): error_group = error_data.error_group - if (error_data.error_level == ErrorLevels.CRITICAL_ERROR): - self.error_report['validated'] = "false" - total_errors = self.error_report['results']['totalErrors'] - results_error_count = self.error_report['results'][error_group]['errors'] + if error_data.error_level == ErrorLevels.CRITICAL_ERROR: + self.error_report["validated"] = "false" + total_errors = self.error_report["results"]["totalErrors"] + results_error_count = self.error_report["results"][error_group]["errors"] results_error_count += 1 total_errors += 1 - self.error_report['results'][error_group]['fields'].append(error_data.name) - self.error_report['results'][error_group]['errors'] = results_error_count - self.error_report['results']['totalErrors'] = total_errors + self.error_report["results"][error_group]["fields"].append(error_data.name) + self.error_report["results"][error_group]["errors"] = results_error_count + self.error_report["results"]["totalErrors"] = total_errors diff --git a/lambdas/shared/src/common/validator/validator.py b/lambdas/shared/src/common/validator/validator.py index 95021a6f70..70fc7e996c 100644 --- a/lambdas/shared/src/common/validator/validator.py +++ b/lambdas/shared/src/common/validator/validator.py @@ -14,22 +14,21 @@ class DataType(Enum): - FHIR = 'FHIR' - FHIRJSON = 'FHIRJSON' - CSV = 'CSV' - CSVROW = 'CSVROW' + FHIR = "FHIR" + FHIRJSON = "FHIRJSON" + CSV = "CSV" + CSVROW = "CSVROW" class Validator: - - def __init__(self, schema_file = '', data_type: DataType = None, filepath = ''): + def __init__(self, schema_file="", data_type: DataType = None, filepath=""): self.filepath = filepath self.json_data = {} self.schema_file = schema_file - self.csv_row = '' - self.csv_header = '' + self.csv_row = "" + self.csv_header = "" self.data_type = data_type - self.data_parser = '' + self.data_parser = "" self.error_records: list[ErrorReport] = [] def _get_csv_line_parser(self, csv_row, csv_header): @@ -57,8 +56,9 @@ def _get_schema_parser(self, schemafile): schema_parser.parse_schema(schemafile) return schema_parser - def _add_error_record(self, error_record: ErrorReport, - expression_error_group, expression_name, expression_id, error_level): + def _add_error_record( + self, error_record: ErrorReport, expression_error_group, expression_name, expression_id, error_level + ): if error_record is not None: error_record.error_group = expression_error_group error_record.name = expression_name @@ -69,91 +69,102 @@ def _add_error_record(self, error_record: ErrorReport, # Function to help identify a parent failure in the error list def _check_error_record_for_fail(self, expression_id): for error_record in self.error_records: - if (error_record.id == expression_id): + if error_record.id == expression_id: return True return False # validate a single expression against the data file - def _validate_expression(self, expression_validator: ExpressionChecker, expression, - inc_header_in_row_count) -> ErrorReport | int: + def _validate_expression( + self, expression_validator: ExpressionChecker, expression, inc_header_in_row_count + ) -> ErrorReport | int: row = 1 if inc_header_in_row_count: row = 2 if self.isCSV: - expression_fieldname = expression['fieldNameCSV'] + expression_fieldname = expression["fieldNameCSV"] else: - expression_fieldname = expression['fieldNameFHIR'] + expression_fieldname = expression["fieldNameFHIR"] - expression_id = expression['expressionId'] - error_level = expression['errorLevel'] - expression_name = expression['expression']['expressionName'] - expression_type = expression['expression']['expressionType'] - expression_rule = expression['expression']['expressionRule'] - expression_error_group = expression['errorGroup'] + expression_id = expression["expressionId"] + error_level = expression["errorLevel"] + expression_name = expression["expression"]["expressionName"] + expression_type = expression["expression"]["expressionType"] + expression_rule = expression["expression"]["expressionRule"] + expression_error_group = expression["errorGroup"] # Check to see if the expression has a parent, if so did the parent validate - if ('parentExpression' in expression): - parent_expression = expression['parentExpression'] - if (self._check_error_record_for_fail(parent_expression)): - error_record = {'code': ExceptionMessages.PARENT_FAILED, - 'message': ExceptionMessages.MESSAGES[ExceptionMessages.PARENT_FAILED] - + ', Parent ID: ' + parent_expression} - self._add_error_record(error_record, expression_error_group, expression_name, expression_id, error_level) + if "parentExpression" in expression: + parent_expression = expression["parentExpression"] + if self._check_error_record_for_fail(parent_expression): + error_record = { + "code": ExceptionMessages.PARENT_FAILED, + "message": ExceptionMessages.MESSAGES[ExceptionMessages.PARENT_FAILED] + + ", Parent ID: " + + parent_expression, + } + self._add_error_record( + error_record, expression_error_group, expression_name, expression_id, error_level + ) return error_record try: expression_values = self.data_parser.get_key_value(expression_fieldname) except Exception as e: - message = f'Data get values Unexpected exception [{e.__class__.__name__}]: {e}' + message = f"Data get values Unexpected exception [{e.__class__.__name__}]: {e}" error_report = ErrorReport(code=ExceptionMessages.PARSING_ERROR, message=message) # original code had self.CriticalErrorLevel. Replaced with error_level - self._add_error_record(error_report, - expression_error_group, expression_name, expression_id, error_level) + self._add_error_record(error_report, expression_error_group, expression_name, expression_id, error_level) return error_report for value in expression_values: error_record: ErrorReport | None = None try: error_record = expression_validator.validate_expression( - expression_type, expression_rule, expression_fieldname, value, row) + expression_type, expression_rule, expression_fieldname, value, row + ) if error_record is not None: - self._add_error_record(error_record, expression_error_group, - expression_name, expression_id, error_level) + self._add_error_record( + error_record, expression_error_group, expression_name, expression_id, error_level + ) except Exception: - print(f'Exception validating expression {expression_id} on row {row}: {error_record}') + print(f"Exception validating expression {expression_id} on row {row}: {error_record}") row += 1 return row - - def validate_fhir(self, filepath, summarise=False, report_unexpected_exception=True, - inc_header_in_row_count=True) -> list[ErrorReport]: + def validate_fhir( + self, filepath, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True + ) -> list[ErrorReport]: self.data_type = DataType.FHIR self.filepath = filepath return self.run_validation(summarise, report_unexpected_exception, inc_header_in_row_count) - def validate_csv(self, filepath, summarise=False, report_unexpected_exception=True, - inc_header_in_row_count=True) -> list[ErrorReport]: + def validate_csv( + self, filepath, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True + ) -> list[ErrorReport]: self.data_type = DataType.CSV self.filepath = filepath return self.run_validation(summarise, report_unexpected_exception, inc_header_in_row_count) - def validate_csv_row(self, csv_row, csv_header, summarise=False, report_unexpected_exception=True, - inc_header_in_row_count=True) -> list[ErrorReport]: + def validate_csv_row( + self, csv_row, csv_header, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True + ) -> list[ErrorReport]: self.data_type = DataType.CSVROW self.csv_row = csv_row self.csv_header = csv_header return self.run_validation(summarise, report_unexpected_exception, inc_header_in_row_count) - def validate_fhir_json(self, json_data, summarise=False, report_unexpected_exception=True, - inc_header_in_row_count=True) -> list[ErrorReport]: + def validate_fhir_json( + self, json_data, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True + ) -> list[ErrorReport]: self.data_type = DataType.FHIRJSON self.json_data = json_data return self.validate_fhir_json(json_data, summarise, report_unexpected_exception, inc_header_in_row_count) # run the validation against the data - def run_validation(self, summarise=False, report_unexpected_exception=True, - inc_header_in_row_count=True) -> list[ErrorReport]: + def run_validation( + self, summarise=False, report_unexpected_exception=True, inc_header_in_row_count=True + ) -> list[ErrorReport]: try: self.error_records.clear() @@ -173,21 +184,21 @@ def run_validation(self, summarise=False, report_unexpected_exception=True, except Exception as e: if report_unexpected_exception: - message = f'Data Parser Unexpected exception [{e.__class__.__name__}]: {e}' + message = f"Data Parser Unexpected exception [{e.__class__.__name__}]: {e}" return [ErrorReport(code=0, message=message)] try: schemaParser = self._get_schema_parser(self.schema_file) except Exception as e: if report_unexpected_exception: - message = f'Schema Parser Unexpected exception [{e.__class__.__name__}]: {e}' + message = f"Schema Parser Unexpected exception [{e.__class__.__name__}]: {e}" return [ErrorReport(code=0, message=message)] try: expression_validator = ExpressionChecker(self.data_parser, summarise, report_unexpected_exception) except Exception as e: if report_unexpected_exception: - message = f'Expression Checker Unexpected exception [{e.__class__.__name__}]: {e}' + message = f"Expression Checker Unexpected exception [{e.__class__.__name__}]: {e}" return [ErrorReport(code=0, message=message)] # get list of expressions @@ -195,7 +206,7 @@ def run_validation(self, summarise=False, report_unexpected_exception=True, expressions = schemaParser.get_expressions() except Exception as e: if report_unexpected_exception: - message = f'Expression Getter Unexpected exception [{e.__class__.__name__}]: {e}' + message = f"Expression Getter Unexpected exception [{e.__class__.__name__}]: {e}" return [ErrorReport(code=0, message=message)] for expression in expressions: @@ -207,7 +218,7 @@ def run_validation(self, summarise=False, report_unexpected_exception=True, # Report Generation # Build the error Report def build_error_report(self, eventId): - OccurrenceDateTime = self.data_parser.get_key_single_value('occurrenceDateTime') + OccurrenceDateTime = self.data_parser.get_key_single_value("occurrenceDateTime") dq_reporter = DQReporter() dq_report = dq_reporter.generate_error_report(eventId, OccurrenceDateTime, self.error_records) @@ -216,6 +227,6 @@ def build_error_report(self, eventId): # Check all errors to see if we have a critical error that would fail the validation def has_validation_failed(self): for error_record in self.error_records: - if (error_record.error_level == ErrorLevels.CRITICAL_ERROR): + if error_record.error_level == ErrorLevels.CRITICAL_ERROR: return True return False diff --git a/lambdas/shared/tests/test_common/test_authentication.py b/lambdas/shared/tests/test_common/test_authentication.py index 498dc2439f..69cc81781a 100644 --- a/lambdas/shared/tests/test_common/test_authentication.py +++ b/lambdas/shared/tests/test_common/test_authentication.py @@ -9,7 +9,8 @@ import responses from responses import matchers -from common.authentication import AppRestrictedAuth, Service +from common.authentication import AppRestrictedAuth +from common.authentication import Service from common.models.errors import UnhandledResponseError diff --git a/lambdas/shared/tests/test_common/test_cache.py b/lambdas/shared/tests/test_common/test_cache.py index 3e3a2bd194..8125099ac8 100644 --- a/lambdas/shared/tests/test_common/test_cache.py +++ b/lambdas/shared/tests/test_common/test_cache.py @@ -70,7 +70,7 @@ def test_write_to_file(self): self.cache.put(key, new_value) # Then - with open(self.cache.cache_file.name, "r") as stored: + with open(self.cache.cache_file.name) as stored: content = json.loads(stored.read()) self.assertDictEqual(content[key], new_value) @@ -83,6 +83,6 @@ def test_cache_create_empty(self): self.cache = Cache(tempfile.gettempdir()) # Then - with open(self.cache.cache_file.name, "r") as stored: + with open(self.cache.cache_file.name) as stored: content = stored.read() self.assertEqual(len(content), 0) diff --git a/lambdas/shared/tests/test_common/validator/Xtest_application_csv_old.py b/lambdas/shared/tests/test_common/validator/Xtest_application_csv_old.py index 824219606b..1b16daa32d 100644 --- a/lambdas/shared/tests/test_common/validator/Xtest_application_csv_old.py +++ b/lambdas/shared/tests/test_common/validator/Xtest_application_csv_old.py @@ -11,7 +11,7 @@ data_folder = parent_folder / "data" csvFilePath = data_folder / "test_data_ok.csv" # Passes -dataType = 'CSV' +dataType = "CSV" schemaFilePath = parent_folder / "schemas/test_school_schema.json" @@ -28,7 +28,7 @@ if len(error_report) > 0: print(error_report) else: - print('Validated Successfully') + print("Validated Successfully") end = time.time() print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/test_application_csv_row.py b/lambdas/shared/tests/test_common/validator/test_application_csv_row.py index f8bdc75c00..6a9e38d684 100644 --- a/lambdas/shared/tests/test_common/validator/test_application_csv_row.py +++ b/lambdas/shared/tests/test_common/validator/test_application_csv_row.py @@ -5,16 +5,19 @@ from common.validator.validator import Validator -CSV_HEADER = 'academic_year,time_period,time_identifier,geographic_level,' \ -'country_code,country_name,region_code,region_name,new_la_code,la_name,' \ -'old_la_code,school_type,num_schools,enrolments,present_sessions,overall_attendance,' \ -'approved_educational_activity,overall_absence,authorised_absence,unauthorised_absence,' \ -'late_sessions,possible_sessions,reason_present_am,reason_present_pm,reason_present,' \ -'reason_l_present_late_before_registers_closed' +CSV_HEADER = ( + "academic_year,time_period,time_identifier,geographic_level," + "country_code,country_name,region_code,region_name,new_la_code,la_name," + "old_la_code,school_type,num_schools,enrolments,present_sessions,overall_attendance," + "approved_educational_activity,overall_absence,authorised_absence,unauthorised_absence," + "late_sessions,possible_sessions,reason_present_am,reason_present_pm,reason_present," + "reason_l_present_late_before_registers_closed" +) schema_data_folder = Path(__file__).parent / "schemas" schemaFilePath = schema_data_folder / "test_school_schema.json" + class TestValidator(unittest.TestCase): def setUp(self): with open(schemaFilePath) as JSON: @@ -22,23 +25,25 @@ def setUp(self): self.validator = Validator(SchemaFile) def test_run_validation_csv_row_success(self): - - good_row = '202223,202223,Spring term,Local authority,E92000001,England,E12000004,' \ - 'East Midlands,E06000016,Leicester,856,Primary,66,23057,2367094,' \ - '2380687,13593,166808,99826,66982,34090,2547495,1157575,1180365,2337940,29154' + good_row = ( + "202223,202223,Spring term,Local authority,E92000001,England,E12000004," + "East Midlands,E06000016,Leicester,856,Primary,66,23057,2367094," + "2380687,13593,166808,99826,66982,34090,2547495,1157575,1180365,2337940,29154" + ) error_report = self.validator.validate_csv_row(good_row, CSV_HEADER, True, True, True) self.assertTrue(error_report == []) def test_run_validation_csv_row_failure(self): # empty time_period - bad_row = '202223,,Spring term,Local authority,E92000001,England,E12000004,' \ - 'East Midlands,E06000016,Leicester,856,Primary,66,23057,2367094,' \ - '2380687,13593,166808,99826,66982,34090,2547495,1157575,1180365,2337940,29154' + bad_row = ( + "202223,,Spring term,Local authority,E92000001,England,E12000004," + "East Midlands,E06000016,Leicester,856,Primary,66,23057,2367094," + "2380687,13593,166808,99826,66982,34090,2547495,1157575,1180365,2337940,29154" + ) error_report = self.validator.validate_csv_row(bad_row, CSV_HEADER, True, True, True) self.assertTrue(len(error_report) > 0) error = error_report[0] - self.assertEqual(error.id, 'check2') - self.assertEqual(error.message, 'Value not empty failure') - self.assertEqual(error.name, 'Time Period Not Empty Check') - self.assertEqual(error.details, 'Value is empty, not as expected') - + self.assertEqual(error.id, "check2") + self.assertEqual(error.message, "Value not empty failure") + self.assertEqual(error.name, "Time Period Not Empty Check") + self.assertEqual(error.details, "Value is empty, not as expected") diff --git a/lambdas/shared/tests/test_common/validator/test_application_fhir.py b/lambdas/shared/tests/test_common/validator/test_application_fhir.py index bd25811c4a..11efba5c25 100644 --- a/lambdas/shared/tests/test_common/validator/test_application_fhir.py +++ b/lambdas/shared/tests/test_common/validator/test_application_fhir.py @@ -11,7 +11,6 @@ class TestApplication(unittest.TestCase): def setUp(self): - validation_folder = Path(__file__).parent self.FHIRFilePath = validation_folder / "data/vaccination2.json" self.schemaFilePath = validation_folder / "schemas/schema.json" @@ -25,24 +24,24 @@ def test_validation(self): validator = Validator(SchemaFile) # FHIR File Path not needed error_list = validator.validate_fhir(self.FHIRFilePath, True, True, True) - error_report = validator.build_error_report('25a8cc4d-1875-4191-ac6d-2d63a0ebc64b') # include eventID if known + error_report = validator.build_error_report("25a8cc4d-1875-4191-ac6d-2d63a0ebc64b") # include eventID if known failed_validation = validator.has_validation_failed() if len(error_list) > 0: print(error_list) else: - print('Validated Successfully') + print("Validated Successfully") - print('--------------------------------------------------------------------') + print("--------------------------------------------------------------------") print(error_report) - print('--------------------------------------------------------------------') + print("--------------------------------------------------------------------") - if (failed_validation): - print('Validation failed due to a critical validation failure...') + if failed_validation: + print("Validation failed due to a critical validation failure...") else: - print('Validation Successful, see reports for details') + print("Validation Successful, see reports for details") end = time.time() - print('Time Taken : ') + print("Time Taken : ") print(end - start) diff --git a/lambdas/shared/tests/test_common/validator/test_expression_checker.py b/lambdas/shared/tests/test_common/validator/test_expression_checker.py index 27ef1fff85..8833a1fc6d 100644 --- a/lambdas/shared/tests/test_common/validator/test_expression_checker.py +++ b/lambdas/shared/tests/test_common/validator/test_expression_checker.py @@ -1,4 +1,3 @@ - import unittest from unittest.mock import MagicMock from unittest.mock import patch @@ -10,22 +9,16 @@ class TestExpressionChecker(unittest.TestCase): - def setUp(self): - - self.MockLookUpData = patch( - 'common.validator.expression_checker.LookUpData').start() - self.MockKeyData = patch( - 'common.validator.expression_checker.KeyData').start() + self.MockLookUpData = patch("common.validator.expression_checker.LookUpData").start() + self.MockKeyData = patch("common.validator.expression_checker.KeyData").start() self.mock_summarise = MagicMock() self.mock_report_exception = MagicMock() self.mock_data_parser = MagicMock() self.expression_checker = ExpressionChecker( - self.mock_data_parser, - self.mock_summarise, - self.mock_report_exception + self.mock_data_parser, self.mock_summarise, self.mock_report_exception ) def tearDown(self): @@ -33,33 +26,23 @@ def tearDown(self): def test_validate_datetime_valid(self): result = self.expression_checker.validate_expression( - "DATETIME", - rule="", - field_name="timestamp", - field_value="2022-01-01T12:00:00", - row={} + "DATETIME", rule="", field_name="timestamp", field_value="2022-01-01T12:00:00", row={} + ) + self.assertEqual( + result.message, "Unexpected exception [ValueError]: Invalid isoformat string: '2022-01-01T12:00:00'" ) - self.assertEqual(result.message, "Unexpected exception [ValueError]: Invalid isoformat string: '2022-01-01T12:00:00'") self.assertEqual(result.code, ExceptionMessages.UNEXPECTED_EXCEPTION) self.assertEqual(result.field, "timestamp") def test_validate_uuid_valid(self): result = self.expression_checker.validate_expression( - "UUID", - rule="", - field_name="id", - field_value="550e8400-e29b-41d4-a716-446655440000", - row={} + "UUID", rule="", field_name="id", field_value="550e8400-e29b-41d4-a716-446655440000", row={} ) self.assertTrue(result is None) def test_validate_integer_invalid(self): result = self.expression_checker.validate_expression( - "INT", - rule="", - field_name="age", - field_value="hello world", - row={} + "INT", rule="", field_name="age", field_value="hello world", row={} ) self.assertEqual(result.code, ExceptionMessages.UNEXPECTED_EXCEPTION) self.assertEqual(result.field, "age") @@ -70,21 +53,13 @@ def test_validate_in_array(self): self.mock_data_parser.get_key_values.return_value = ["val1", "val2"] result = self.expression_checker.validate_expression( - "INARRAY", - rule="", - field_name="some_field", - field_value="val2", - row={} + "INARRAY", rule="", field_name="some_field", field_value="val2", row={} ) self.assertEqual(result.message, "Value not in array check failed") self.assertEqual(result.field, "some_field") def test_validate_expression_type_not_found(self): result = self.expression_checker.validate_expression( - "UNKNOWN", - rule="", - field_name="field", - field_value="value", - row={} + "UNKNOWN", rule="", field_name="field", field_value="value", row={} ) self.assertIn("Schema expression not found", result) diff --git a/lambdas/shared/tests/test_common/validator/test_parser.py b/lambdas/shared/tests/test_common/validator/test_parser.py index ca4f77e1f7..cc4834e661 100644 --- a/lambdas/shared/tests/test_common/validator/test_parser.py +++ b/lambdas/shared/tests/test_common/validator/test_parser.py @@ -6,26 +6,23 @@ class TestParse(unittest.TestCase): - def setUp(self): self.fhir_data_folder = Path(__file__).parent / "data" def test_parse_fhir_key_exists(self): - fhirFilePath = self.fhir_data_folder / "vaccination.json" fhir_parser = FHIRParser() fhir_parser.parse_fhir_file(fhirFilePath) - my_value = fhir_parser.get_key_value('vaccineCode|coding|0|code') - self.assertEqual(my_value, ['42223111000001107']) + my_value = fhir_parser.get_key_value("vaccineCode|coding|0|code") + self.assertEqual(my_value, ["42223111000001107"]) def test_parse_fhir_key_not_exists(self): - fhirFilePath = self.fhir_data_folder / "vaccination.json" fhir_parser = FHIRParser() fhir_parser.parse_fhir_file(fhirFilePath) - my_value = fhir_parser.get_key_value('vaccineCode|coding|1') - self.assertEqual(my_value, ['']) - my_value = fhir_parser.get_key_value('vaccineCode|coding|1|codes') - self.assertEqual(my_value, ['']) + my_value = fhir_parser.get_key_value("vaccineCode|coding|1") + self.assertEqual(my_value, [""]) + my_value = fhir_parser.get_key_value("vaccineCode|coding|1|codes") + self.assertEqual(my_value, [""]) diff --git a/lambdas/shared/tests/test_common/validator/test_validator.py b/lambdas/shared/tests/test_common/validator/test_validator.py index a29c1b4fe2..a32cbaea04 100644 --- a/lambdas/shared/tests/test_common/validator/test_validator.py +++ b/lambdas/shared/tests/test_common/validator/test_validator.py @@ -10,11 +10,11 @@ class TestValidator(unittest.TestCase): def setUp(self): - self.mock_expression_checker = patch('common.validator.validator.ExpressionChecker').start() - self.mock_schema_parser = patch('common.validator.validator.SchemaParser').start() - self.mock_fhir_parser = patch('common.validator.validator.FHIRParser').start() - self.mock_csv_parser = patch('common.validator.validator.CSVParser').start() - self.mock_csv_line_parser = patch('common.validator.validator.CSVLineParser').start() + self.mock_expression_checker = patch("common.validator.validator.ExpressionChecker").start() + self.mock_schema_parser = patch("common.validator.validator.SchemaParser").start() + self.mock_fhir_parser = patch("common.validator.validator.FHIRParser").start() + self.mock_csv_parser = patch("common.validator.validator.CSVParser").start() + self.mock_csv_line_parser = patch("common.validator.validator.CSVLineParser").start() def tearDown(self): patch.stopall() @@ -24,14 +24,18 @@ def test_run_validation_csv(self): self.mock_csv_parser.parse_csv_file.return_value = None self.mock_schema_parser.parse_schema.return_value = None self.mock_schema_parser.getExpressions.return_value = [ - {'fieldNameCSV': 'test_field', 'expressionId': 'exp1', 'errorLevel': 1, - 'expression': {'expressionName': 'Test', 'expressionType': 'type', 'expressionRule': 'rule'}, - 'errorGroup': 'group'} + { + "fieldNameCSV": "test_field", + "expressionId": "exp1", + "errorLevel": 1, + "expression": {"expressionName": "Test", "expressionType": "type", "expressionRule": "rule"}, + "errorGroup": "group", + } ] self.mock_expression_checker.return_value.validate_expression.return_value = None validator = Validator() - result = validator.validate_csv('file.csv') + result = validator.validate_csv("file.csv") self.assertIsInstance(result, list) self.assertEqual(len(result), 0) @@ -39,20 +43,20 @@ def test_run_validation_fhir_exception(self): # Simulate exception in FHIRParser self.mock_fhir_parser.side_effect = Exception("FHIR error") validator = Validator() - result = validator.validate_fhir('file.fhir') + result = validator.validate_fhir("file.fhir") self.assertIsInstance(result, list) self.assertEqual(result[0].code, 0) - self.assertIn('Data Parser Unexpected exception', result[0].message) + self.assertIn("Data Parser Unexpected exception", result[0].message) def test_run_validation_schema_exception(self): # Simulate exception in SchemaParser self.mock_fhir_parser.return_value.parse_fhir_file.return_value = None self.mock_schema_parser.side_effect = Exception("Schema error") validator = Validator() - result = validator.validate_fhir('file.fhir') + result = validator.validate_fhir("file.fhir") self.assertIsInstance(result, list) self.assertEqual(result[0].code, 0) - self.assertIn('Schema Parser Unexpected exception', result[0].message) + self.assertIn("Schema Parser Unexpected exception", result[0].message) def test_run_validation_expression_checker_exception(self): # Simulate exception in ExpressionChecker @@ -61,10 +65,10 @@ def test_run_validation_expression_checker_exception(self): self.mock_schema_parser.return_value.get_expressions.return_value = [] self.mock_expression_checker.side_effect = Exception("ExpressionChecker error") validator = Validator() - result = validator.validate_fhir('file.fhir') + result = validator.validate_fhir("file.fhir") self.assertIsInstance(result, list) self.assertEqual(result[0].code, 0) - self.assertIn('Expression Checker Unexpected exception', result[0].message) + self.assertIn("Expression Checker Unexpected exception", result[0].message) def test_run_validation_expression_getter_exception(self): # Simulate exception in getExpressions @@ -73,19 +77,19 @@ def test_run_validation_expression_getter_exception(self): self.mock_schema_parser.return_value.get_expressions.side_effect = Exception("Expressions error") self.mock_expression_checker.return_value = MagicMock() validator = Validator() - result = validator.validate_fhir('file.fhir') + result = validator.validate_fhir("file.fhir") self.assertIsInstance(result, list) self.assertEqual(result[0].code, 0) - self.assertIn('Expression Getter Unexpected exception', result[0].message) + self.assertIn("Expression Getter Unexpected exception", result[0].message) - @patch('common.validator.validator.DQReporter') + @patch("common.validator.validator.DQReporter") def test_build_error_report(self, mock_dq_reporter): - mock_dq_reporter.return_value.generate_error_report.return_value = {'report': 'ok'} - validator = Validator(data_type=DataType.CSV, filepath='file.csv') + mock_dq_reporter.return_value.generate_error_report.return_value = {"report": "ok"} + validator = Validator(data_type=DataType.CSV, filepath="file.csv") validator.data_parser = MagicMock() - validator.data_parser.get_key_single_value.return_value = '2023-01-01' - result = validator.build_error_report('event123') - self.assertEqual(result, {'report': 'ok'}) + validator.data_parser.get_key_single_value.return_value = "2023-01-01" + result = validator.build_error_report("event123") + self.assertEqual(result, {"report": "ok"}) mock_dq_reporter.return_value.generate_error_report.assert_called_once() def test_has_validation_failed_true(self):