fix suername validation

Author: fimran08

tests/e2e_automation/features/batchTests/Steps/test_update_batch_steps.py

@@ -3,7 +3,11 @@
 import pandas as pd
 from pytest_bdd import given, scenarios, then, when
 from src.objectModels.batch.batch_file_builder import build_batch_file
-from utilities.batch_file_helper import read_and_validate_csv_bus_ack_file_content
+from utilities.batch_file_helper import (
+    read_and_validate_csv_bus_ack_file_content,
+    validate_json_bus_ack_file_failure_records,
+    validate_json_bus_ack_file_structure_and_metadata,
+)
 from utilities.enums import GenderCode
 from utilities.error_constants import ERROR_MAP
 
@@ -27,13 +31,14 @@
 from .batch_common_steps import (
     build_dataFrame_using_datatable,
     create_batch_file,
-    json_bus_ack_will_only_contain_file_metadata_and_correct_failure_record_entries,
 )
 
 scenarios("batchTests/update_batch.feature")
 
 
-@given("batch file is created for below data as full dataset and each record has a valid update record in the same file")
+@given(
+    "batch file is created for below data as full dataset and each record has a valid update record in the same file"
+)
 def valid_batch_file_is_created_with_details(datatable, context):
     build_dataFrame_using_datatable(datatable, context)
     df_new = context.vaccine_df.copy()
@@ -55,7 +60,9 @@ def create_valid_vaccination_record_through_api(context):
 )
 def create_valid_vaccination_record_with_same_unique_id_as_batch_file(context):
     valid_json_payload_is_created(context)
-    context.immunization_object.identifier[0].value = f"Fail-duplicate{str(uuid.uuid4())}-duplicate"
+    context.immunization_object.identifier[0].value = (
+        f"Fail-duplicate{str(uuid.uuid4())}-duplicate"
+    )
     Trigger_the_post_create_request(context)
     The_request_will_have_status_code(context, 201)
     validateCreateLocation(context)
@@ -94,14 +101,20 @@ def upload_batch_file_to_s3_for_update(context):
     create_batch_file(context)
 
 
-@then("The delta and imms event table will be populated with the correct data for api created event")
-@given("The delta and imms event table will be populated with the correct data for api created event")
+@then(
+    "The delta and imms event table will be populated with the correct data for api created event"
+)
+@given(
+    "The delta and imms event table will be populated with the correct data for api created event"
+)
 def validate_imms_delta_table_for_api_created_event(context):
     validate_imms_event_table_by_operation(context, "created")
     validate_imms_delta_table_by_ImmsID(context)
 
 
-@when("Send a update for Immunization event created with vaccination detail being updated through API request")
+@when(
+    "Send a update for Immunization event created with vaccination detail being updated through API request"
+)
 def send_update_for_immunization_event_with_vaccination_detail_updated(context):
     valid_json_payload_is_created(context)
     row = context.vaccine_df.loc[0]
@@ -110,11 +123,15 @@ def send_update_for_immunization_event_with_vaccination_detail_updated(context):
     context.immunization_object.contained[1].name[0].family = row["PERSON_SURNAME"]
     reverse_gender_map = {v.value: v.name for v in GenderCode}
     code = row["PERSON_GENDER_CODE"]
-    context.immunization_object.contained[1].gender = reverse_gender_map.get(code, "unknown")
-    context.immunization_object.contained[
-        1
-    ].birthDate = f"{row['PERSON_DOB'][:4]}-{row['PERSON_DOB'][4:6]}-{row['PERSON_DOB'][6:]}"
-    context.immunization_object.contained[1].address[0].postalCode = row["PERSON_POSTCODE"]
+    context.immunization_object.contained[1].gender = reverse_gender_map.get(
+        code, "unknown"
+    )
+    context.immunization_object.contained[1].birthDate = (
+        f"{row['PERSON_DOB'][:4]}-{row['PERSON_DOB'][4:6]}-{row['PERSON_DOB'][6:]}"
+    )
+    context.immunization_object.contained[1].address[0].postalCode = row[
+        "PERSON_POSTCODE"
+    ]
     context.immunization_object.identifier[0].value = row["UNIQUE_ID"]
     context.immunization_object.identifier[0].system = row["UNIQUE_ID_URI"]
     send_update_for_immunization_event(context)
@@ -128,7 +145,9 @@ def api_request_will_be_successful_and_tables_will_be_updated_correctly(context)
     validate_delta_table_for_updated_event(context)
 
 
-@when("Update to above vaccination record is made through batch file upload with mandatory field missing")
+@when(
+    "Update to above vaccination record is made through batch file upload with mandatory field missing"
+)
 def upload_batch_file_to_s3_for_update_with_mandatory_field_missing(context):
     # Build base record
     record = build_batch_file(context)
@@ -145,7 +164,9 @@ def upload_batch_file_to_s3_for_update_with_mandatory_field_missing(context):
         "UNIQUE_ID_URI": context.create_object.identifier[0].system,
     }
     context.vaccine_df.loc[0, list(base_fields.keys())] = list(base_fields.values())
-    context.vaccine_df = pd.concat([context.vaccine_df.loc[[0]]] * 19, ignore_index=True)
+    context.vaccine_df = pd.concat(
+        [context.vaccine_df.loc[[0]]] * 19, ignore_index=True
+    )
     missing_cases = {
         0: {"SITE_CODE": "", "PERSON_SURNAME": "empty_site_code"},
         1: {"SITE_CODE_TYPE_URI": "", "PERSON_SURNAME": "empty_site_code_uri"},
@@ -177,16 +198,26 @@ def upload_batch_file_to_s3_for_update_with_mandatory_field_missing(context):
     create_batch_file(context)
 
 
-@then("csv bus ack will have error records for all the updated records in the batch file")
+@then(
+    "csv bus ack will have error records for all the updated records in the batch file"
+)
 def all_records_are_processed_successfully_in_the_batch_file(context):
     file_rows = read_and_validate_csv_bus_ack_file_content(context, False, True)
     all_valid = validate_bus_ack_file_for_error_by_surname(context, file_rows)
     assert all_valid, "One or more records failed validation checks"
 
 
-@then("json bus ack will have error records for all the updated records in the batch file")
+@then(
+    "json bus ack will have error records for all the updated records in the batch file"
+)
 def json_bus_ack_will_have_error_records_for_all_updated_records_in_batch_file(context):
-    json_bus_ack_will_only_contain_file_metadata_and_correct_failure_record_entries(context)
+    json_content = context.fileContentJson
+    assert json_content is not None, "BUS Ack JSON content is None"
+    validate_json_bus_ack_file_structure_and_metadata(context)
+    success = validate_json_bus_ack_file_failure_records(
+        context, expected_failure=True, use_username_for_error_lookup=True
+    )
+    assert success, "Failed to validate JSON bus ack file failure records"
 
 
 def validate_bus_ack_file_for_error_by_surname(context, file_rows) -> bool:
@@ -198,7 +229,9 @@ def validate_bus_ack_file_for_error_by_surname(context, file_rows) -> bool:
         bus_ack_row_number = batch_idx + 2
         row_data_list = file_rows.get(bus_ack_row_number)
         if not row_data_list:
-            print(f"Batch row {batch_idx}: No BUS ACK entry found for row number {bus_ack_row_number}")
+            print(
+                f"Batch row {batch_idx}: No BUS ACK entry found for row number {bus_ack_row_number}"
+            )
             overall_valid = False
             continue
         surname = str(row.get("PERSON_SURNAME", "")).strip()

tests/e2e_automation/utilities/batch_file_helper.py

@@ -189,7 +189,6 @@ def validate_bus_ack_file_for_error(context, file_rows) -> bool:
 
 
 def read_and_validate_csv_bus_ack_file_content(context, by_local_id: bool = True, by_row_number: bool = False) -> dict:
-    # Prevent invalid combinations
     if by_local_id and by_row_number:
         raise ValueError("Choose only one mode: by_local_id OR by_row_number")
 
@@ -291,78 +290,89 @@ def validate_json_bus_ack_file_structure_and_metadata(context):
     )
 
 
-def validate_json_bus_ack_file_failure_records(context, expected_failure: bool = True):
+def validate_json_bus_ack_file_failure_records(
+    context, expected_failure: bool = True, use_username_for_error_lookup: bool = False
+):
     data = json.loads(context.fileContentJson)
     report = BatchReport(**data)
     failures = report.failures or []
+
     if not expected_failure:
-        if not failures or len(failures) == 0:
+        if not failures:
             return True
-        else:
-            print(f"Found {len(failures)} failure records in BUS ACK file as not expected")
-            return False
-    else:
-        fail_mask = context.vaccine_df["UNIQUE_ID"].str.startswith("Fail-", na=False) | (
-            context.vaccine_df["UNIQUE_ID"].str.strip() == ""
-        )
-        fail_df = context.vaccine_df[fail_mask]
+        print(f"Found {len(failures)} failure records in BUS ACK file as not expected")
+        return False
 
-        # Build expected localId values
-        expected_local_ids = set(fail_df["UNIQUE_ID"].astype(str) + "^" + fail_df["UNIQUE_ID_URI"].astype(str))
+    fail_mask = context.vaccine_df["UNIQUE_ID"].str.startswith("Fail-", na=False) | (
+        context.vaccine_df["UNIQUE_ID"].str.strip() == ""
+    )
+    fail_df = context.vaccine_df[fail_mask]
 
-        overall_valid = True
+    expected_local_ids = set(fail_df["UNIQUE_ID"].astype(str) + "^" + fail_df["UNIQUE_ID_URI"].astype(str))
 
-        for failure in failures:
-            row_valid = True
+    overall_valid = True
 
-            row_id = failure.rowId
-            response_code = failure.responseCode
-            response_display = failure.responseDisplay
-            severity = failure.severity
-            local_id = failure.localId
-            operation_outcome = failure.operationOutcome
+    for failure in failures:
+        row_valid = True
 
-            # --- Validate localId exists ---
-            if local_id not in expected_local_ids:
-                print(f"Failure rowId {row_id}: localId '{local_id}' not expected")
-                row_valid = False
+        row_id = failure.rowId
+        local_id = failure.localId
+        operation_outcome = failure.operationOutcome
 
-            # --- Validate fixed fields ---
-            if response_code != "30002":
-                print(f"Failure rowId {row_id}: responseCode != '30002'")
-                row_valid = False
+        if local_id not in expected_local_ids:
+            print(f"Failure rowId {row_id}: localId '{local_id}' not expected")
+            row_valid = False
 
-            if response_display != "Business Level Response Value - Processing Error":
-                print(f"Failure rowId {row_id}: responseDisplay incorrect")
-                row_valid = False
+        if failure.responseCode != "30002":
+            print(f"Failure rowId {row_id}: responseCode != '30002'")
+            row_valid = False
 
-            if severity != "Fatal":
-                print(f"Failure rowId {row_id}: severity != 'Fatal'")
-                row_valid = False
+        if failure.responseDisplay != "Business Level Response Value - Processing Error":
+            print(f"Failure rowId {row_id}: responseDisplay incorrect")
+            row_valid = False
 
-            try:
-                df_row = context.vaccine_df.loc[row_id - 2]
-                prefix = str(df_row["UNIQUE_ID"]).strip()
+        if failure.severity != "Fatal":
+            print(f"Failure rowId {row_id}: severity != 'Fatal'")
+            row_valid = False
 
-                if prefix in ["", " ", "nan"]:
-                    expected_error = df_row["PERSON_SURNAME"]
-                else:
-                    parts = prefix.split("-")
-                    expected_error = parts[2] if len(parts) > 2 else "invalid_prefix_format"
+        try:
+            df_row = context.vaccine_df.loc[row_id - 2]
+            expected_error = get_expected_error(df_row, use_username_for_error_lookup)
 
-                expected_diagnostic = ERROR_MAP.get(expected_error, {}).get("diagnostics")
+            expected_diagnostic = ERROR_MAP.get(expected_error, {}).get("diagnostics")
 
-                if operation_outcome != expected_diagnostic:
-                    print(
-                        f"Failure rowId {row_id}: operationOutcome mismatch. "
-                        f"Expected '{expected_diagnostic}', got '{operation_outcome}'"
-                    )
-                    row_valid = False
+            # Duplicate case
+            if expected_error == "duplicate" and expected_diagnostic:
+                expected_diagnostic = expected_diagnostic.replace(
+                    "<identifier>",
+                    f"{context.immunization_object.identifier[0].system}#"
+                    f"{context.immunization_object.identifier[0].value}",
+                )
 
-            except Exception as e:
-                print(f"Failure rowId {row_id}: error resolving expected diagnostics: {e}")
+            if operation_outcome != expected_diagnostic:
+                print(
+                    f"Failure rowId {row_id}: operationOutcome mismatch. "
+                    f"Expected '{expected_diagnostic}', got '{operation_outcome}'"
+                )
                 row_valid = False
 
-            overall_valid = overall_valid and row_valid
+        except Exception as e:
+            print(f"Failure rowId {row_id}: error resolving expected diagnostics: {e}")
+            row_valid = False
+
+        overall_valid = overall_valid and row_valid
+
+    return overall_valid
+
+
+def get_expected_error(df_row, use_surname: bool):
+    prefix = str(df_row["UNIQUE_ID"]).strip()
+
+    if prefix in ["", " ", "nan"]:
+        return df_row.get("PERSON_SURNAME", "").strip()
+
+    if use_surname:
+        return str(df_row.get("PERSON_SURNAME", "")).strip()
 
-        return overall_valid
+    parts = prefix.split("-")
+    return parts[2] if len(parts) > 2 else "invalid_prefix_format"