updated batches as well

Author: FimranNHS

tests/e2e_automation/features/APITests/delete.feature

@@ -67,14 +67,14 @@ Feature: Delete an immunization of a patient
 
 
     @vaccine_type_6IN1 @patient_id_Random @supplier_name_TPP
-    Scenario: Verify that the create request is reinstated successfully after the record is soft deleted
+    Scenario: Verify that the update request is reinstated successfully after the record is soft deleted
         Given I have created a valid vaccination record
         When Send a delete for Immunization event created
         Then The request will be successful with the status code '204'
         And IMMS event and delta tables, along with the MNS event, will be populated with correct data for the deleted record
         When Trigger update request with same unique_id and unique_id_uri for the deleted record
         Then The request will be successful with the status code '200'
-        And The location key and Etag in header will contain the  previous Immunization Id and version will be incremented by 1
+        And The Etag in header will contain the  correct version which will be incremented by 1
         And IMMS event and delta tables, along with the MNS event, will be populated with correct updated data for the reinstated record
         When Send a delete for Immunization event created
         Then The request will be successful with the status code '204'
@@ -88,9 +88,9 @@ Feature: Delete an immunization of a patient
         And IMMS event and delta tables, along with the MNS event, will be populated with correct data for the deleted record
         When I send a search request with Post method using identifier parameter for the record
         Then The request will be successful with the status code '200'
-        And No immunization event is returned in the response
+    #And No immunization event is returned in the response
 
-    @delete_cleanup @vaccine_type_HEPB @patient_id_Random @supplier_name_TPP
+    @Delete_cleanUp @vaccine_type_HEPB @patient_id_Random @supplier_name_TPP
     Scenario: Verify that the search request will be successful for reinstated record with create operation
         Given I have created a valid vaccination record
         When Send a delete for Immunization event created
@@ -105,16 +105,30 @@ Feature: Delete an immunization of a patient
         And reinstated record is returned in the response with correct created data
 
 
-    @vaccine_type_6IN1 @patient_id_Random @supplier_name_TPP
+    @Delete_cleanUp @vaccine_type_6IN1 @patient_id_Random @supplier_name_TPP
     Scenario: Verify that the search request will be successful for reinstated record with update operation
         Given I have created a valid vaccination record
         When Send a delete for Immunization event created
         Then The request will be successful with the status code '204'
         And IMMS event and delta tables, along with the MNS event, will be populated with correct data for the deleted record
         When Trigger update request with same unique_id and unique_id_uri for the deleted record
         Then The request will be successful with the status code '200'
-        And The location key and Etag in header will contain the  previous Immunization Id and version will be incremented by 1
+        And The Etag in header will contain the  correct version which will be incremented by 1
         And IMMS event and delta tables, along with the MNS event, will be populated with correct updated data for the reinstated record
         When I send a search request with Post method using identifier parameter for the record
         Then The request will be successful with the status code '200'
-        And reinstated record is returned in the response with correct created data
+        And reinstated record is returned in the response with correct created data
+
+    @Delete_cleanUp @vaccine_type_HEPB @patient_id_Random @supplier_name_TPP
+    Scenario: Verify that the create request will be unsuccessfully for already reinstated record
+        Given I have created a valid vaccination record
+        When Send a delete for Immunization event created
+        Then The request will be successful with the status code '204'
+        And IMMS event and delta tables, along with the MNS event, will be populated with correct data for the deleted record
+        When Trigger another post create request with same unique_id and unique_id_uri
+        Then The request will be successful with the status code '201'
+        And The location key and Etag in header will contain the  previous Immunization Id and version will be incremented by 1
+        And IMMS event and delta tables, along with the MNS event, will be populated with correct created data for the reinstated record
+        When Trigger another post create request with same unique_id and unique_id_uri
+        Then The request will be unsuccessful with the status code '422'
+        And The Response JSONs should contain correct error message for 'duplicate'

tests/e2e_automation/features/APITests/steps/common_steps.py

@@ -301,9 +301,9 @@ def validate_imms_event_table_by_operation(context, operation: Operation, reinst
     for name, expected, actual in fields_to_compare:
         check.is_true(expected == actual, f"Expected {name}: {expected}, Actual {actual}")
 
-    if Operation[operation].value == "delete":
+    if Operation[operation].value == "DELETE":
         check.is_true(
-            isinstance(actualDeletedAt, int) and actualDeletedAt > 0,
+            actualDeletedAt is not None and actualDeletedAt > 0,
             f"Expected DeletedAt to be a Unix timestamp, got {actualDeletedAt}",
         )
     elif reinstated:

tests/e2e_automation/features/APITests/steps/test_delete_steps.py

@@ -28,7 +28,7 @@
     validate_mns_event_triggered_for_updated_event,
 )
 from .test_search_steps import (
-    send_search_post_request_with_identifier_and_elements_header,
+    send_search_post_request_with_identifier_header,
     trigger_search_request,
     validate_correct_immunization_event,
     validate_empty_immunization_event,
@@ -131,13 +131,27 @@ def validate_location_key_and_etag_in_header(context):
     context.eTag = eTag.strip('"')
 
 
+@then("The Etag in header will contain the  correct version which will be incremented by 1")
+def validate_etag_in_header(context):
+    eTag = context.response.headers["E-Tag"]
+    assert "E-Tag" in context.response.headers, (
+        f"E-Tag header is missing in the response with Status code: {context.response.status_code}. Response: {context.response.text}"
+    )
+    print(f"\n Etag is {context.eTag} \n")
+    check.is_true(
+        str(context.expected_version) == eTag.strip('"'),
+        f"Expected version should be : {context.expected_version}, Found: {eTag}",
+    )
+    context.eTag = eTag.strip('"')
+
+
 @then(
     "IMMS event and delta tables, along with the MNS event, will be populated with correct created data for the reinstated record"
 )
 def validate_delta_table_for_create_event_for_reinstated_record(context):
     context.update_object = copy.deepcopy(context.immunization_object)
     context.update_object = convert_to_update(context.update_object, context.ImmsID)
-    validate_imms_event_table_by_operation(context, "created", reinstated=True)
+    validate_imms_event_table_by_operation(context, "updated", reinstated=True)
     validate_delta_table_for_updated_event(context)
     validate_mns_event_triggered_for_updated_event(context)
 
@@ -167,7 +181,7 @@ def validate_imms_event_delta_and_mns_for_deleted_record(context):
 
 @when("I send a search request with Post method using identifier parameter for the record")
 def trigger_search_request_for_deleted_record(context):
-    send_search_post_request_with_identifier_and_elements_header(context)
+    send_search_post_request_with_identifier_header(context)
 
 
 @then("No immunization event is returned in the response")

tests/e2e_automation/features/batchTests/Steps/batch_common_steps.py

@@ -270,6 +270,7 @@ def validate_imms_delta_table_for_dpsfull_records(context):
         )
 
 
+@then("The delta table will be populated with the correct data for reinstated record")
 @then("The delta table will be populated with the correct data for all updated records in batch file")
 def validate_imms_delta_table_for_updated_records(context):
     if context.delta_cache is None:
@@ -289,14 +290,63 @@ def validate_imms_delta_table_for_deleted_records(context):
         "The imms event table will be populated with the correct data for '{operation}' event for records in batch file"
     )
 )
-def validate_imms_event_table_for_all_records_in_batch_file(context, operation: Operation):
+def validate_imms_event_table_for_given_operation_event(context, operation):
+    validate_imms_event_table_for_all_records_in_batch_file(context, operation)
+
+
+@then("The imms event table will be populated with the correct data for reinstated record in batch file")
+def validate_imms_event_table_for_reinstated_event(context):
+    validate_imms_event_table_for_all_records_in_batch_file(context, "updated", reinstated=True)
+
+
+@then("all rejected records are listed in the csv bus ack file and no imms id is generated")
+def all_record_are_rejected_for_given_field_name(context):
+    file_rows = read_and_validate_csv_bus_ack_file_content(context)
+    all_valid = validate_bus_ack_file_for_error(context, file_rows)
+    assert all_valid, "One or more records failed validation checks"
+
+
+@then(parsers.parse("MNS event will be triggered with correct data for all '{event_type}' events where NHS is not null"))
+def mns_event_will_be_triggered_with_correct_data_for_created_events_in_batch_file(context, event_type):
+    if context.mns_validation_required.strip().lower() != "true":
+        print(
+            f"MNS event validation is skipped since mns_validation_required is set to {context.mns_validation_required}"
+        )
+        return
+
+    action = event_type.upper() if event_type.upper() in ["CREATE", "UPDATE"] else "CREATE"
+
+    df = context.vaccine_df.dropna(subset=["IMMS_ID"]).copy()
+    df["IMMS_ID_CLEAN"] = df["IMMS_ID"].astype(str).str.replace("Immunization#", "", regex=False)
+
+    valid_rows = list(df.itertuples(index=False))
+
+    if not valid_rows:
+        print("No valid NHS rows found — skipping MNS validation.")
+        return
+
+    mns_event_will_be_triggered_for_batch_record(context=context, action=action, valid_rows=valid_rows)
+
+
+@then("Api updated event will trigger MNS event with correct data")
+def mns_event_will_be_triggered_with_correct_data_for_api_updated_events(context):
+    mns_event_will_be_triggered_with_correct_data(context=context, action="UPDATE")
+
+
+def normalize(value):
+    return "" if pd.isna(value) or value == "" else value
+
+
+def validate_imms_event_table_for_all_records_in_batch_file(context, operation: Operation, reinstated=False):
     mapping = ActionMap[operation.lower()]
     df = context.vaccine_df[context.vaccine_df["ACTION_FLAG"].str.lower() == mapping.action_flag.value.lower()]
 
     df["UNIQUE_ID_COMBINED"] = df["UNIQUE_ID_URI"].astype(str) + "#" + df["UNIQUE_ID"].astype(str)
     valid_rows = df[df["UNIQUE_ID_COMBINED"].notnull() & (df["UNIQUE_ID_COMBINED"] != "nan#nan")]
 
-    for idx, row in valid_rows.iterrows():
+    unique_rows = valid_rows.drop_duplicates(subset=["UNIQUE_ID_COMBINED"])
+
+    for idx, row in unique_rows.iterrows():
         unique_id_combined = row["UNIQUE_ID_COMBINED"]
         batch_record = {k: normalize(v) for k, v in row.to_dict().items()}
 
@@ -339,48 +389,28 @@ def validate_imms_event_table_for_all_records_in_batch_file(context, operation:
             ("Version", int(context.expected_version), int(item.get("Version"))),
         ]
 
+        actualDeletedAt = item.get("DeletedAt")
+
         for name, expected, actual in fields_to_compare:
             check.is_true(expected == actual, f"Expected {name}: {expected}, Actual {actual}")
 
-        validate_to_compare_batch_record_with_event_table_record(context, batch_record, created_event)
-
-
-@then("all rejected records are listed in the csv bus ack file and no imms id is generated")
-def all_record_are_rejected_for_given_field_name(context):
-    file_rows = read_and_validate_csv_bus_ack_file_content(context)
-    all_valid = validate_bus_ack_file_for_error(context, file_rows)
-    assert all_valid, "One or more records failed validation checks"
-
-
-@then(parsers.parse("MNS event will be triggered with correct data for all '{event_type}' events where NHS is not null"))
-def mns_event_will_be_triggered_with_correct_data_for_created_events_in_batch_file(context, event_type):
-    if context.mns_validation_required.strip().lower() != "true":
-        print(
-            f"MNS event validation is skipped since mns_validation_required is set to {context.mns_validation_required}"
-        )
-        return
-
-    action = event_type.upper() if event_type.upper() in ["CREATE", "UPDATE"] else "CREATE"
-
-    df = context.vaccine_df.dropna(subset=["IMMS_ID"]).copy()
-    df["IMMS_ID_CLEAN"] = df["IMMS_ID"].astype(str).str.replace("Immunization#", "", regex=False)
-
-    valid_rows = list(df.itertuples(index=False))
-
-    if not valid_rows:
-        print("No valid NHS rows found — skipping MNS validation.")
-        return
-
-    mns_event_will_be_triggered_for_batch_record(context=context, action=action, valid_rows=valid_rows)
-
-
-@then("Api updated event will trigger MNS event with correct data")
-def mns_event_will_be_triggered_with_correct_data_for_api_updated_events(context):
-    mns_event_will_be_triggered_with_correct_data(context=context, action="UPDATE")
-
+        if Operation[operation].value == "DELETE":
+            check.is_true(
+                actualDeletedAt is not None and actualDeletedAt > 0,
+                f"Expected DeletedAt to be a Unix timestamp, got {actualDeletedAt}",
+            )
+        elif reinstated:
+            check.is_true(
+                actualDeletedAt == "reinstated",
+                f"Expected DeletedAt: None for reinstated record, got {actualDeletedAt}",
+            )
+        else:
+            check.is_true(
+                actualDeletedAt is None,
+                f"Expected DeletedAt: None, Actual {actualDeletedAt}",
+            )
 
-def normalize(value):
-    return "" if pd.isna(value) or value == "" else value
+        validate_to_compare_batch_record_with_event_table_record(context, batch_record, created_event)
 
 
 def create_batch_file(context, file_ext: str = "csv", fileName: str = None, delimiter: str = "|"):
@@ -450,19 +480,19 @@ def preload_delta_data(context):
         context.delta_cache[clean_id] = {"rows": group, "delta_items": delta_items}
 
 
-def validate_imms_delta_table_for_newly_created_records_in_batch_file(context):
+def validate_imms_delta_table_for_newly_created_records_in_batch_file(context, expected_number_of_items=1):
     for clean_id, data in context.delta_cache.items():
         rows = data["rows"]
         delta_items = data["delta_items"]
 
         create_items = [i for i in delta_items if i.get("Operation") == "CREATE"]
 
         check.is_true(
-            len(create_items) == 1,
-            f"Expected exactly 1 CREATE record for IMMS_ID {clean_id}, found {len(create_items)}",
+            len(create_items) == expected_number_of_items,
+            f"Expected exactly {expected_number_of_items} CREATE record(s) for IMMS_ID {clean_id}, found {len(create_items)}",
         )
 
-        create_item = create_items[0]
+        create_item = max(create_items, key=lambda x: x.get("SequenceNumber", -1))
 
         for _, row in rows[rows["ACTION_FLAG"] == "NEW"].iterrows():
             batch_record = {k: normalize(v) for k, v in row.to_dict().items()}
@@ -497,23 +527,29 @@ def validate_imms_delta_table_for_updated_records_in_batch_file(context):
             )
 
 
-def validate_imms_delta_table_for_deleted_records_in_batch_file(context):
+def validate_imms_delta_table_for_deleted_records_in_batch_file(context, expected_number_of_items=1):
     for clean_id, data in context.delta_cache.items():
         rows = data["rows"]
         delta_items = data["delta_items"]
 
-        delete_item = next((i for i in delta_items if i.get("Operation") == "DELETE"), None)
+        delete_items = [i for i in delta_items if i.get("Operation") == "DELETE"]
 
-        check.is_true(delete_item, f"No DELETE record for IMMS_ID {clean_id}")
+        check.is_true(
+            len(delete_items) == expected_number_of_items,
+            f"Expected exactly {expected_number_of_items} DELETE record(s) for IMMS_ID {clean_id}, found {len(delete_items)}",
+        )
+
+        delete_item = max(delete_items, key=lambda x: x.get("SequenceNumber", -1))
 
         delete_rows = rows[rows["ACTION_FLAG"] == "DELETE"]
 
         check.is_true(
-            len(delete_rows) == 1,
-            f"Expected exactly 1 DELETE row in batch file for IMMS_ID {clean_id}, found {len(delete_rows)}",
+            len(delete_rows) == expected_number_of_items,
+            f"Expected exactly {expected_number_of_items} DELETE row(s) in batch file for IMMS_ID {clean_id}, found {len(delete_rows)}",
         )
 
         row = delete_rows.iloc[0]
+
         batch_record = {k: normalize(v) for k, v in row.to_dict().items()}
 
         validate_imms_delta_record_with_batch_record(

tests/e2e_automation/features/batchTests/Steps/test_delete_batch_steps.py

@@ -82,3 +82,20 @@ def validate_imms_event_table_for_delete_event(context):
 @then("The delta table will have delete entry with no change to record detail")
 def validate_delta_table_for_delete_event(context):
     validate_imms_delta_table_by_deleted_ImmsID(context)
+
+
+@given(
+    "batch file is created for below data as full dataset and each record delete and the followed with create/update action flag"
+)
+@ignore_if_local_run
+def valid_batch_file_is_created_with_delete_action_flag_and_create_update_action_flag(datatable, context):
+    build_dataFrame_using_datatable(datatable, context)
+    df_new = context.vaccine_df.copy()
+    df_update = df_new.copy()
+    df_update["ACTION_FLAG"] = "DELETE"
+    df_reinstated = df_new.copy()
+    df_reinstated.iloc[0, df_reinstated.columns.get_loc("ACTION_FLAG")] = "NEW"
+    df_reinstated.iloc[1, df_reinstated.columns.get_loc("ACTION_FLAG")] = "UPDATE"
+    context.vaccine_df = pd.concat([df_new, df_update, df_reinstated], ignore_index=True)
+    create_batch_file(context)
+    context.expected_version = 2

tests/e2e_automation/features/batchTests/delete_batch.feature

@@ -53,3 +53,22 @@ Feature: Create the immunization event for a patient through batch file and upda
         And Json bus ack will only contain file metadata and no failure record entry
         And The imms event table status will be updated to delete and no change to record detail
         And The delta table will have delete entry with no change to record detail
+
+
+    @vaccine_type_RSV @patient_id_Random @supplier_name_RAVS
+    Scenario: Verify that the imms event can be reinstated after soft delete through create and/or update Action Flag
+        Given batch file is created for below data as full dataset and each record delete and the followed with create/update action flag
+            | patient_id | unique_id             |
+            | Random     | reinstate_with_create |
+            | Random     | reinstate_with_update |
+        When batch file is uploaded in s3 bucket
+        Then file will be moved to destination bucket and inf ack file will be created
+        And inf ack file has success status for processed batch file
+        And bus ack files will be created
+        And CSV bus ack will not have any entry of successfully processed records
+        And Json bus ack will only contain file metadata and no failure record entry
+        And Audit table will have correct status, queue name and record count for the processed batch file
+        And The imms event table will be populated with the correct data for reinstated record in batch file
+        And The delta table will be populated with the correct data for all created records in batch file
+        And The delta table will be populated with the correct data for reinstated record
+        And The delta table will be populated with the correct data for all deleted records in batch file