Baseline lint after rebase

Author: dlzhry2nhs

lambdas/ack_backend/src/ack_processor.py

@@ -52,8 +52,9 @@ def lambda_handler(event, _):
     update_ack_file(file_key, created_at_formatted_string, ack_data_rows)
 
     if is_ack_processing_complete(message_id, total_ack_rows_processed):
-        complete_batch_file_process(message_id, supplier, vaccine_type, created_at_formatted_string, file_key,
-                                    total_ack_rows_processed)
+        complete_batch_file_process(
+            message_id, supplier, vaccine_type, created_at_formatted_string, file_key, total_ack_rows_processed
+        )
 
     return {
         "statusCode": 200,

lambdas/ack_backend/src/audit_table.py

@@ -1,4 +1,5 @@
 """Add the filename to the audit table and check for duplicates."""
+
 from typing import Optional
 from common.clients import dynamodb_client, logger
 from common.models.errors import UnhandledAuditTableError
@@ -33,8 +34,7 @@ def change_audit_table_status_to_processed(file_key: str, message_id: str) -> No
 def get_record_count_by_message_id(event_message_id: str) -> Optional[int]:
     """Retrieves full audit entry by unique event message ID"""
     audit_record = dynamodb_client.get_item(
-        TableName=AUDIT_TABLE_NAME,
-        Key={AuditTableKeys.MESSAGE_ID: {"S": event_message_id}}
+        TableName=AUDIT_TABLE_NAME, Key={AuditTableKeys.MESSAGE_ID: {"S": event_message_id}}
     )
 
     record_count = audit_record.get("Item", {}).get(AuditTableKeys.RECORD_COUNT, {}).get("N")

lambdas/ack_backend/src/update_ack_file.py

@@ -4,8 +4,15 @@
 from io import StringIO, BytesIO
 from audit_table import change_audit_table_status_to_processed
 from common.clients import get_s3_client, logger
-from constants import ACK_HEADERS, get_source_bucket_name, get_ack_bucket_name, COMPLETED_ACK_DIR, TEMP_ACK_DIR, \
-    BATCH_FILE_PROCESSING_DIR, BATCH_FILE_ARCHIVE_DIR
+from constants import (
+    ACK_HEADERS,
+    get_source_bucket_name,
+    get_ack_bucket_name,
+    COMPLETED_ACK_DIR,
+    TEMP_ACK_DIR,
+    BATCH_FILE_PROCESSING_DIR,
+    BATCH_FILE_ARCHIVE_DIR,
+)
 from logging_decorators import complete_batch_file_process_logging_decorator
 
 
@@ -51,21 +58,15 @@ def complete_batch_file_process(
     vaccine_type: str,
     created_at_formatted_string: str,
     file_key: str,
-    total_ack_rows_processed: int
+    total_ack_rows_processed: int,
 ) -> dict:
     """Mark the batch file as processed. This involves moving the ack and original file to destinations and updating
     the audit table status"""
     ack_filename = f"{file_key.replace('.csv', f'_BusAck_{created_at_formatted_string}.csv')}"
 
+    move_file(get_ack_bucket_name(), f"{TEMP_ACK_DIR}/{ack_filename}", f"{COMPLETED_ACK_DIR}/{ack_filename}")
     move_file(
-        get_ack_bucket_name(),
-        f"{TEMP_ACK_DIR}/{ack_filename}",
-        f"{COMPLETED_ACK_DIR}/{ack_filename}"
-    )
-    move_file(
-        get_source_bucket_name(),
-        f"{BATCH_FILE_PROCESSING_DIR}/{file_key}",
-        f"{BATCH_FILE_ARCHIVE_DIR}/{file_key}"
+        get_source_bucket_name(), f"{BATCH_FILE_PROCESSING_DIR}/{file_key}", f"{BATCH_FILE_ARCHIVE_DIR}/{file_key}"
     )
 
     change_audit_table_status_to_processed(file_key, message_id)
@@ -75,7 +76,7 @@ def complete_batch_file_process(
         "file_key": file_key,
         "supplier": supplier,
         "vaccine_type": vaccine_type,
-        "row_count": total_ack_rows_processed
+        "row_count": total_ack_rows_processed,
     }
 
 

lambdas/ack_backend/src/utils_for_ack_lambda.py

@@ -1,4 +1,5 @@
 """Utils for ack lambda"""
+
 from audit_table import get_record_count_by_message_id
 
 _BATCH_EVENT_ID_TO_RECORD_COUNT_MAP: dict[str, int] = {}

lambdas/ack_backend/tests/test_ack_processor.py

@@ -189,53 +189,22 @@ def test_lambda_handler_main(self):
             {
                 "description": "Multiple messages: all with diagnostics (failure messages)",
                 "messages": [
-                    {
-                        "row_id": "row^1",
-                        "diagnostics": DiagnosticsDictionaries.UNIQUE_ID_MISSING
-                    },
-                    {
-                        "row_id": "row^2",
-                        "diagnostics": DiagnosticsDictionaries.NO_PERMISSIONS
-                    },
-                    {
-                        "row_id": "row^3",
-                        "diagnostics": DiagnosticsDictionaries.RESOURCE_NOT_FOUND_ERROR
-                    },
+                    {"row_id": "row^1", "diagnostics": DiagnosticsDictionaries.UNIQUE_ID_MISSING},
+                    {"row_id": "row^2", "diagnostics": DiagnosticsDictionaries.NO_PERMISSIONS},
+                    {"row_id": "row^3", "diagnostics": DiagnosticsDictionaries.RESOURCE_NOT_FOUND_ERROR},
                 ],
             },
             {
                 "description": "Multiple messages: mixture of success and failure messages",
                 "messages": [
-                    {
-                        "row_id": "row^1",
-                        "imms_id": "TEST_IMMS_ID"
-                    },
-                    {
-                        "row_id": "row^2",
-                        "diagnostics": DiagnosticsDictionaries.UNIQUE_ID_MISSING
-                    },
-                    {
-                        "row_id": "row^3",
-                        "diagnostics": DiagnosticsDictionaries.CUSTOM_VALIDATION_ERROR
-                    },
-                    {
-                        "row_id": "row^4"
-                    },
-                    {
-                        "row_id": "row^5",
-                        "diagnostics": DiagnosticsDictionaries.CUSTOM_VALIDATION_ERROR
-                    },
-                    {
-                        "row_id": "row^6",
-                        "diagnostics": DiagnosticsDictionaries.CUSTOM_VALIDATION_ERROR
-                    },
-                    {
-                        "row_id": "row^7"
-                    },
-                    {
-                        "row_id": "row^8",
-                        "diagnostics": DiagnosticsDictionaries.IDENTIFIER_DUPLICATION_ERROR
-                    },
+                    {"row_id": "row^1", "imms_id": "TEST_IMMS_ID"},
+                    {"row_id": "row^2", "diagnostics": DiagnosticsDictionaries.UNIQUE_ID_MISSING},
+                    {"row_id": "row^3", "diagnostics": DiagnosticsDictionaries.CUSTOM_VALIDATION_ERROR},
+                    {"row_id": "row^4"},
+                    {"row_id": "row^5", "diagnostics": DiagnosticsDictionaries.CUSTOM_VALIDATION_ERROR},
+                    {"row_id": "row^6", "diagnostics": DiagnosticsDictionaries.CUSTOM_VALIDATION_ERROR},
+                    {"row_id": "row^7"},
+                    {"row_id": "row^8", "diagnostics": DiagnosticsDictionaries.IDENTIFIER_DUPLICATION_ERROR},
                 ],
             },
             {
@@ -244,12 +213,7 @@ def test_lambda_handler_main(self):
             },
             {
                 "description": "Single row: malformed diagnostics info from forwarder",
-                "messages": [
-                    {
-                        "row_id": "row^1",
-                        "diagnostics": "SHOULD BE A DICTIONARY, NOT A STRING"
-                    }
-                ],
+                "messages": [{"row_id": "row^1", "diagnostics": "SHOULD BE A DICTIONARY, NOT A STRING"}],
             },
         ]
 
@@ -278,60 +242,49 @@ def test_lambda_handler_updates_ack_file_but_does_not_mark_complete_when_records
         # Original source file had 100 records
         add_audit_entry_to_table(self.dynamodb_client, mock_batch_message_id, record_count=100)
         array_of_success_messages = [
-            {**BASE_SUCCESS_MESSAGE, "row_id": f"{mock_batch_message_id}^{i}", "imms_id": f"imms_{i}",
-             "local_id": f"local^{i}"}
+            {
+                **BASE_SUCCESS_MESSAGE,
+                "row_id": f"{mock_batch_message_id}^{i}",
+                "imms_id": f"imms_{i}",
+                "local_id": f"local^{i}",
+            }
             for i in range(1, 4)
         ]
-        test_event = {
-            "Records": [
-                {"body": json.dumps(array_of_success_messages)}
-            ]
-        }
+        test_event = {"Records": [{"body": json.dumps(array_of_success_messages)}]}
 
         response = lambda_handler(event=test_event, context={})
 
         self.assertEqual(response, EXPECTED_ACK_LAMBDA_RESPONSE_FOR_SUCCESS)
         validate_ack_file_content(
             self.s3_client,
-            [
-                *array_of_success_messages
-            ],
+            [*array_of_success_messages],
             existing_file_content=ValidValues.ack_headers,
         )
         self.assert_ack_and_source_file_locations_correct(
             MOCK_MESSAGE_DETAILS.file_key,
             MOCK_MESSAGE_DETAILS.temp_ack_file_key,
             MOCK_MESSAGE_DETAILS.archive_ack_file_key,
-            is_complete=False
+            is_complete=False,
         )
         self.assert_audit_entry_status_equals(mock_batch_message_id, "Preprocessed")
 
     @patch("utils_for_ack_lambda.get_record_count_by_message_id", return_value=500)
-    def test_lambda_handler_uses_message_id_to_record_count_cache_to_reduce_ddb_calls(
-        self,
-        mock_get_record_count: Mock
-    ):
+    def test_lambda_handler_uses_message_id_to_record_count_cache_to_reduce_ddb_calls(self, mock_get_record_count: Mock):
         """The DynamoDB Audit table is used to store the total record count for each source file. To reduce calls each
         time - this test checks that we cache the value as this lambda is called many times for large files"""
         mock_batch_message_id = "622cdeea-461e-4a83-acb5-7871d47ddbcd"
 
         # Original source file had 500 records
         add_audit_entry_to_table(self.dynamodb_client, mock_batch_message_id, record_count=500)
 
-        message_one = [{**BASE_SUCCESS_MESSAGE, "row_id": f"{mock_batch_message_id}^1", "imms_id": "imms_1",
-                        "local_id": "local^1"}]
-        message_two = [{**BASE_SUCCESS_MESSAGE, "row_id": f"{mock_batch_message_id}^2", "imms_id": "imms_2",
-                        "local_id": "local^2"}]
-        test_event_one = {
-            "Records": [
-                {"body": json.dumps(message_one)}
-            ]
-        }
-        test_event_two = {
-            "Records": [
-                {"body": json.dumps(message_two)}
-            ]
-        }
+        message_one = [
+            {**BASE_SUCCESS_MESSAGE, "row_id": f"{mock_batch_message_id}^1", "imms_id": "imms_1", "local_id": "local^1"}
+        ]
+        message_two = [
+            {**BASE_SUCCESS_MESSAGE, "row_id": f"{mock_batch_message_id}^2", "imms_id": "imms_2", "local_id": "local^2"}
+        ]
+        test_event_one = {"Records": [{"body": json.dumps(message_one)}]}
+        test_event_two = {"Records": [{"body": json.dumps(message_two)}]}
 
         response = lambda_handler(event=test_event_one, context={})
         self.assertEqual(response, EXPECTED_ACK_LAMBDA_RESPONSE_FOR_SUCCESS)
@@ -342,16 +295,14 @@ def test_lambda_handler_uses_message_id_to_record_count_cache_to_reduce_ddb_call
         mock_get_record_count.assert_called_once_with(mock_batch_message_id)
         validate_ack_file_content(
             self.s3_client,
-            [
-                *message_one, *message_two
-            ],
+            [*message_one, *message_two],
             existing_file_content=ValidValues.ack_headers,
         )
         self.assert_ack_and_source_file_locations_correct(
             MOCK_MESSAGE_DETAILS.file_key,
             MOCK_MESSAGE_DETAILS.temp_ack_file_key,
             MOCK_MESSAGE_DETAILS.archive_ack_file_key,
-            is_complete=False
+            is_complete=False,
         )
         self.assertEqual(_BATCH_EVENT_ID_TO_RECORD_COUNT_MAP[mock_batch_message_id], 500)
         self.assert_audit_entry_status_equals(mock_batch_message_id, "Preprocessed")
@@ -379,32 +330,27 @@ def test_lambda_handler_updates_ack_file_and_marks_complete_when_all_records_pro
         )
 
         array_of_success_messages = [
-            {**BASE_SUCCESS_MESSAGE, "row_id": f"{mock_batch_message_id}^{i}", "imms_id": f"imms_{i}",
-             "local_id": f"local^{i}"}
+            {
+                **BASE_SUCCESS_MESSAGE,
+                "row_id": f"{mock_batch_message_id}^{i}",
+                "imms_id": f"imms_{i}",
+                "local_id": f"local^{i}",
+            }
             for i in range(50, 101)
         ]
-        test_event = {
-            "Records": [
-                {"body": json.dumps(array_of_success_messages)}
-            ]
-        }
+        test_event = {"Records": [{"body": json.dumps(array_of_success_messages)}]}
 
         response = lambda_handler(event=test_event, context={})
 
         self.assertEqual(response, EXPECTED_ACK_LAMBDA_RESPONSE_FOR_SUCCESS)
         validate_ack_file_content(
-            self.s3_client,
-            [
-                *array_of_success_messages
-            ],
-            existing_file_content=existing_ack_content,
-            is_complete=True
+            self.s3_client, [*array_of_success_messages], existing_file_content=existing_ack_content, is_complete=True
         )
         self.assert_ack_and_source_file_locations_correct(
             MOCK_MESSAGE_DETAILS.file_key,
             MOCK_MESSAGE_DETAILS.temp_ack_file_key,
             MOCK_MESSAGE_DETAILS.archive_ack_file_key,
-            is_complete=True
+            is_complete=True,
         )
         self.assert_audit_entry_status_equals(mock_batch_message_id, "Processed")
 

lambdas/ack_backend/tests/test_audit_table.py

@@ -34,10 +34,7 @@ def test_get_record_count_by_message_id_returns_the_record_count(self):
         test_message_id = "1234"
 
         self.mock_dynamodb_client.get_item.return_value = {
-            "Item": {
-                "message_id": {"S": test_message_id},
-                "record_count": {"N": "1000"}
-            }
+            "Item": {"message_id": {"S": test_message_id}, "record_count": {"N": "1000"}}
         }
 
         self.assertEqual(audit_table.get_record_count_by_message_id(test_message_id), 1000)
@@ -46,10 +43,6 @@ def test_get_record_count_by_message_id_returns_none_if_record_count_not_set(sel
         """Test that if the record count has not yet been set on the audit item then None is returned"""
         test_message_id = "1234"
 
-        self.mock_dynamodb_client.get_item.return_value = {
-            "Item": {
-                "message_id": {"S": test_message_id}
-            }
-        }
+        self.mock_dynamodb_client.get_item.return_value = {"Item": {"message_id": {"S": test_message_id}}}
 
         self.assertIsNone(audit_table.get_record_count_by_message_id(test_message_id))

lambdas/ack_backend/tests/test_splunk_logging.py

@@ -379,17 +379,18 @@ def test_splunk_update_ack_file_not_logged(self):
             patch("common.log_decorator.send_log_to_firehose") as mock_send_log_to_firehose,  # noqa: E999
             patch("common.log_decorator.logger") as mock_logger,  # noqa: E999
             patch("ack_processor.is_ack_processing_complete", return_value=False),
-            patch("update_ack_file.change_audit_table_status_to_processed")
-                as mock_change_audit_table_status_to_processed,  # noqa: E999
+            patch(
+                "update_ack_file.change_audit_table_status_to_processed"
+            ) as mock_change_audit_table_status_to_processed,  # noqa: E999
         ):  # noqa: E999
             result = lambda_handler(generate_event(messages), context={})
 
         self.assertEqual(result, EXPECTED_ACK_LAMBDA_RESPONSE_FOR_SUCCESS)
 
         expected_secondlast_logger_info_data = {
-                **ValidValues.mock_message_expected_log_value,
-                "message_id": "test^98",
-            }
+            **ValidValues.mock_message_expected_log_value,
+            "message_id": "test^98",
+        }
         expected_last_logger_info_data = self.expected_lambda_handler_logs(success=True, number_of_rows=98)
 
         all_logger_info_call_args = self.extract_all_call_args_for_logger_info(mock_logger)
@@ -419,22 +420,23 @@ def test_splunk_update_ack_file_logged(self):
             patch("common.log_decorator.send_log_to_firehose") as mock_send_log_to_firehose,  # noqa: E999
             patch("common.log_decorator.logger") as mock_logger,  # noqa: E999
             patch("ack_processor.is_ack_processing_complete", return_value=True),
-            patch("update_ack_file.change_audit_table_status_to_processed")
-                as mock_change_audit_table_status_to_processed,  # noqa: E999
+            patch(
+                "update_ack_file.change_audit_table_status_to_processed"
+            ) as mock_change_audit_table_status_to_processed,  # noqa: E999
         ):  # noqa: E999
             result = lambda_handler(generate_event(messages), context={})
 
         self.assertEqual(result, EXPECTED_ACK_LAMBDA_RESPONSE_FOR_SUCCESS)
 
         expected_thirdlast_logger_info_data = {
-                **ValidValues.mock_message_expected_log_value,
-                "message_id": "test^99",
-            }
+            **ValidValues.mock_message_expected_log_value,
+            "message_id": "test^99",
+        }
         expected_secondlast_logger_info_data = {
-                **ValidValues.upload_ack_file_expected_log,
-                "message_id": "test",
-                "time_taken": "1.0s"
-            }
+            **ValidValues.upload_ack_file_expected_log,
+            "message_id": "test",
+            "time_taken": "1.0s",
+        }
         expected_last_logger_info_data = self.expected_lambda_handler_logs(
             success=True, number_of_rows=99, ingestion_complete=True
         )

lambdas/ack_backend/tests/test_update_ack_file.py

@@ -21,7 +21,7 @@
     generate_expected_ack_file_row,
     generate_sample_existing_ack_content,
     generate_expected_ack_content,
-    MOCK_MESSAGE_DETAILS
+    MOCK_MESSAGE_DETAILS,
 )
 
 from unittest.mock import patch