VED-767 Non-ASCII Bug (#819)

* typo

* processing/{file_key}

* run standalone

* unit test

Author: JamesW1-NHS

recordprocessor/src/batch_processor.py

@@ -50,7 +50,7 @@ def process_csv_to_fhir(incoming_message_body: dict) -> int:
             logger.info(f"Encode error at row {row_count} with {encoder}. Switch to {new_encoder}")
             encoder = new_encoder
             # load alternative encoder
-            csv_reader = get_csv_content_dict_reader(file_key, encoder=encoder)
+            csv_reader = get_csv_content_dict_reader(f"processing/{file_key}", encoder=encoder)
             # re-read the file and skip processed rows
             row_count, err = process_rows(file_id, vaccine, supplier, file_key, allowed_operations,
                                           created_at_formatted_string, csv_reader, target_disease, row_count)

recordprocessor/tests/test_process_csv_to_fhir.py

@@ -4,11 +4,13 @@
 from unittest.mock import patch
 from copy import deepcopy
 import boto3
-from moto import mock_s3, mock_firehose
-from tests.utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import (
+from moto import mock_s3, mock_firehose, mock_dynamodb
+
+from tests.utils_for_recordprocessor_tests.generic_setup_and_teardown import (
     GenericSetUp,
     GenericTearDown,
 )
+from tests.utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import add_entry_to_table
 from tests.utils_for_recordprocessor_tests.values_for_recordprocessor_tests import (
     MockFileDetails,
     ValidMockFileContent,
@@ -19,20 +21,21 @@
 with patch("os.environ", MOCK_ENVIRONMENT_DICT):
     from batch_processor import process_csv_to_fhir
 
-
+dynamodb_client = boto3.client("dynamodb", region_name=REGION_NAME)
 s3_client = boto3.client("s3", region_name=REGION_NAME)
 firehose_client = boto3.client("firehose", region_name=REGION_NAME)
 test_file = MockFileDetails.rsv_emis
 
 
-@patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
 @mock_s3
 @mock_firehose
+@mock_dynamodb
+@patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
 class TestProcessCsvToFhir(unittest.TestCase):
     """Tests for process_csv_to_fhir function"""
 
     def setUp(self) -> None:
-        GenericSetUp(s3_client, firehose_client)
+        GenericSetUp(s3_client=s3_client, firehose_client=firehose_client, dynamodb_client=dynamodb_client)
 
         redis_patcher = patch("mappings.redis_client")
         self.addCleanup(redis_patcher.stop)
@@ -43,7 +46,7 @@ def setUp(self) -> None:
         }])
 
     def tearDown(self) -> None:
-        GenericTearDown(s3_client, firehose_client)
+        GenericTearDown(s3_client=s3_client, firehose_client=firehose_client, dynamodb_client=dynamodb_client)
 
     @staticmethod
     def upload_source_file(file_key, file_content):

recordprocessor/tests/test_recordprocessor_edge_cases.py

@@ -1,7 +1,7 @@
 import unittest
 import os
 from io import BytesIO
-from unittest.mock import patch
+from unittest.mock import call, patch
 from batch_processor import process_csv_to_fhir
 from tests.utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import create_patch
 
@@ -68,6 +68,7 @@ def test_process_large_file_cp1252(self):
         message_body = {
                     "vaccine_type": "vax-type-1",
                     "supplier": "test-supplier",
+                    "filename": "test-filename"
                 }
         self.mock_map_target_disease.return_value = "some disease"
 
@@ -78,6 +79,10 @@ def test_process_large_file_cp1252(self):
         self.mock_logger_warning.assert_called()
         warning_call_args = self.mock_logger_warning.call_args[0][0]
         self.assertTrue(warning_call_args.startswith("Encoding Error: 'utf-8' codec can't decode byte 0xe9"))
+        self.mock_s3_get_object.assert_has_calls([
+            call(Bucket=None, Key="test-filename"),
+            call(Bucket=None, Key="processing/test-filename"),
+        ])
 
     def test_process_large_file_utf8(self):
         """ Test processing a large file with utf-8 encoding """

recordprocessor/tests/utils_for_recordprocessor_tests/generic_setup_and_teardown.py

@@ -30,7 +30,6 @@ def __init__(self, s3_client=None, firehose_client=None, sqs_client=None, dynamo
             for bucket_name in [
                 BucketNames.SOURCE,
                 BucketNames.DESTINATION,
-                BucketNames.CONFIG,
                 BucketNames.MOCK_FIREHOSE,
             ]:
                 s3_client.create_bucket(
@@ -91,7 +90,6 @@ def __init__(self, s3_client=None, firehose_client=None, sqs_client=None, dynamo
             for bucket_name in [
                 BucketNames.SOURCE,
                 BucketNames.DESTINATION,
-                BucketNames.CONFIG,
                 BucketNames.MOCK_FIREHOSE,
             ]:
                 for obj in s3_client.list_objects_v2(Bucket=bucket_name).get("Contents", []):