diff --git a/lambdas/filenameprocessor/poetry.lock b/lambdas/filenameprocessor/poetry.lock index 6b47fcee7a..aa2c2d57ff 100644 --- a/lambdas/filenameprocessor/poetry.lock +++ b/lambdas/filenameprocessor/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "async-timeout" @@ -1204,47 +1204,48 @@ files = [ [[package]] name = "moto" -version = "4.2.14" -description = "" +version = "5.1.22" +description = "A library that allows you to easily mock out tests based on AWS infrastructure" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "moto-4.2.14-py2.py3-none-any.whl", hash = "sha256:6d242dbbabe925bb385ddb6958449e5c827670b13b8e153ed63f91dbdb50372c"}, - {file = "moto-4.2.14.tar.gz", hash = "sha256:8f9263ca70b646f091edcc93e97cda864a542e6d16ed04066b1370ed217bd190"}, + {file = "moto-5.1.22-py3-none-any.whl", hash = "sha256:d9f20ae3cf29c44f93c1f8f06c8f48d5560e5dc027816ef1d0d2059741ffcfbe"}, + {file = "moto-5.1.22.tar.gz", hash = "sha256:e5b2c378296e4da50ce5a3c355a1743c8d6d396ea41122f5bb2a40f9b9a8cc0e"}, ] [package.dependencies] boto3 = ">=1.9.201" -botocore = ">=1.12.201" -cryptography = ">=3.3.1" +botocore = ">=1.20.88,<1.35.45 || >1.35.45,<1.35.46 || >1.35.46" +cryptography = ">=35.0.0" Jinja2 = ">=2.10.1" python-dateutil = ">=2.1,<3.0.0" requests = ">=2.5" -responses = ">=0.13.0" +responses = ">=0.15.0,<0.25.5 || >0.25.5" werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.5.0)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -apigatewayv2 = ["PyYAML (>=5.1)"] +all = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-sam-translator (<=1.103.0)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0,<=1.41.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsonpath_ng", "jsonschema", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.3)", "pydantic (<=2.12.4)", "pyparsing (>=3.0.7)", "setuptools"] +apigateway = ["PyYAML (>=5.1)", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)"] +apigatewayv2 = ["PyYAML (>=5.1)", "openapi-spec-validator (>=0.5.0)"] appsync = ["graphql-core"] awslambda = ["docker (>=3.0.0)"] batch = ["docker (>=3.0.0)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.0)"] -dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.0)"] -ec2 = ["sshpubkeys (>=3.1.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0,<=1.41.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.3)", "pyparsing (>=3.0.7)", "setuptools"] +cognitoidp = ["joserfc (>=0.9.0)"] +dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.6.3)"] +dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.6.3)"] +events = ["jsonpath_ng"] glue = ["pyparsing (>=3.0.7)"] -iotdata = ["jsondiff (>=1.1.2)"] -proxy = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.0)"] -s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.0)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +proxy = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-sam-translator (<=1.103.0)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0,<=1.41.0)", "docker (>=2.5.1)", "graphql-core", "joserfc (>=0.9.0)", "jsonpath_ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.3)", "pydantic (<=2.12.4)", "pyparsing (>=3.0.7)", "setuptools"] +quicksight = ["jsonschema"] +resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0,<=1.41.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.3)", "pyparsing (>=3.0.7)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.6.3)"] +s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.6.3)"] +server = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-sam-translator (<=1.103.0)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0,<=1.41.0)", "docker (>=3.0.0)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "joserfc (>=0.9.0)", "jsonpath_ng", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.3)", "pydantic (<=2.12.4)", "pyparsing (>=3.0.7)", "setuptools"] ssm = ["PyYAML (>=5.1)"] +stepfunctions = ["antlr4-python3-runtime", "jsonpath_ng"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] [[package]] @@ -1524,10 +1525,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.37.4,<2.0a.0" +botocore = ">=1.37.4,<2.0a0" [package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] [[package]] name = "simplejson" @@ -1781,4 +1782,4 @@ test = ["pytest", "pytest-cov"] [metadata] lock-version = "2.1" python-versions = "~3.11" -content-hash = "309c99af5e6acc3430ee31b5fe9c0a9d2c48ec3feefec37c37cb4eb490b4624b" +content-hash = "5cd10b1890a7760def9877a41c755017fb85bc4f398fc64a2db30191cd9b56d4" diff --git a/lambdas/filenameprocessor/pyproject.toml b/lambdas/filenameprocessor/pyproject.toml index 08ff17c5c9..b949df8bcf 100644 --- a/lambdas/filenameprocessor/pyproject.toml +++ b/lambdas/filenameprocessor/pyproject.toml @@ -15,7 +15,7 @@ python = "~3.11" boto3 = "~1.42.74" boto3-stubs-lite = { extras = ["dynamodb"], version = "~1.42.74" } aws-lambda-typing = "~2.20.0" -moto = "^4" +moto = "^5.1.22" requests = "~2.32.5" responses = "~0.26.0" pydantic = "~1.10.13" diff --git a/lambdas/filenameprocessor/tests/test_elasticache.py b/lambdas/filenameprocessor/tests/test_elasticache.py index 5cef68f2c9..1b63b12de6 100644 --- a/lambdas/filenameprocessor/tests/test_elasticache.py +++ b/lambdas/filenameprocessor/tests/test_elasticache.py @@ -4,29 +4,28 @@ from unittest import TestCase from unittest.mock import Mock, patch -from boto3 import client as boto3_client -from moto import mock_s3 +from moto import mock_aws from utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT from utils_for_tests.utils_for_filenameprocessor_tests import ( GenericSetUp, GenericTearDown, + create_boto3_clients, create_mock_hget, ) # Ensure environment variables are mocked before importing from src files with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT): - from common.clients import REGION_NAME from elasticache import ( get_supplier_permissions_from_cache, get_supplier_system_from_cache, get_valid_vaccine_types_from_cache, ) -s3_client = boto3_client("s3", region_name=REGION_NAME) +s3_client = None -@mock_s3 +@mock_aws @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT) @patch("elasticache.get_redis_client") class TestElasticache(TestCase): @@ -34,6 +33,8 @@ class TestElasticache(TestCase): def setUp(self): """Set up the S3 buckets""" + global s3_client + (s3_client,) = create_boto3_clients("s3") GenericSetUp(s3_client) def tearDown(self): diff --git a/lambdas/filenameprocessor/tests/test_lambda_handler.py b/lambdas/filenameprocessor/tests/test_lambda_handler.py index 8f19134905..20f187feb5 100644 --- a/lambdas/filenameprocessor/tests/test_lambda_handler.py +++ b/lambdas/filenameprocessor/tests/test_lambda_handler.py @@ -9,8 +9,7 @@ from unittest.mock import ANY, Mock, patch import fakeredis -from boto3 import client as boto3_client -from moto import mock_dynamodb, mock_firehose, mock_s3, mock_sqs +from moto import mock_aws from utils_for_tests.mock_environment_variables import ( MOCK_ENVIRONMENT_DICT, @@ -22,6 +21,7 @@ GenericSetUp, GenericTearDown, assert_audit_table_entry, + create_boto3_clients, create_mock_hget, ) from utils_for_tests.values_for_tests import ( @@ -34,15 +34,14 @@ # Ensure environment variables are mocked before importing from src files with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT): - from common.clients import REGION_NAME from common.models.batch_constants import AUDIT_TABLE_NAME, AuditTableKeys, FileStatus from constants import EXTENDED_ATTRIBUTES_VACC_TYPE from file_name_processor import handle_record, lambda_handler -s3_client = boto3_client("s3", region_name=REGION_NAME) -sqs_client = boto3_client("sqs", region_name=REGION_NAME) -firehose_client = boto3_client("firehose", region_name=REGION_NAME) -dynamodb_client = boto3_client("dynamodb", region_name=REGION_NAME) +s3_client = None +sqs_client = None +firehose_client = None +dynamodb_client = None # NOTE: The default throughout these tests is to use permissions config which allows all suppliers full permissions # for all vaccine types. This default is overridden for some specific tests. @@ -52,10 +51,7 @@ @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT) -@mock_s3 -@mock_sqs -@mock_firehose -@mock_dynamodb +@mock_aws class TestLambdaHandlerDataSource(TestCase): """Tests for lambda_handler when a data sources (vaccine data) file is received.""" @@ -87,6 +83,10 @@ def run(self, result=None): super().run(result) def setUp(self): + global s3_client, sqs_client, firehose_client, dynamodb_client + s3_client, sqs_client, firehose_client, dynamodb_client = create_boto3_clients( + "s3", "sqs", "firehose", "dynamodb" + ) GenericSetUp(s3_client, firehose_client, sqs_client, dynamodb_client) self.logger_patcher = patch("file_name_processor.logger") self.mock_logger = self.logger_patcher.start() @@ -483,7 +483,9 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi s3_client.put_object(Bucket=BucketNames.SOURCE, Key=bad_ext_key, Body=MOCK_EXTENDED_ATTRIBUTES_FILE_CONTENT) with patch("file_name_processor.uuid4", return_value="EA_bad_ext_id"): lambda_handler(self.make_event([self.make_record(bad_ext_key)]), None) - item = self.get_audit_table_items()[-1] + item = next( + item for item in self.get_audit_table_items() if item[AuditTableKeys.MESSAGE_ID]["S"] == "EA_bad_ext_id" + ) self.assertEqual(item[AuditTableKeys.STATUS]["S"], "Failed") s3_client.get_object(Bucket=BucketNames.SOURCE, Key=f"extended-attributes-archive/{bad_ext_key}") """ @@ -673,14 +675,15 @@ def test_lambda_adds_event_to_audit_table_as_failed_when_unexpected_exception_is @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT) -@mock_s3 -@mock_dynamodb -@mock_sqs -@mock_firehose +@mock_aws class TestUnexpectedBucket(TestCase): """Tests for lambda_handler when an unexpected bucket name is used""" def setUp(self): + global s3_client, sqs_client, firehose_client, dynamodb_client + s3_client, sqs_client, firehose_client, dynamodb_client = create_boto3_clients( + "s3", "sqs", "firehose", "dynamodb" + ) GenericSetUp(s3_client, firehose_client, sqs_client, dynamodb_client) def tearDown(self): diff --git a/lambdas/filenameprocessor/tests/test_send_sqs_message.py b/lambdas/filenameprocessor/tests/test_send_sqs_message.py index 3c2a442b1e..2812be16c1 100644 --- a/lambdas/filenameprocessor/tests/test_send_sqs_message.py +++ b/lambdas/filenameprocessor/tests/test_send_sqs_message.py @@ -5,34 +5,38 @@ from unittest import TestCase from unittest.mock import patch -from boto3 import client as boto3_client -from moto import mock_sqs +from moto import mock_aws from utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT, Sqs +from utils_for_tests.utils_for_filenameprocessor_tests import create_boto3_clients, reset_common_clients from utils_for_tests.values_for_tests import MockFileDetails # Ensure environment variables are mocked before importing from src files with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT): - from common.clients import REGION_NAME from models.errors import UnhandledSqsError from send_sqs_message import make_and_send_sqs_message, send_to_supplier_queue -sqs_client = boto3_client("sqs", region_name=REGION_NAME) +sqs_client = None FLU_EMIS_FILE_DETAILS = MockFileDetails.emis_flu RSV_RAVS_FILE_DETAILS = MockFileDetails.ravs_rsv_1 NON_EXISTENT_QUEUE_ERROR_MESSAGE = ( - "An unexpected error occurred whilst sending to SQS: An error occurred (AWS.SimpleQueueService.NonExistent" - + "Queue) when calling the SendMessage operation: The specified queue does not exist for this wsdl version." + "An unexpected error occurred whilst sending to SQS: An error occurred " + "(AWS.SimpleQueueService.NonExistentQueue) when calling the SendMessage operation" ) -@mock_sqs +@mock_aws @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT) class TestSendSQSMessage(TestCase): """Tests for send_sqs_message functions""" + def setUp(self): + global sqs_client + reset_common_clients() + (sqs_client,) = create_boto3_clients("sqs") + def test_send_to_supplier_queue_success(self): """Test send_to_supplier_queue function for a successful message send""" # Set up the sqs_queue @@ -75,7 +79,7 @@ def test_send_to_supplier_queue_failure_due_to_queue_does_not_exist(self): vaccine_type=FLU_EMIS_FILE_DETAILS.vaccine_type, supplier=FLU_EMIS_FILE_DETAILS.supplier, ) - self.assertEqual(NON_EXISTENT_QUEUE_ERROR_MESSAGE, str(context.exception)) + self.assertIn(NON_EXISTENT_QUEUE_ERROR_MESSAGE, str(context.exception)) def test_make_and_send_sqs_message_success(self): """Test make_and_send_sqs_message function for a successful message send""" diff --git a/lambdas/filenameprocessor/tests/test_utils_for_filenameprocessor.py b/lambdas/filenameprocessor/tests/test_utils_for_filenameprocessor.py index 23da1deedc..c648c2c358 100644 --- a/lambdas/filenameprocessor/tests/test_utils_for_filenameprocessor.py +++ b/lambdas/filenameprocessor/tests/test_utils_for_filenameprocessor.py @@ -4,8 +4,7 @@ from unittest import TestCase from unittest.mock import patch -from boto3 import client as boto3_client -from moto import mock_s3 +from moto import mock_aws from utils_for_tests.mock_environment_variables import ( MOCK_ENVIRONMENT_DICT, @@ -13,23 +12,25 @@ from utils_for_tests.utils_for_filenameprocessor_tests import ( GenericSetUp, GenericTearDown, + create_boto3_clients, ) # Ensure environment variables are mocked before importing from src files with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT): - from common.clients import REGION_NAME from constants import AUDIT_TABLE_TTL_DAYS from utils_for_filenameprocessor import get_creation_and_expiry_times -s3_client = boto3_client("s3", region_name=REGION_NAME) +s3_client = None -@mock_s3 +@mock_aws class TestUtilsForFilenameprocessor(TestCase): """Tests for utils_for_filenameprocessor functions""" def setUp(self): """Set up the s3 buckets""" + global s3_client + (s3_client,) = create_boto3_clients("s3") GenericSetUp(s3_client) def tearDown(self): diff --git a/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py b/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py index 8c16d525b5..e35a315740 100644 --- a/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py +++ b/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py @@ -35,6 +35,11 @@ class Sqs: # NOTE: FILE_NAME_GSI and CONFIG_BUCKET_NAME environment variables are set in the terraform, # but not used in the src code and so are not set here. MOCK_ENVIRONMENT_DICT = { + "AWS_ACCESS_KEY_ID": "testing", + "AWS_SECRET_ACCESS_KEY": "testing", + "AWS_SESSION_TOKEN": "testing", + "AWS_REGION": REGION_NAME, + "AWS_DEFAULT_REGION": REGION_NAME, "SOURCE_BUCKET_NAME": BucketNames.SOURCE, "ACK_BUCKET_NAME": BucketNames.DESTINATION, "DPS_BUCKET_NAME": BucketNames.DPS_DESTINATION, diff --git a/lambdas/filenameprocessor/tests/utils_for_tests/utils_for_filenameprocessor_tests.py b/lambdas/filenameprocessor/tests/utils_for_tests/utils_for_filenameprocessor_tests.py index 4be470faf9..48424a0873 100644 --- a/lambdas/filenameprocessor/tests/utils_for_tests/utils_for_filenameprocessor_tests.py +++ b/lambdas/filenameprocessor/tests/utils_for_tests/utils_for_filenameprocessor_tests.py @@ -14,22 +14,48 @@ # Ensure environment variables are mocked before importing from src files with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT): - from common.clients import REGION_NAME + import common.clients as common_clients from common.models.batch_constants import AUDIT_TABLE_NAME, AuditTableKeys from common.models.constants import RedisHashKeys from constants import ODS_CODE_TO_SUPPLIER_SYSTEM_HASH_KEY MOCK_ODS_CODE_TO_SUPPLIER = {"YGM41": "EMIS", "X8E5B": "RAVS"} +REGION_NAME = common_clients.REGION_NAME + +COMMON_CLIENT_CACHE_NAMES = ( + "global_s3_client", + "global_sqs_client", + "global_firehose_client", + "global_secrets_manager_client", + "global_dynamodb_client", + "global_dynamodb_resource", + "global_kinesis_client", +) + + +def reset_common_clients() -> None: + for client_cache_name in COMMON_CLIENT_CACHE_NAMES: + setattr(common_clients, client_cache_name, None) + -dynamodb_client = boto3_client("dynamodb", region_name=REGION_NAME) +def create_boto3_clients(*service_names: str): + return tuple(boto3_client(service_name, region_name=REGION_NAME) for service_name in service_names) + + +def get_dynamodb_client(): + return boto3_client("dynamodb", region_name=REGION_NAME) def assert_audit_table_entry(file_details: FileDetails, expected_status: str) -> None: """Assert that the file details are in the audit table""" - table_entry = dynamodb_client.get_item( - TableName=AUDIT_TABLE_NAME, - Key={AuditTableKeys.MESSAGE_ID: {"S": file_details.message_id}}, - ).get("Item") + table_entry = ( + get_dynamodb_client() + .get_item( + TableName=AUDIT_TABLE_NAME, + Key={AuditTableKeys.MESSAGE_ID: {"S": file_details.message_id}}, + ) + .get("Item") + ) assert table_entry == { **file_details.audit_table_entry, "status": {"S": expected_status}, @@ -67,6 +93,8 @@ def __init__( sqs_client=None, dynamodb_client=None, ): + reset_common_clients() + if s3_client: for bucket_name in [ BucketNames.SOURCE, @@ -133,3 +161,5 @@ def __init__( if dynamodb_client: dynamodb_client.delete_table(TableName=AUDIT_TABLE_NAME) + + reset_common_clients() diff --git a/lambdas/recordprocessor/poetry.lock b/lambdas/recordprocessor/poetry.lock index 4aeeabf467..0aefa51651 100644 --- a/lambdas/recordprocessor/poetry.lock +++ b/lambdas/recordprocessor/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "async-timeout" @@ -1153,47 +1153,48 @@ files = [ [[package]] name = "moto" -version = "4.2.14" -description = "" +version = "5.1.22" +description = "A library that allows you to easily mock out tests based on AWS infrastructure" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "moto-4.2.14-py2.py3-none-any.whl", hash = "sha256:6d242dbbabe925bb385ddb6958449e5c827670b13b8e153ed63f91dbdb50372c"}, - {file = "moto-4.2.14.tar.gz", hash = "sha256:8f9263ca70b646f091edcc93e97cda864a542e6d16ed04066b1370ed217bd190"}, + {file = "moto-5.1.22-py3-none-any.whl", hash = "sha256:d9f20ae3cf29c44f93c1f8f06c8f48d5560e5dc027816ef1d0d2059741ffcfbe"}, + {file = "moto-5.1.22.tar.gz", hash = "sha256:e5b2c378296e4da50ce5a3c355a1743c8d6d396ea41122f5bb2a40f9b9a8cc0e"}, ] [package.dependencies] boto3 = ">=1.9.201" -botocore = ">=1.12.201" -cryptography = ">=3.3.1" +botocore = ">=1.20.88,<1.35.45 || >1.35.45,<1.35.46 || >1.35.46" +cryptography = ">=35.0.0" Jinja2 = ">=2.10.1" python-dateutil = ">=2.1,<3.0.0" requests = ">=2.5" -responses = ">=0.13.0" +responses = ">=0.15.0,<0.25.5 || >0.25.5" werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.5.0)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -apigatewayv2 = ["PyYAML (>=5.1)"] +all = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-sam-translator (<=1.103.0)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0,<=1.41.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsonpath_ng", "jsonschema", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.3)", "pydantic (<=2.12.4)", "pyparsing (>=3.0.7)", "setuptools"] +apigateway = ["PyYAML (>=5.1)", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)"] +apigatewayv2 = ["PyYAML (>=5.1)", "openapi-spec-validator (>=0.5.0)"] appsync = ["graphql-core"] awslambda = ["docker (>=3.0.0)"] batch = ["docker (>=3.0.0)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.0)"] -dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.0)"] -ec2 = ["sshpubkeys (>=3.1.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0,<=1.41.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.3)", "pyparsing (>=3.0.7)", "setuptools"] +cognitoidp = ["joserfc (>=0.9.0)"] +dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.6.3)"] +dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.6.3)"] +events = ["jsonpath_ng"] glue = ["pyparsing (>=3.0.7)"] -iotdata = ["jsondiff (>=1.1.2)"] -proxy = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.0)"] -s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.0)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +proxy = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-sam-translator (<=1.103.0)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0,<=1.41.0)", "docker (>=2.5.1)", "graphql-core", "joserfc (>=0.9.0)", "jsonpath_ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.3)", "pydantic (<=2.12.4)", "pyparsing (>=3.0.7)", "setuptools"] +quicksight = ["jsonschema"] +resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0,<=1.41.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.3)", "pyparsing (>=3.0.7)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.6.3)"] +s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.6.3)"] +server = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-sam-translator (<=1.103.0)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0,<=1.41.0)", "docker (>=3.0.0)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "joserfc (>=0.9.0)", "jsonpath_ng", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.3)", "pydantic (<=2.12.4)", "pyparsing (>=3.0.7)", "setuptools"] ssm = ["PyYAML (>=5.1)"] +stepfunctions = ["antlr4-python3-runtime", "jsonpath_ng"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] [[package]] @@ -1453,10 +1454,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.37.4,<2.0a.0" +botocore = ">=1.37.4,<2.0a0" [package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] [[package]] name = "simplejson" @@ -1680,4 +1681,4 @@ test = ["pytest", "pytest-cov"] [metadata] lock-version = "2.1" python-versions = "~3.11" -content-hash = "b89d47790456715a030777efc6bf4cff2fd5b223e262e057a353e9bcbaeb1ac9" +content-hash = "0dc343827151a31620526e5271dfe3792493cbf77387e7e8c78af6f9b6ad166d" diff --git a/lambdas/recordprocessor/pyproject.toml b/lambdas/recordprocessor/pyproject.toml index 3f952b8c64..7321238d95 100644 --- a/lambdas/recordprocessor/pyproject.toml +++ b/lambdas/recordprocessor/pyproject.toml @@ -15,7 +15,7 @@ python = "~3.11" boto3 = "~1.42.74" boto3-stubs-lite = {extras = ["dynamodb"], version = "~1.42.74"} aws-lambda-typing = "~2.20.0" -moto = "^4" +moto = "^5.1.22" simplejson = "^3.20.2" coverage = "^7.13.5" redis = "^6.4.0" diff --git a/lambdas/recordprocessor/tests/test_file_level_validation.py b/lambdas/recordprocessor/tests/test_file_level_validation.py index ad25d2c7f5..410e47a08c 100644 --- a/lambdas/recordprocessor/tests/test_file_level_validation.py +++ b/lambdas/recordprocessor/tests/test_file_level_validation.py @@ -3,9 +3,6 @@ import unittest from unittest.mock import patch -# If mock_s3 is not imported here then tests in other files fail when running 'make test'. It is not clear why this is. -from moto import mock_s3 # noqa: F401 - from tests.utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import ( convert_string_to_dict_reader, ) diff --git a/lambdas/recordprocessor/tests/test_logging_decorator.py b/lambdas/recordprocessor/tests/test_logging_decorator.py index 2ae892e7c2..3f00c6486f 100644 --- a/lambdas/recordprocessor/tests/test_logging_decorator.py +++ b/lambdas/recordprocessor/tests/test_logging_decorator.py @@ -7,8 +7,7 @@ from datetime import datetime from unittest.mock import patch -from boto3 import client as boto3_client -from moto import mock_firehose, mock_s3 +from moto import mock_aws from utils_for_recordprocessor_tests.mock_environment_variables import ( MOCK_ENVIRONMENT_DICT, @@ -21,7 +20,6 @@ ) with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT): - from common.clients import REGION_NAME from file_level_validation import file_level_validation from models.errors import InvalidHeaders, NoOperationPermissions @@ -29,10 +27,11 @@ from utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import ( GenericSetUp, GenericTearDown, + create_boto3_clients, ) -s3_client = boto3_client("s3", region_name=REGION_NAME) -firehose_client = boto3_client("firehose", region_name=REGION_NAME) +s3_client = None +firehose_client = None MOCK_FILE_DETAILS = MockFileDetails.flu_emis COMMON_LOG_DATA = { "function_name": "record_processor_file_level_validation", @@ -45,14 +44,15 @@ } -@mock_s3 -@mock_firehose +@mock_aws @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT) class TestLoggingDecorator(unittest.TestCase): """Tests for the logging_decorator and its helper functions""" def setUp(self): """Set up the S3 buckets and upload the valid FLU/EMIS file example""" + global s3_client, firehose_client + s3_client, firehose_client = create_boto3_clients("s3", "firehose") GenericSetUp(s3_client, firehose_client) def tearDown(self): diff --git a/lambdas/recordprocessor/tests/test_process_csv_to_fhir.py b/lambdas/recordprocessor/tests/test_process_csv_to_fhir.py index faebf85a16..e6b4b69100 100644 --- a/lambdas/recordprocessor/tests/test_process_csv_to_fhir.py +++ b/lambdas/recordprocessor/tests/test_process_csv_to_fhir.py @@ -5,8 +5,7 @@ from copy import deepcopy from unittest.mock import Mock, call, patch -import boto3 -from moto import mock_dynamodb, mock_firehose, mock_s3 +from moto import mock_aws from utils_for_recordprocessor_tests.mock_environment_variables import ( MOCK_ENVIRONMENT_DICT, @@ -16,9 +15,9 @@ GenericSetUp, GenericTearDown, add_entry_to_table, + create_boto3_clients, ) from utils_for_recordprocessor_tests.values_for_recordprocessor_tests import ( - REGION_NAME, MockFileDetails, ValidMockFileContent, ) @@ -27,20 +26,20 @@ from batch_processor import process_csv_to_fhir from common.models.batch_constants import AUDIT_TABLE_NAME, FileStatus -dynamodb_client = boto3.client("dynamodb", region_name=REGION_NAME) -s3_client = boto3.client("s3", region_name=REGION_NAME) -firehose_client = boto3.client("firehose", region_name=REGION_NAME) +dynamodb_client = None +s3_client = None +firehose_client = None test_file = MockFileDetails.rsv_emis -@mock_s3 -@mock_firehose -@mock_dynamodb +@mock_aws @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT) class TestProcessCsvToFhir(unittest.TestCase): """Tests for process_csv_to_fhir function""" def setUp(self) -> None: + global dynamodb_client, s3_client, firehose_client + dynamodb_client, s3_client, firehose_client = create_boto3_clients("dynamodb", "s3", "firehose") GenericSetUp( s3_client=s3_client, firehose_client=firehose_client, diff --git a/lambdas/recordprocessor/tests/test_process_row.py b/lambdas/recordprocessor/tests/test_process_row.py index 979c00bba8..dfbf62dbb0 100644 --- a/lambdas/recordprocessor/tests/test_process_row.py +++ b/lambdas/recordprocessor/tests/test_process_row.py @@ -5,8 +5,7 @@ from decimal import Decimal from unittest.mock import patch -from boto3 import client as boto3_client -from moto import mock_s3 +from moto import mock_aws from tests.utils_for_recordprocessor_tests.mock_environment_variables import ( MOCK_ENVIRONMENT_DICT, @@ -14,6 +13,7 @@ from tests.utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import ( GenericSetUp, GenericTearDown, + create_boto3_clients, ) from tests.utils_for_recordprocessor_tests.values_for_recordprocessor_tests import ( MockFieldDictionaries, @@ -21,10 +21,9 @@ ) with patch("os.environ", MOCK_ENVIRONMENT_DICT): - from common.clients import REGION_NAME from process_row import process_row -s3_client = boto3_client("s3", region_name=REGION_NAME) +s3_client = None ROW_DETAILS = MockFieldDictionaries.all_fields Allowed_Operations = {"CREATE", "UPDATE", "DELETE"} expected_successful_result = { @@ -144,12 +143,14 @@ } -@mock_s3 +@mock_aws @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT) class TestProcessRow(unittest.TestCase): """Tests for process_row""" def setUp(self) -> None: + global s3_client + (s3_client,) = create_boto3_clients("s3") GenericSetUp(s3_client) def tearDown(self) -> None: diff --git a/lambdas/recordprocessor/tests/test_recordprocessor_main.py b/lambdas/recordprocessor/tests/test_recordprocessor_main.py index 0443dc8b3b..ca257757b9 100644 --- a/lambdas/recordprocessor/tests/test_recordprocessor_main.py +++ b/lambdas/recordprocessor/tests/test_recordprocessor_main.py @@ -7,8 +7,7 @@ from json import JSONDecodeError from unittest.mock import Mock, patch -from boto3 import client as boto3_client -from moto import mock_dynamodb, mock_firehose, mock_kinesis, mock_s3 +from moto import mock_aws from utils_for_recordprocessor_tests.mock_environment_variables import ( MOCK_ENVIRONMENT_DICT, @@ -21,10 +20,10 @@ GenericTearDown, add_entry_to_table, assert_audit_table_entry, + create_boto3_clients, create_patch, ) from utils_for_recordprocessor_tests.values_for_recordprocessor_tests import ( - REGION_NAME, FileDetails, InfAckFileRows, MockFhirImmsResources, @@ -39,23 +38,24 @@ from common.models.batch_constants import AUDIT_TABLE_NAME, AuditTableKeys, FileStatus from constants import Diagnostics -s3_client = boto3_client("s3", region_name=REGION_NAME) -kinesis_client = boto3_client("kinesis", region_name=REGION_NAME) -firehose_client = boto3_client("firehose", region_name=REGION_NAME) -dynamo_db_client = boto3_client("dynamodb", region_name=REGION_NAME) +s3_client = None +kinesis_client = None +firehose_client = None +dynamo_db_client = None yesterday = datetime.now(UTC) - timedelta(days=1) mock_rsv_emis_file = MockFileDetails.rsv_emis @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT) -@mock_dynamodb -@mock_s3 -@mock_kinesis -@mock_firehose +@mock_aws class TestRecordProcessor(unittest.TestCase): """Tests for main function for RecordProcessor""" def setUp(self) -> None: + global s3_client, kinesis_client, firehose_client, dynamo_db_client + s3_client, kinesis_client, firehose_client, dynamo_db_client = create_boto3_clients( + "s3", "kinesis", "firehose", "dynamodb" + ) GenericSetUp(s3_client, firehose_client, kinesis_client, dynamo_db_client) mock_redis = Mock() @@ -168,7 +168,7 @@ def make_kinesis_assertions(self, test_cases): # Ensure that arrival times are sequential approximate_arrival_timestamp = kinesis_record["ApproximateArrivalTimestamp"] - self.assertGreater( + self.assertGreaterEqual( approximate_arrival_timestamp, previous_approximate_arrival_time_stamp, ) diff --git a/lambdas/recordprocessor/tests/test_send_to_kinesis.py b/lambdas/recordprocessor/tests/test_send_to_kinesis.py index 4a6d3deae8..31e0b48124 100644 --- a/lambdas/recordprocessor/tests/test_send_to_kinesis.py +++ b/lambdas/recordprocessor/tests/test_send_to_kinesis.py @@ -1,8 +1,7 @@ import unittest from unittest.mock import patch -from boto3 import client as boto3_client -from moto import mock_kinesis +from moto import mock_aws from tests.utils_for_recordprocessor_tests.mock_environment_variables import ( MOCK_ENVIRONMENT_DICT, @@ -10,20 +9,20 @@ from tests.utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import ( GenericSetUp, GenericTearDown, -) -from tests.utils_for_recordprocessor_tests.values_for_recordprocessor_tests import ( - REGION_NAME, + create_boto3_clients, ) with patch("os.environ", MOCK_ENVIRONMENT_DICT): from send_to_kinesis import send_to_kinesis -kinesis_client = boto3_client("kinesis", region_name=REGION_NAME) +kinesis_client = None -@mock_kinesis +@mock_aws class TestSendToKinesis(unittest.TestCase): def setUp(self) -> None: + global kinesis_client + (kinesis_client,) = create_boto3_clients("kinesis") GenericSetUp(None, None, kinesis_client) def tearDown(self) -> None: diff --git a/lambdas/recordprocessor/tests/test_utils_for_recordprocessor.py b/lambdas/recordprocessor/tests/test_utils_for_recordprocessor.py index 6ebe2f54f0..ef51fe3d78 100644 --- a/lambdas/recordprocessor/tests/test_utils_for_recordprocessor.py +++ b/lambdas/recordprocessor/tests/test_utils_for_recordprocessor.py @@ -5,8 +5,7 @@ from io import StringIO from unittest.mock import patch -import boto3 -from moto import mock_s3 +from moto import mock_aws from tests.utils_for_recordprocessor_tests.mock_environment_variables import ( MOCK_ENVIRONMENT_DICT, @@ -15,9 +14,9 @@ from tests.utils_for_recordprocessor_tests.utils_for_recordprocessor_tests import ( GenericSetUp, GenericTearDown, + create_boto3_clients, ) from tests.utils_for_recordprocessor_tests.values_for_recordprocessor_tests import ( - REGION_NAME, MockFileDetails, ValidMockFileContent, ) @@ -29,16 +28,18 @@ get_environment, ) -s3_client = boto3.client("s3", region_name=REGION_NAME) +s3_client = None test_file = MockFileDetails.rsv_emis @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT) -@mock_s3 +@mock_aws class TestUtilsForRecordprocessor(unittest.TestCase): """Tests for utils_for_recordprocessor""" def setUp(self) -> None: + global s3_client + (s3_client,) = create_boto3_clients("s3") GenericSetUp(s3_client) def tearDown(self) -> None: diff --git a/lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/mock_environment_variables.py b/lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/mock_environment_variables.py index 6773990313..fb7c8136a8 100644 --- a/lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/mock_environment_variables.py +++ b/lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/mock_environment_variables.py @@ -32,6 +32,11 @@ class Sqs: MOCK_ENVIRONMENT_DICT = { + "AWS_ACCESS_KEY_ID": "testing", + "AWS_SECRET_ACCESS_KEY": "testing", + "AWS_SESSION_TOKEN": "testing", + "AWS_REGION": REGION_NAME, + "AWS_DEFAULT_REGION": REGION_NAME, "ENVIRONMENT": "internal-dev", "LOCAL_ACCOUNT_ID": "123456789012", "SOURCE_BUCKET_NAME": BucketNames.SOURCE, diff --git a/lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/utils_for_recordprocessor_tests.py b/lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/utils_for_recordprocessor_tests.py index a599c88ad9..03cc251a09 100644 --- a/lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/utils_for_recordprocessor_tests.py +++ b/lambdas/recordprocessor/tests/utils_for_recordprocessor_tests/utils_for_recordprocessor_tests.py @@ -21,10 +21,33 @@ with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT): from csv import DictReader - from common.clients import REGION_NAME + import common.clients as common_clients from common.models.batch_constants import AUDIT_TABLE_NAME, AuditTableKeys -dynamodb_client = boto3_client("dynamodb", region_name=REGION_NAME) +REGION_NAME = common_clients.REGION_NAME + +COMMON_CLIENT_CACHE_NAMES = ( + "global_s3_client", + "global_sqs_client", + "global_firehose_client", + "global_secrets_manager_client", + "global_dynamodb_client", + "global_dynamodb_resource", + "global_kinesis_client", +) + + +def reset_common_clients() -> None: + for client_cache_name in COMMON_CLIENT_CACHE_NAMES: + setattr(common_clients, client_cache_name, None) + + +def create_boto3_clients(*service_names: str): + return tuple(boto3_client(service_name, region_name=REGION_NAME) for service_name in service_names) + + +def get_dynamodb_client(): + return boto3_client("dynamodb", region_name=REGION_NAME) def convert_string_to_dict_reader(data_string: str): @@ -48,6 +71,8 @@ def __init__( kinesis_client=None, dynamo_db_client=None, ): + reset_common_clients() + if s3_client: for bucket_name in [ BucketNames.SOURCE, @@ -93,7 +118,7 @@ def __init__( dynamo_db_client=None, ): if s3_client: - for bucket_name in [BucketNames.SOURCE, BucketNames.DESTINATION]: + for bucket_name in [BucketNames.SOURCE, BucketNames.DESTINATION, BucketNames.MOCK_FIREHOSE]: for obj in s3_client.list_objects_v2(Bucket=bucket_name).get("Contents", []): s3_client.delete_object(Bucket=bucket_name, Key=obj["Key"]) s3_client.delete_bucket(Bucket=bucket_name) @@ -110,11 +135,13 @@ def __init__( if dynamo_db_client: dynamo_db_client.delete_table(TableName=AUDIT_TABLE_NAME) + reset_common_clients() + def add_entry_to_table(file_details: MockFileDetails, file_status: str) -> None: """Add an entry to the audit table""" audit_table_entry = {**file_details.audit_table_entry, "status": {"S": file_status}} - dynamodb_client.put_item(TableName=AUDIT_TABLE_NAME, Item=audit_table_entry) + get_dynamodb_client().put_item(TableName=AUDIT_TABLE_NAME, Item=audit_table_entry) def deserialize_dynamodb_types(dynamodb_table_entry_with_types): @@ -127,10 +154,14 @@ def deserialize_dynamodb_types(dynamodb_table_entry_with_types): def assert_audit_table_entry(file_details: FileDetails, expected_status: str, row_count: int | None = None) -> None: """Assert that the file details are in the audit table""" - table_entry = dynamodb_client.get_item( - TableName=AUDIT_TABLE_NAME, - Key={AuditTableKeys.MESSAGE_ID: {"S": file_details.message_id}}, - ).get("Item") + table_entry = ( + get_dynamodb_client() + .get_item( + TableName=AUDIT_TABLE_NAME, + Key={AuditTableKeys.MESSAGE_ID: {"S": file_details.message_id}}, + ) + .get("Item") + ) expected_result = {**file_details.audit_table_entry, "status": {"S": expected_status}} if row_count is not None: diff --git a/lambdas/shared/src/common/batch/audit_table.py b/lambdas/shared/src/common/batch/audit_table.py index 8a2366bba2..f49fe42681 100644 --- a/lambdas/shared/src/common/batch/audit_table.py +++ b/lambdas/shared/src/common/batch/audit_table.py @@ -7,8 +7,6 @@ ITEM_EXISTS_CONDITION_EXPRESSION = f"attribute_exists({AuditTableKeys.MESSAGE_ID})" NOTHING_TO_UPDATE_ERROR_MESSAGE = "Improper usage: you must provide at least one attribute to update" -dynamodb_client = get_dynamodb_client() - def create_audit_table_item( message_id: str, @@ -37,7 +35,7 @@ def create_audit_table_item( } try: - dynamodb_client.put_item(TableName=AUDIT_TABLE_NAME, Item=audit_item) + get_dynamodb_client().put_item(TableName=AUDIT_TABLE_NAME, Item=audit_item) except Exception as error: logger.error(error) raise UnhandledAuditTableError(error) from error @@ -61,7 +59,7 @@ def update_audit_table_item( update_expression, expression_attr_names, expression_attr_values = _build_ddb_update_parameters(attrs_to_update) try: - dynamodb_client.update_item( + get_dynamodb_client().update_item( TableName=AUDIT_TABLE_NAME, Key={AuditTableKeys.MESSAGE_ID: {audit_table_key_data_types_map[AuditTableKeys.MESSAGE_ID]: message_id}}, UpdateExpression=update_expression, @@ -114,7 +112,7 @@ def _build_audit_table_update_log_message(file_key: str, message_id: str, attrs_ def get_ingestion_start_time_by_message_id(event_message_id: str) -> int: """Retrieves ingestion start time by unique event message ID""" # Required by JSON ack file - audit_record = dynamodb_client.get_item( + audit_record = get_dynamodb_client().get_item( TableName=AUDIT_TABLE_NAME, Key={AuditTableKeys.MESSAGE_ID: {"S": event_message_id}} ) @@ -132,7 +130,7 @@ def get_ingestion_start_time_by_message_id(event_message_id: str) -> int: def get_record_count_and_failures_by_message_id(event_message_id: str) -> tuple[int, int]: """Retrieves total record count and total failures by unique event message ID""" - audit_record = dynamodb_client.get_item( + audit_record = get_dynamodb_client().get_item( TableName=AUDIT_TABLE_NAME, Key={AuditTableKeys.MESSAGE_ID: {"S": event_message_id}} ) @@ -152,7 +150,7 @@ def increment_records_failed_count(message_id: str) -> None: try: # Use SET with if_not_exists to safely increment the counter attribute - dynamodb_client.update_item( + get_dynamodb_client().update_item( TableName=AUDIT_TABLE_NAME, Key={AuditTableKeys.MESSAGE_ID: {"S": message_id}}, UpdateExpression="SET #attribute = if_not_exists(#attribute, :initial) + :increment",