Merge branch 'master' into feature/VED-1098

Author: motola

infrastructure/account/.terraform.lock.hcl

@@ -182,7 +182,7 @@ resource "aws_lambda_function" "mns_publisher_lambda" {
   package_type  = "Image"
   image_uri     = module.mns_publisher_docker_image.image_uri
   architectures = ["x86_64"]
-  timeout       = 120
+  timeout       = 300
 
   vpc_config {
     subnet_ids         = var.private_subnet_ids

infrastructure/instance/.terraform.lock.hcl

@@ -2,7 +2,7 @@ resource "aws_sqs_queue" "mns_outbound_events" {
   name                       = "${var.mns_publisher_resource_name_prefix}-queue"
   fifo_queue                 = false
   kms_master_key_id          = aws_kms_key.mns_outbound_events.arn
-  visibility_timeout_seconds = 180
+  visibility_timeout_seconds = 450
   redrive_policy = jsonencode({
     deadLetterTargetArn = aws_sqs_queue.mns_outbound_events_dlq.arn
     maxReceiveCount     = 2

infrastructure/instance/modules/mns_publisher/mns_publisher_lambda.tf

@@ -4,7 +4,10 @@
     {
       "Effect": "Allow",
       "Action": "secretsmanager:GetSecretValue",
-      "Resource": "arn:aws:secretsmanager:eu-west-2:${account_id}:secret:imms/outbound/*/*"
+      "Resource": [
+        "arn:aws:secretsmanager:eu-west-2:${account_id}:secret:imms/outbound/*/*",
+        "arn:aws:secretsmanager:eu-west-2:${account_id}:secret:imms/pds/*/*"
+      ]
     }
   ]
 }

infrastructure/instance/modules/mns_publisher/sqs_mns_outbound_events.tf

@@ -1,7 +1,7 @@
 resource "aws_sqs_queue" "id_sync_queue" {
   name                       = "imms-${local.resource_scope}-id-sync-queue"
   kms_master_key_id          = data.aws_kms_key.existing_id_sync_sqs_encryption_key.arn
-  visibility_timeout_seconds = 360
+  visibility_timeout_seconds = 1080 # as per AWS docs to be 6 times the Lambda function timeout but kept to 3 times
   redrive_policy = jsonencode({
     deadLetterTargetArn = aws_sqs_queue.id_sync_dlq.arn
     maxReceiveCount     = 4

infrastructure/instance/policies/secret_manager.json

@@ -28,6 +28,5 @@
     level=os.environ.get("LOG_LEVEL", "INFO"),
     # Serialise uncaught exceptions as structured JSON.
     log_uncaught_exceptions=True,
-    # Set POWERTOOLS_LOGGER_LOG_CALLABLE_LOCATION=true to re-enable locally.
-    location=os.environ.get("POWERTOOLS_LOGGER_LOG_CALLABLE_LOCATION", "false").lower() == "true",
+    location=False,
 )

infrastructure/instance/sqs_id_sync.tf

@@ -1,16 +1,15 @@
 """
-- Parses the incoming AWS event into `AwsLambdaEvent` and iterate its `records`.
-- Delegate each record to `process_record` and collect `nhs_number` from each result.
-- If any record has status == "error" raise `IdSyncException` with aggregated nhs_numbers.
-- Any unexpected error is wrapped into `IdSyncException(message="Error processing id_sync event")`.
+- Parses the incoming AWS event into `AwsLambdaEvent` and iterates its `records`.
+- Delegates each record to `process_record` with per-record exception isolation.
+- Returns {"batchItemFailures": [...]} for any failed records so SQS only re-drives the failing messages.
+- A handler-level exception (bad event schema etc.) re-raises to trigger full batch retry.
 """
 
 from typing import Any
 
 from common.aws_lambda_event import AwsLambdaEvent
 from common.clients import STREAM_NAME, logger
 from common.log_decorator import logging_decorator
-from exceptions.id_sync_exception import IdSyncException
 from record_processor import process_record
 
 
@@ -25,28 +24,32 @@ def handler(event_data: dict[str, Any], _context) -> dict[str, Any]:
 
         logger.info("id_sync processing event with %d records", len(records))
 
-        error_count = 0
+        batch_item_failures = []
 
         for record in records:
-            result = process_record(record)
-
-            if result.get("status") == "error":
-                error_count += 1
-
-        if error_count > 0:
-            raise IdSyncException(
-                message=f"Processed {len(records)} records with {error_count} errors",
-            )
+            try:
+                result = process_record(record)
+                if result.get("status") == "error":
+                    message_id = record.get("messageId")
+                    logger.error(
+                        "id_sync record processing failed for messageId: %s — %s",
+                        message_id,
+                        result.get("message"),
+                    )
+                    batch_item_failures.append({"itemIdentifier": message_id})
+            except Exception:
+                message_id = record.get("messageId")
+                logger.exception("Unexpected error processing messageId: %s", message_id)
+                batch_item_failures.append({"itemIdentifier": message_id})
+
+        if batch_item_failures:
+            logger.error("id_sync completed with %d/%d failures", len(batch_item_failures), len(records))
+            return {"batchItemFailures": batch_item_failures}
 
         response = {"status": "success", "message": f"Successfully processed {len(records)} records"}
-
         logger.info("id_sync handler completed: %s", response)
         return response
 
-    except IdSyncException as e:
-        logger.exception(f"id_sync error: {e.message}")
-        raise
     except Exception:
-        msg = "Error processing id_sync event"
-        logger.exception(msg)
-        raise IdSyncException(message=msg)
+        logger.exception("Unexpected error processing id_sync event")
+        raise