CCM-13278: CCM-12616 + CCM-13278

Makefile

@@ -12,10 +12,10 @@ quick-start: config clean build serve-docs # Quick start target to setup, build
 dependencies:: # Install dependencies needed to build and test the project @Pipeline
 	$(MAKE) -C src/cloudevents install
 	$(MAKE) -C src/eventcatalogasyncapiimporter install
+	$(MAKE) -C lambdas/mesh-acknowledge install
+	$(MAKE) -C utils/py-utils install
 	$(MAKE) -C lambdas/mesh-poll install
 	$(MAKE) -C lambdas/mesh-download install
-	$(MAKE) -C utils/metric-publishers install
-	$(MAKE) -C utils/event-publisher-py install
 	$(MAKE) -C utils/py-mock-mesh install
 	./scripts/set-github-token.sh
 	npm install --workspaces
@@ -44,10 +44,10 @@ clean:: # Clean-up project resources (main) @Operations
 	$(MAKE) -C src/cloudevents clean && \
 	$(MAKE) -C src/eventcatalogasyncapiimporter clean && \
 	$(MAKE) -C src/eventcatalogasyncapiimporter clean-output && \
+	$(MAKE) -C lambdas/mesh-acknowledge clean && \
+	$(MAKE) -C utils/py-utils clean && \
 	$(MAKE) -C lambdas/mesh-poll clean && \
 	$(MAKE) -C lambdas/mesh-download clean && \
-	$(MAKE) -C utils/metric-publishers clean && \
-	$(MAKE) -C utils/event-publisher-py clean && \
 	$(MAKE) -C utils/py-mock-mesh clean && \
 	$(MAKE) -C src/python-schema-generator clean && \
 	rm -f .version

infrastructure/terraform/components/dl/README.md

@@ -45,6 +45,7 @@ No requirements.
 | <a name="module_kms"></a> [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-kms.zip | n/a |
 | <a name="module_lambda_apim_key_generation"></a> [lambda\_apim\_key\_generation](#module\_lambda\_apim\_key\_generation) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_lambda_lambda_apim_refresh_token"></a> [lambda\_lambda\_apim\_refresh\_token](#module\_lambda\_lambda\_apim\_refresh\_token) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
+| <a name="module_mesh_acknowledge"></a> [mesh\_acknowledge](#module\_mesh\_acknowledge) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_mesh_download"></a> [mesh\_download](#module\_mesh\_download) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_mesh_poll"></a> [mesh\_poll](#module\_mesh\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_pdm_mock"></a> [pdm\_mock](#module\_pdm\_mock) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
@@ -58,6 +59,7 @@ No requirements.
 | <a name="module_s3bucket_static_assets"></a> [s3bucket\_static\_assets](#module\_s3bucket\_static\_assets) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a |
 | <a name="module_sqs_core_notifier"></a> [sqs\_core\_notifier](#module\_sqs\_core\_notifier) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
 | <a name="module_sqs_event_publisher_errors"></a> [sqs\_event\_publisher\_errors](#module\_sqs\_event\_publisher\_errors) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
+| <a name="module_sqs_mesh_acknowledge"></a> [sqs\_mesh\_acknowledge](#module\_sqs\_mesh\_acknowledge) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-sqs.zip | n/a |
 | <a name="module_sqs_mesh_download"></a> [sqs\_mesh\_download](#module\_sqs\_mesh\_download) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
 | <a name="module_sqs_pdm_poll"></a> [sqs\_pdm\_poll](#module\_sqs\_pdm\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
 | <a name="module_sqs_pdm_uploader"></a> [sqs\_pdm\_uploader](#module\_sqs\_pdm\_uploader) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |

infrastructure/terraform/components/dl/cloudwatch_event_rule_mesh_inbox_message_downloaded.tf

@@ -25,3 +25,10 @@ resource "aws_cloudwatch_event_target" "pdm-uploader-target" {
   target_id      = "pdm-uploader-target"
   event_bus_name = aws_cloudwatch_event_bus.main.name
 }
+
+resource "aws_cloudwatch_event_target" "mesh-acknowledge-target" {
+  rule           = aws_cloudwatch_event_rule.mesh_inbox_message_downloaded.name
+  arn            = module.sqs_mesh_acknowledge.sqs_queue_arn
+  target_id      = "mesh-acknowledge-target"
+  event_bus_name = aws_cloudwatch_event_bus.main.name
+}

infrastructure/terraform/components/dl/lambda_event_source_mapping_mesh_acknowledge_lambda.tf

@@ -0,0 +1,10 @@
+resource "aws_lambda_event_source_mapping" "sqs_mesh_acknowledge_lambda" {
+  event_source_arn                   = module.sqs_mesh_acknowledge.sqs_queue_arn
+  function_name                      = module.mesh_acknowledge.function_name
+  batch_size                         = var.queue_batch_size
+  maximum_batching_window_in_seconds = var.queue_batch_window_seconds
+
+  function_response_types = [
+    "ReportBatchItemFailures"
+  ]
+}

infrastructure/terraform/components/dl/locals.tf

@@ -5,7 +5,9 @@ locals {
   apim_api_key_ssm_parameter_name      = "/${var.component}/${var.environment}/apim/api_key"
   apim_private_key_ssm_parameter_name  = "/${var.component}/${var.environment}/apim/private_key"
   apim_keystore_s3_bucket              = "nhs-${var.aws_account_id}-${var.region}-${var.environment}-${var.component}-static-assets"
-  ssm_mesh_prefix                      = "/${var.component}/${var.environment}/mesh"
+  ssm_prefix                           = "/${var.component}/${var.environment}"
+  ssm_mesh_prefix                      = "${local.ssm_prefix}/mesh"
+  ssm_senders_prefix                   = "${local.ssm_prefix}/senders"
   mock_mesh_endpoint                   = "s3://${module.s3bucket_non_pii_data.bucket}/mock-mesh"
   root_domain_name                     = "${var.environment}.${local.acct.route53_zone_names["digital-letters"]}"
   root_domain_id                       = local.acct.route53_zone_ids["digital-letters"]

infrastructure/terraform/components/dl/module_lambda_mesh_acknowledge.tf

@@ -0,0 +1,150 @@
+module "mesh_acknowledge" {
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip"
+
+  function_name = "mesh-acknowledge"
+  description   = "A lambda function for acknowledging MESH messages"
+  aws_account_id = var.aws_account_id
+  component      = local.component
+  environment    = var.environment
+  project        = var.project
+  region         = var.region
+  group          = var.group
+
+  log_retention_in_days = var.log_retention_in_days
+  kms_key_arn           = module.kms.key_arn
+
+  iam_policy_document = {
+    body = data.aws_iam_policy_document.mesh_acknowledge_lambda.json
+  }
+
+  function_s3_bucket      = local.acct.s3_buckets["lambda_function_artefacts"]["id"]
+  function_code_base_path = local.aws_lambda_functions_dir_path
+  function_code_dir       = "mesh-acknowledge/target/dist"
+  function_include_common = true
+  function_module_name    = "mesh_acknowledge"
+  handler_function_name   = "handler.handler"
+  runtime                 = "python3.14"
+  memory                  = 128
+  timeout                 = 5
+  log_level               = var.log_level
+
+  force_lambda_code_deploy = var.force_lambda_code_deploy
+  enable_lambda_insights   = false
+
+  log_destination_arn       = local.log_destination_arn
+  log_subscription_role_arn = local.acct.log_subscription_role_arn
+
+  lambda_env_vars = {
+    DLQ_URL                             = module.sqs_mesh_acknowledge.sqs_dlq_url
+    ENVIRONMENT                         = var.environment
+    EVENT_PUBLISHER_DLQ_URL             = module.sqs_event_publisher_errors.sqs_queue_url
+    EVENT_PUBLISHER_EVENT_BUS_ARN       = aws_cloudwatch_event_bus.main.arn
+    MOCK_MESH_BUCKET                    = module.s3bucket_non_pii_data.bucket
+    SSM_MESH_PREFIX                     = "${local.ssm_mesh_prefix}"
+    SSM_SENDERS_PREFIX                  = "${local.ssm_senders_prefix}"
+    USE_MESH_MOCK                       = var.enable_mock_mesh ? "true" : "false"
+  }
+
+}
+
+data "aws_iam_policy_document" "mesh_acknowledge_lambda" {
+  statement {
+    sid    = "KMSPermissions"
+    effect = "Allow"
+
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey",
+    ]
+
+    resources = [
+      module.kms.key_arn,
+    ]
+  }
+
+  statement {
+    sid    = "SQSPermissions"
+    effect = "Allow"
+
+    actions = [
+      "sqs:ReceiveMessage",
+      "sqs:DeleteMessage",
+      "sqs:GetQueueAttributes",
+    ]
+
+    resources = [
+      module.sqs_mesh_acknowledge.sqs_queue_arn,
+    ]
+  }
+
+  statement {
+    sid    = "SQSDLQPermissions"
+    effect = "Allow"
+
+    actions = [
+      "sqs:SendMessage",
+    ]
+
+    resources = [
+      module.sqs_mesh_acknowledge.sqs_dlq_arn,
+    ]
+  }
+
+  statement {
+    sid    = "EventBridgePermissions"
+    effect = "Allow"
+
+    actions = [
+      "events:PutEvents",
+    ]
+
+    resources = [
+      aws_cloudwatch_event_bus.main.arn,
+    ]
+  }
+
+  statement {
+    sid    = "DLQPermissions"
+    effect = "Allow"
+
+    actions = [
+      "sqs:SendMessage",
+      "sqs:SendMessageBatch",
+    ]
+
+    resources = [
+      module.sqs_event_publisher_errors.sqs_queue_arn,
+    ]
+  }
+
+  statement {
+    sid    = "SSMPermissions"
+    effect = "Allow"
+
+    actions = [
+      "ssm:GetParameter",
+      "ssm:GetParametersByPath",
+    ]
+
+    resources = [
+      "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter${local.ssm_prefix}/*"
+    ]
+  }
+
+  # Grant S3 PutObject permissions for the mock-mesh directory only when the mock is enabled
+  dynamic "statement" {
+    for_each = var.enable_mock_mesh ? [1] : []
+    content {
+      sid    = "MockMeshPutObject"
+      effect = "Allow"
+
+      actions = [
+        "s3:PutObject",
+      ]
+
+      resources = [
+        "${module.s3bucket_non_pii_data.arn}/mock-mesh/*"
+      ]
+    }
+  }
+}

infrastructure/terraform/components/dl/module_lambda_mesh_download.tf

@@ -37,15 +37,14 @@ module "mesh_download" {
   log_subscription_role_arn = local.acct.log_subscription_role_arn
 
   lambda_env_vars = {
-    CERTIFICATE_EXPIRY_METRIC_NAME      = "mesh-download-client-certificate-near-expiry"
-    CERTIFICATE_EXPIRY_METRIC_NAMESPACE = "dl-mesh-download"
     DOWNLOAD_METRIC_NAME                = "mesh-download-successful-downloads"
     DOWNLOAD_METRIC_NAMESPACE           = "dl-mesh-download"
     ENVIRONMENT                         = var.environment
     EVENT_PUBLISHER_DLQ_URL             = module.sqs_event_publisher_errors.sqs_queue_url
     EVENT_PUBLISHER_EVENT_BUS_ARN       = aws_cloudwatch_event_bus.main.arn
     PII_BUCKET                          = module.s3bucket_pii_data.bucket
-    SSM_PREFIX                          = "${local.ssm_mesh_prefix}"
+    SSM_MESH_PREFIX                     = "${local.ssm_mesh_prefix}"
+    SSM_SENDERS_PREFIX                  = "${local.ssm_senders_prefix}"
     USE_MESH_MOCK                       = var.enable_mock_mesh ? "true" : "false"
   }
 
@@ -172,7 +171,8 @@ data "aws_iam_policy_document" "mesh_download_lambda" {
     ]
 
     resources = [
-      "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter${local.ssm_mesh_prefix}/*"
+      "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter${local.ssm_mesh_prefix}/*",
+      "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter${local.ssm_senders_prefix}/*"
     ]
   }
 }

infrastructure/terraform/components/dl/module_lambda_mesh_poll.tf

@@ -45,7 +45,8 @@ module "mesh_poll" {
     MAXIMUM_RUNTIME_MILLISECONDS        = "240000"  # 4 minutes (Lambda has 5 min timeout)
     POLLING_METRIC_NAME                 = "mesh-poll-successful-polls"
     POLLING_METRIC_NAMESPACE            = "dl-mesh-poll"
-    SSM_PREFIX                          = "${local.ssm_mesh_prefix}"
+    SSM_MESH_PREFIX                     = "${local.ssm_mesh_prefix}"
+    SSM_SENDERS_PREFIX                  = "${local.ssm_senders_prefix}"
     USE_MESH_MOCK                       = var.enable_mock_mesh ? "true" : "false"
   }
 
@@ -144,7 +145,8 @@ data "aws_iam_policy_document" "mesh_poll_lambda" {
     ]
 
     resources = [
-      "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter${local.ssm_mesh_prefix}/*"
+      "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter${local.ssm_mesh_prefix}/*",
+      "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter${local.ssm_senders_prefix}/*"
     ]
   }
 }

infrastructure/terraform/components/dl/module_sqs_mesh_acknowledge.tf

@@ -0,0 +1,38 @@
+module "sqs_mesh_acknowledge" {
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-sqs.zip"
+
+  aws_account_id = var.aws_account_id
+  component      = local.component
+  environment    = var.environment
+  project        = var.project
+  region         = var.region
+  name           = "mesh-acknowledge"
+
+  sqs_kms_key_arn = module.kms.key_arn
+
+  visibility_timeout_seconds = 60
+
+  create_dlq = true
+
+  sqs_policy_overload = data.aws_iam_policy_document.sqs_mesh_acknowledge.json
+}
+
+data "aws_iam_policy_document" "sqs_mesh_acknowledge" {
+  statement {
+    sid    = "AllowEventBridgeToSendMessage"
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["events.amazonaws.com"]
+    }
+
+    actions = [
+      "sqs:SendMessage"
+    ]
+
+    resources = [
+      "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-mesh-acknowledge-queue"
+    ]
+  }
+}

infrastructure/terraform/components/dl/module_sqs_mesh_download.tf

@@ -32,7 +32,7 @@ data "aws_iam_policy_document" "sqs_mesh_download" {
     ]
 
     resources = [
-      "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-${local.component}-mesh-download-queue"
+      "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-mesh-download-queue"
     ]
   }
 }

infrastructure/terraform/components/dl/pre.sh

@@ -23,5 +23,6 @@ npm run lambda-build --workspaces --if-present
 # Build Python lambdas
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
 
+make -C "$ROOT/lambdas/mesh-acknowledge" package
 make -C "$ROOT/lambdas/mesh-poll" package
 make -C "$ROOT/lambdas/mesh-download" package

infrastructure/terraform/components/dl/ssm_parameter_mesh_config.tf → infrastructure/terraform/components/dl/ssm_parameter_mesh.tf

@@ -8,7 +8,7 @@ resource "aws_ssm_parameter" "mesh_config" {
   value = var.enable_mock_mesh ? jsonencode({
     mesh_endpoint         = local.mock_mesh_endpoint
     mesh_mailbox          = "mock-mailbox"
-    mesh_mailbox_password = "mock-password"
+    mesh_mailbox_password = "UNSET"
     mesh_shared_key       = "mock-shared-key"
   }) : jsonencode({
     mesh_endpoint         = "UNSET"

lambdas/mesh-acknowledge/.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.venv

lambdas/mesh-acknowledge/Makefile

@@ -0,0 +1,34 @@
+PACKAGE=mesh_acknowledge
+VERSION=0.1.0
+
+install:
+	pip install -r requirements.txt
+
+install-dev:
+	pip install -r requirements-dev.txt
+
+test:
+	cd ../.. && PYTHONPATH=lambdas/mesh-acknowledge:$$PYTHONPATH pytest lambdas/mesh-acknowledge/mesh_acknowledge/__tests__/ -v
+
+coverage:
+	cd ../.. && PYTHONPATH=lambdas/mesh-acknowledge:$$PYTHONPATH pytest lambdas/mesh-acknowledge/mesh_acknowledge/__tests__/ \
+		--cov=lambdas/mesh-acknowledge/mesh_acknowledge \
+		--cov-config=lambdas/mesh-acknowledge/pytest.ini \
+		--cov-report=html:lambdas/mesh-acknowledge/htmlcov \
+		--cov-report=term-missing \
+		--cov-report=xml:lambdas/mesh-acknowledge/coverage.xml \
+		--cov-branch
+
+lint:
+	pylint mesh_acknowledge
+
+format:
+	autopep8 -ri .
+
+package:
+	../../utils/package_python_lambda.sh meshacknowledgelambda
+
+clean:
+	rm -rf target
+
+.PHONY: install install-dev test coverage lint format package clean

lambdas/mesh-acknowledge/mesh_acknowledge/__init__.py

@@ -0,0 +1,9 @@
+"""
+MESH Acknowledge Lambda
+
+This lambda handles acknowledging received MESH files, by sending a message to the MESH inbox of
+their sender.
+"""
+
+__version__ = '0.1.0'
+from .handler import *

lambdas/mesh-acknowledge/mesh_acknowledge/__tests__/__init__.py

@@ -0,0 +1 @@
+# Test package init