VED-901: Update extended attribute file destination. (#1141)

* VED-901: Update extended attributes destination prefix.

* VED-901: Update tests.

* VED-901: Upload EA files directly to the S3 submission upload bucket.

* VED-901: Lint tfvars files.

Author: mfjarvis

infrastructure/instance/environments/prod/blue/variables.tfvars

@@ -6,4 +6,5 @@ pds_environment                   = "prod"
 batch_error_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = false
-dspp_kms_key_alias                = "nhsd-dspp-core-prod-extended-attributes-gdp-key"
+dspp_submission_s3_bucket_name    = "nhsd-dspp-core-prod-s3-submission-upload"
+dspp_submission_kms_key_alias     = "nhsd-dspp-core-prod-s3-submission-upload-key"

infrastructure/instance/environments/prod/green/variables.tfvars

@@ -6,4 +6,5 @@ pds_environment                   = "prod"
 batch_error_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = false
-dspp_kms_key_alias                = "nhsd-dspp-core-prod-extended-attributes-gdp-key"
+dspp_submission_s3_bucket_name    = "nhsd-dspp-core-prod-s3-submission-upload"
+dspp_submission_kms_key_alias     = "nhsd-dspp-core-prod-s3-submission-upload-key"

infrastructure/instance/file_name_processor.tf

@@ -3,18 +3,8 @@ locals {
   filename_lambda_dir     = abspath("${path.root}/../../lambdas/filenameprocessor")
   filename_lambda_files   = fileset(local.filename_lambda_dir, "**")
   filename_lambda_dir_sha = sha1(join("", [for f in local.filename_lambda_files : filesha1("${local.filename_lambda_dir}/${f}")]))
-  dps_bucket_name_for_extended_attribute = (
-    var.environment == "prod"
-    ? "nhsd-dspp-core-prod-extended-attributes-gdp"
-    : "nhsd-dspp-core-ref-extended-attributes-gdp"
-  )
-  dps_bucket_arn_for_extended_attribute = [
-    "arn:aws:s3:::${local.dps_bucket_name_for_extended_attribute}/*"
-  ]
 }
 
-
-
 resource "aws_ecr_repository" "file_name_processor_lambda_repository" {
   image_scanning_configuration {
     scan_on_push = true
@@ -79,7 +69,7 @@ resource "aws_ecr_repository_policy" "filenameprocessor_lambda_ECRImageRetreival
         ],
         "Condition" : {
           "StringLike" : {
-            "aws:sourceArn" : "arn:aws:lambda:eu-west-2:${var.immunisation_account_id}:function:${local.short_prefix}-filenameproc_lambda"
+            "aws:sourceArn" : "arn:aws:lambda:eu-west-2:${var.immunisation_account_id}:function:${local.short_prefix}-filenameproc-lambda"
           }
         }
       }
@@ -116,7 +106,7 @@ resource "aws_iam_policy" "filenameprocessor_lambda_exec_policy" {
           "logs:CreateLogStream",
           "logs:PutLogEvents"
         ]
-        Resource = "arn:aws:logs:${var.aws_region}:${var.immunisation_account_id}:log-group:/aws/lambda/${local.short_prefix}-filenameproc_lambda:*"
+        Resource = "arn:aws:logs:${var.aws_region}:${var.immunisation_account_id}:log-group:/aws/lambda/${local.short_prefix}-filenameproc-lambda:*"
       },
       {
         Effect = "Allow"
@@ -178,7 +168,7 @@ resource "aws_iam_policy" "filenameprocessor_lambda_exec_policy" {
         "Action" : [
           "s3:PutObject"
         ],
-        "Resource" : local.dps_bucket_arn_for_extended_attribute
+        "Resource" : ["arn:aws:s3:::${var.dspp_submission_s3_bucket_name}/*"]
       }
     ]
   })
@@ -264,14 +254,14 @@ resource "aws_iam_policy" "filenameprocessor_dps_extended_attribute_kms_policy"
       {
         Effect = "Allow",
         Action = [
-          "kms:Decrypt",
+          "kms:Encrypt",
           "kms:GenerateDataKey",
-          "kms:DescribeKey"
+          "kms:DescribeKey",
         ],
         Resource = "arn:aws:kms:eu-west-2:${var.dspp_core_account_id}:key/*",
         "Condition" = {
           "ForAnyValue:StringEquals" = {
-            "kms:ResourceAliases" = "alias/${var.dspp_kms_key_alias}"
+            "kms:ResourceAliases" = "alias/${var.dspp_submission_kms_key_alias}"
           }
         }
       }
@@ -311,7 +301,7 @@ resource "aws_iam_role_policy_attachment" "filenameprocessor_lambda_dynamo_acces
 
 # Lambda Function with Security Group and VPC.
 resource "aws_lambda_function" "file_processor_lambda" {
-  function_name = "${local.short_prefix}-filenameproc_lambda"
+  function_name = "${local.short_prefix}-filenameproc-lambda"
   role          = aws_iam_role.filenameprocessor_lambda_exec_role.arn
   package_type  = "Image"
   image_uri     = module.file_processor_docker_image.image_uri
@@ -329,7 +319,7 @@ resource "aws_lambda_function" "file_processor_lambda" {
       DPS_ACCOUNT_ID       = var.dspp_core_account_id
       SOURCE_BUCKET_NAME   = aws_s3_bucket.batch_data_source_bucket.bucket
       ACK_BUCKET_NAME      = aws_s3_bucket.batch_data_destination_bucket.bucket
-      DPS_BUCKET_NAME      = local.dps_bucket_name_for_extended_attribute
+      DPS_BUCKET_NAME      = var.dspp_submission_s3_bucket_name
       QUEUE_URL            = aws_sqs_queue.batch_file_created.url
       REDIS_HOST           = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address
       REDIS_PORT           = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port
@@ -371,7 +361,7 @@ resource "aws_s3_bucket_notification" "datasources_lambda_notification" {
 }
 
 resource "aws_cloudwatch_log_group" "file_name_processor_log_group" {
-  name              = "/aws/lambda/${local.short_prefix}-filenameproc_lambda"
+  name              = "/aws/lambda/${local.short_prefix}-filenameproc-lambda"
   retention_in_days = 30
 }
 

infrastructure/instance/variables.tf

@@ -27,10 +27,16 @@ variable "mns_account_id" {
   default     = "631615744739"
 }
 
-variable "dspp_kms_key_alias" {
-  description = "Alias name of the DPS KMS key allowed for SSE-KMS encryption"
+variable "dspp_submission_s3_bucket_name" {
+  description = "Name of the DSPP (DPS) S3 bucket where extended attributes files should be submitted"
   type        = string
-  default     = "nhsd-dspp-core-ref-extended-attributes-gdp-key"
+  default     = "nhsd-dspp-core-ref-s3-submission-upload"
+}
+
+variable "dspp_submission_kms_key_alias" {
+  description = "Alias of the DSPP (DPS) KMS key required to encrypt extended attributes files"
+  type        = string
+  default     = "nhsd-dspp-core-ref-s3-submission-upload-key"
 }
 
 variable "create_mesh_processor" {

lambdas/filenameprocessor/src/constants.py

@@ -27,7 +27,7 @@
 # Currently only COVID extended attributes files are supported, might be extended in future for other vaccine types
 EXTENDED_ATTRIBUTES_VACC_TYPE = "COVID"
 
-DPS_DESTINATION_PREFIX = "dps_destination"
+DPS_DESTINATION_PREFIX = "generic/EXTENDED_ATTRIBUTES_DAILY_1"
 EXTENDED_ATTRIBUTES_ARCHIVE_PREFIX = "extended-attributes-archive"
 VALID_EA_VERSIONS = ["V1_5"]
 ERROR_TYPE_TO_STATUS_CODE_MAP = {

lambdas/filenameprocessor/tests/test_lambda_handler.py

@@ -292,8 +292,8 @@ def test_lambda_handler_extended_attributes_success(self, mock_get_redis_client)
         archived_obj = s3_client.get_object(Bucket=BucketNames.SOURCE, Key=archived_key)
         self.assertIsNotNone(archived_obj)
 
-        # Also verify file copied to DPS destination bucket under dps_destination/<file_key>
-        dps_key = f"dps_destination/{test_cases[0].file_key}"
+        # Also verify file copied to DPS destination bucket under generic/EXTENDED_ATTRIBUTES_DAILY_1/<file_key>
+        dps_key = f"generic/EXTENDED_ATTRIBUTES_DAILY_1/{test_cases[0].file_key}"
         copied_obj = s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=dps_key)
         self.assertIsNotNone(copied_obj)
 
@@ -467,7 +467,7 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi
         # Ensure processed path hit by checking archive move in source bucket
         s3_client.get_object(Bucket=BucketNames.SOURCE, Key=f"extended-attributes-archive/{csv_key}")
         # And verify copy to DPS destination
-        s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=f"dps_destination/{csv_key}")
+        s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=f"generic/EXTENDED_ATTRIBUTES_DAILY_1/{csv_key}")
 
         # .DAT accepted
         dat_key = MockFileDetails.extended_attributes_file.file_key[:-3] + "dat"
@@ -478,7 +478,7 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi
         ):
             lambda_handler(self.make_event([self.make_record(dat_key)]), None)
         s3_client.get_object(Bucket=BucketNames.SOURCE, Key=f"extended-attributes-archive/{dat_key}")
-        s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=f"dps_destination/{dat_key}")
+        s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=f"generic/EXTENDED_ATTRIBUTES_DAILY_1/{dat_key}")
 
         # Invalid extension fails
         bad_ext_key = csv_key[:-3] + "txt"