CCM-14411: Try transforming events using EventBridge

Author: gareth-allan

infrastructure/terraform/components/dl/cloudwatch_event_rule_all_events.tf

@@ -17,4 +17,29 @@ resource "aws_cloudwatch_event_target" "reporting_firehose" {
   arn            = aws_kinesis_firehose_delivery_stream.to_s3_reporting.arn
   role_arn       = aws_iam_role.eventbridge_firehose.arn
   event_bus_name = aws_cloudwatch_event_bus.main.name
+
+  input_transformer {
+    input_paths = {
+      time             = "$.detail.time",
+      type             = "$.detail.type",
+      messageReference = "$.detail.data.messageReference",
+      pageCount        = "$.detail.data.pageCount",
+      reasonCode       = "$.detail.data.reasonCode",
+      reasonText       = "$.detail.data.reasonText",
+      senderId         = "$.detail.data.senderId",
+      supplierId       = "$.detail.data.supplierId",
+    }
+    input_template = <<EOF
+{
+  "time": <time>,
+  "type": <type>,
+  "messageReference": <messageReference>,
+  "pageCount": <pageCount>,
+  "reasonCode": <reasonCode>,
+  "reasonText": <reasonText>,
+  "senderId": <senderId>,
+  "supplierId": <supplierId>
+}
+EOF
+  }
 }

infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf

@@ -7,7 +7,7 @@ resource "aws_kinesis_firehose_delivery_stream" "to_s3_reporting" {
     role_arn   = aws_iam_role.firehose_role.arn
     bucket_arn = module.s3bucket_reporting.arn
 
-    prefix              = "${local.firehose_output_path_prefix}/reporting/parquet/${aws_glue_catalog_table.event_record.name}/senderid=!{partitionKeyFromLambda:senderId}/__year=!{partitionKeyFromLambda:year}/__month=!{partitionKeyFromLambda:month}/__day=!{partitionKeyFromLambda:day}/"
+    prefix              = "${local.firehose_output_path_prefix}/reporting/parquet/${aws_glue_catalog_table.event_record.name}/senderid=!{partitionKeyFromQuery:senderId}/__year=!{partitionKeyFromQuery:year}/__month=!{partitionKeyFromQuery:month}/__day=!{partitionKeyFromQuery:day}/"
     error_output_prefix = "${local.firehose_output_path_prefix}/errors/!{timestamp:yyyy}-!{timestamp:MM}-!{timestamp:dd}-!{timestamp:HH}/!{firehose:error-output-type}/"
 
     buffering_size     = 128
@@ -21,23 +21,15 @@ resource "aws_kinesis_firehose_delivery_stream" "to_s3_reporting" {
       enabled = "true"
 
       processors {
-        type = "Lambda"
+        type = "MetadataExtraction"
 
         parameters {
-          parameter_name  = "LambdaArn"
-          parameter_value = "${module.report_event_transformer.function_arn}:$LATEST"
+          parameter_name  = "JsonParsingEngine"
+          parameter_value = "JQ-1.6"
         }
         parameters {
-          parameter_name  = "RoleArn"
-          parameter_value = aws_iam_role.firehose_role.arn
-        }
-        parameters {
-          parameter_name  = "BufferSizeInMBs"
-          parameter_value = 1
-        }
-        parameters {
-          parameter_name  = "BufferIntervalInSeconds"
-          parameter_value = 301
+          parameter_name  = "MetadataExtractionQuery"
+          parameter_value = "{senderId:.data.senderId, year:.event_timestamp| strftime(\"%Y\"), month:.event_timestamp| strftime(\"%m\"), day:.event_timestamp| strftime(\"%d\")}"
         }
       }
     }