Merge branch 'main' into feature/CCM-14499_pin_github_actions_to_shas

Author: damientobin1

infrastructure/terraform/components/dl/data/failure_codes.csv

@@ -0,0 +1,5 @@
+code,description
+DL_PDMV_001,Letter rejected by PDM
+DL_PDMV_002,Timeout waiting for letter storage
+DL_CLIV_003,Attachment contains a virus
+DL_INTE_001,Request rejected by Core API

infrastructure/terraform/components/dl/glue_catalog_table_failure_code_lookup.tf

@@ -0,0 +1,39 @@
+resource "aws_glue_catalog_table" "failure_code_lookup" {
+  name          = "failure_code_lookup"
+  description   = "Lookup table for failure code descriptions"
+  database_name = aws_glue_catalog_database.reporting.name
+
+  table_type = "EXTERNAL_TABLE"
+
+  storage_descriptor {
+    location = "s3://${module.s3bucket_reporting.bucket}/reference-data/failure_codes/"
+
+    input_format  = "org.apache.hadoop.mapred.TextInputFormat"
+    output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+
+    ser_de_info {
+      name                  = "csv"
+      serialization_library = "org.apache.hadoop.hive.serde2.OpenCSVSerde"
+
+      parameters = {
+        "separatorChar"          = ","
+        "skip.header.line.count" = "1"
+      }
+    }
+
+    columns {
+      name = "code"
+      type = "string"
+    }
+
+    columns {
+      name = "description"
+      type = "string"
+    }
+  }
+
+  parameters = {
+    EXTERNAL       = "TRUE"
+    classification = "csv"
+  }
+}

infrastructure/terraform/components/dl/module_sqs_report_generator.tf

@@ -10,7 +10,7 @@ module "sqs_report_generator" {
 
   sqs_kms_key_arn = module.kms.key_arn
 
-  visibility_timeout_seconds = var.sqs_visibility_timeout_seconds
+  visibility_timeout_seconds = 360
 
   create_dlq = true
 

infrastructure/terraform/components/dl/s3_object_failure_codes.tf

@@ -0,0 +1,17 @@
+# Auto-generated CSV containing failure code definitions
+# Source: src/digital-letters-events/failure-codes.ts
+# Build: make build / make generate (runs generate-dependencies)
+resource "aws_s3_object" "failure_codes" {
+  bucket       = module.s3bucket_reporting.bucket
+  key          = "reference-data/failure_codes/failure_codes.csv"
+  source       = "${path.module}/data/failure_codes.csv"
+  content_type = "text/csv"
+  etag         = filemd5("${path.module}/data/failure_codes.csv")
+
+  tags = merge(
+    local.default_tags,
+    {
+      Name = "${local.csi}-failure-codes-csv"
+    }
+  )
+}

infrastructure/terraform/components/dl/scripts/sql/reports/daily_report.sql

@@ -7,19 +7,35 @@ WITH vars AS (
         e.time,
         CASE
             WHEN e.type LIKE '%.item.dequeued.%'
-            OR e.type LIKE '%.queue.digital.letter.read.%' THEN 'Digital'
-            WHEN e.type LIKE '%.print.letter.transitioned.%' THEN 'Print' ELSE NULL
+            OR e.type LIKE '%.queue.digital.letter.read.%'
+            OR e.type LIKE '%.pdm.resource.submission.rejected.%'
+            OR e.type LIKE '%.pdm.resource.retries.exceeded.%'
+            OR e.type LIKE '%.messages.request.rejected.%' THEN 'Digital'
+            WHEN e.type LIKE '%.print.letter.transitioned.%'
+            OR e.type LIKE '%.print.file.quarantined.%' THEN 'Print' ELSE NULL
         END as communicationtype,
         CASE
             WHEN e.type LIKE '%.item.dequeued.%' THEN 'Unread'
             WHEN e.type LIKE '%.queue.digital.letter.read.%' THEN 'Read'
+            WHEN e.type LIKE '%.pdm.resource.submission.rejected.%' THEN 'Failed'
+            WHEN e.type LIKE '%.pdm.resource.retries.exceeded.%' THEN 'Failed'
+            WHEN e.type LIKE '%.messages.request.rejected.%' THEN 'Failed'
+            WHEN e.type LIKE '%.print.file.quarantined.%' THEN 'Failed'
             WHEN e.letterstatus = 'RETURNED' THEN 'Returned'
             WHEN e.letterstatus = 'FAILED' THEN 'Failed'
             WHEN e.letterstatus = 'DISPATCHED' THEN 'Dispatched'
             WHEN e.letterstatus = 'REJECTED' THEN 'Rejected' ELSE NULL
-        END as status
+        END as status,
+        e.reasoncode,
+        COALESCE(
+            CASE WHEN e.type LIKE '%.messages.request.rejected.%' THEN e.reasontext END,
+            fcl.description,
+            e.reasontext,
+            e.reasoncode
+        ) as reasontext
     FROM event_record e
         CROSS JOIN vars v
+        LEFT JOIN failure_code_lookup fcl ON e.reasoncode = fcl.code
     WHERE e.senderid = v.senderid
         AND e.__year = year(v.dt)
         AND e.__month = month(v.dt)
@@ -31,25 +47,31 @@ WITH vars AS (
             ORDER BY te.time DESC,
                 CASE
                     -- Digital Priority Order
+                    WHEN te.communicationtype = 'Digital' AND te.status = 'Failed' THEN 3
                     WHEN te.status = 'Read' THEN 2
                     WHEN te.status = 'Unread' THEN 1
                     -- Print Priority Order
                     WHEN te.status = 'Returned' THEN 4
-                    WHEN te.status = 'Failed' THEN 3
+                    WHEN te.communicationtype = 'Print' AND te.status = 'Failed' THEN 3
                     WHEN te.status = 'Dispatched' THEN 2
                     WHEN te.status = 'Rejected' THEN 1 ELSE 0
                 END DESC
         ) AS "row_number",
         te.messagereference,
         te.time,
         te.communicationtype,
-        te.status
+        te.status,
+        te.reasoncode,
+        te.reasontext
     FROM "translated_events" AS te
-    where te.status IS NOT NULL
+    WHERE te.status IS NOT NULL
+        AND te.communicationtype IS NOT NULL
 )
 SELECT oe.messagereference as "Message Reference",
     oe.time as "Time",
     oe.communicationtype as "Communication Type",
-    oe.status as "Status"
+    oe.status as "Status",
+    oe.reasoncode as "Reason Code",
+    oe.reasontext as "Reason"
 FROM "ordered_events" AS oe
 WHERE oe.row_number = 1

lambdas/core-notifier-lambda/package.json

@@ -1,6 +1,6 @@
 {
   "dependencies": {
-    "axios": "^1.13.5",
+    "axios": "1.14.0",
     "digital-letters-events": "^0.0.1",
     "sender-management": "^0.0.1",
     "utils": "^0.0.1"

lambdas/core-notifier-lambda/src/__tests__/apis/sqs-handler.test.ts

@@ -5,10 +5,10 @@ import { NotifyMessageProcessor } from 'app/notify-message-processor';
 import { ISenderManagement } from 'sender-management';
 import { SqsHandlerDependencies, createHandler } from 'apis/sqs-handler';
 import { parseSqsRecord } from 'app/parse-sqs-message';
-import { InvalidPdmResourceAvailableEvent } from 'domain/invalid-pdm-resource-available-event';
 import { RequestNotifyError } from 'domain/request-notify-error';
 import { validPdmEvent, validSender } from '__tests__/constants';
 import {
+  InvalidEvent,
   MessageRequestRejected,
   MessageRequestSkipped,
   MessageRequestSubmitted,
@@ -195,14 +195,14 @@ describe('createHandler', () => {
     });
   });
 
-  describe('when parseSqsRecord throws InvalidPdmResourceAvailableEvent', () => {
+  describe('when parseSqsRecord throws InvalidEvent', () => {
     it('marks the message as failed for retry', async () => {
       const sqsEvent = createSqsEvent(1);
       const handler = createHandler(dependencies);
       const { messageId } = sqsEvent.Records[0];
 
       mockParseSqsRecord.mockImplementationOnce(() => {
-        throw new InvalidPdmResourceAvailableEvent(messageId);
+        throw new InvalidEvent('Some validation errors');
       });
 
       const result = await handler(sqsEvent);
@@ -211,7 +211,7 @@ describe('createHandler', () => {
         batchItemFailures: [{ itemIdentifier: messageId }],
       });
       expect(mockLogger.warn).toHaveBeenCalledWith({
-        error: 'Unable to parse PDMResourceAvailable event from SQS message',
+        error: 'Unable to parse event',
         description: 'Failed processing message',
         messageId,
         senderId: undefined,
@@ -256,11 +256,13 @@ describe('createHandler', () => {
       const handler = createHandler(dependencies);
       const { messageId } = sqsEvent.Records[0];
       const errorCode = 'VALIDATION_ERROR';
+      const failureReason = 'Request validation failed';
       const correlationId = 'corr-123';
       const error = new RequestNotifyError(
         new Error('Validation failed'),
         correlationId,
         errorCode,
+        failureReason,
       );
       // Add messageReference property dynamically to trigger the terminal error path
       (error as any).messageReference = messageReference;

lambdas/core-notifier-lambda/src/__tests__/app/notify-api-client.test.ts

@@ -211,8 +211,11 @@ describe('sendRequest', () => {
           mockRequest1.data.attributes.messageReference,
         ),
       ).rejects.toMatchObject({
-        errorCode: 'CM_MISSING_ROUTING_PLAN_TEMPLATE',
+        cause: error,
         correlationId: 'request-item-id_request-item-plan-id',
+        errorCode: 'CM_MISSING_ROUTING_PLAN_TEMPLATE',
+        failureReason:
+          'The templates required to use the routing plan were not found.',
       });
     },
   );

lambdas/core-notifier-lambda/src/__tests__/app/parse-sqs-message.test.ts

@@ -2,10 +2,11 @@ import type { SQSRecord } from 'aws-lambda';
 import { mock } from 'jest-mock-extended';
 import { Logger } from 'utils';
 import { parseSqsRecord } from 'app/parse-sqs-message';
-import { InvalidPdmResourceAvailableEvent } from 'domain/invalid-pdm-resource-available-event';
 import { validPdmEvent } from '__tests__/constants';
+import { InvalidEvent } from 'digital-letters-events';
 
 const mockLogger = mock<Logger>();
+const mockChildLogger = mock<Logger>();
 
 describe('parseSqsRecord', () => {
   const messageId = 'test-message-id-123';
@@ -27,6 +28,10 @@ describe('parseSqsRecord', () => {
     awsRegion: 'eu-west-2',
   });
 
+  beforeAll(() => {
+    mockLogger.child.mockReturnValue(mockChildLogger);
+  });
+
   beforeEach(() => {
     jest.clearAllMocks();
   });
@@ -38,13 +43,12 @@ describe('parseSqsRecord', () => {
       const result = parseSqsRecord(sqsRecord, mockLogger);
 
       expect(result).toEqual(validPdmEvent);
-      expect(mockLogger.info).toHaveBeenCalledWith({
+      expect(mockLogger.child).toHaveBeenCalledWith({ messageId });
+      expect(mockChildLogger.info).toHaveBeenCalledWith({
         description: 'Parsing SQS Record',
-        messageId,
       });
-      expect(mockLogger.info).toHaveBeenCalledWith({
+      expect(mockChildLogger.info).toHaveBeenCalledWith({
         description: 'Parsed valid PDMResourceAvailable Event',
-        messageId,
         messageReference: validPdmEvent.data.messageReference,
         senderId: validPdmEvent.data.senderId,
         resourceId: validPdmEvent.data.resourceId,
@@ -53,20 +57,16 @@ describe('parseSqsRecord', () => {
   });
 
   describe('when SQS record contains an invalid PDMResourceAvailable event', () => {
-    it('logs error and throws InvalidPdmResourceAvailableEvent', () => {
+    it('logs error and throws InvalidEvent', () => {
       const invalidEvent = { ...validPdmEvent, data: {} };
       const sqsRecord = createSqsRecord(invalidEvent);
 
-      expect(() => parseSqsRecord(sqsRecord, mockLogger)).toThrow(
-        InvalidPdmResourceAvailableEvent,
-      );
-
-      expect(mockLogger.error).toHaveBeenCalledWith(
+      expect(() => parseSqsRecord(sqsRecord, mockLogger)).toThrow(InvalidEvent);
+      expect(mockLogger.child).toHaveBeenCalledWith({ messageId });
+      expect(mockChildLogger.error).toHaveBeenCalledWith(
         expect.objectContaining({
-          description:
-            'The SQS message does not contain a valid PDMResourceAvailable event',
-          messageId,
-          error: expect.any(Array),
+          description: 'Error parsing PDMResourceAvailable event',
+          err: expect.any(Array),
         }),
       );
     });
@@ -92,9 +92,8 @@ describe('parseSqsRecord', () => {
       };
 
       expect(() => parseSqsRecord(sqsRecord, mockLogger)).toThrow(SyntaxError);
-      expect(mockLogger.info).toHaveBeenCalledWith({
+      expect(mockChildLogger.info).toHaveBeenCalledWith({
         description: 'Parsing SQS Record',
-        messageId,
       });
     });
   });