Skip to content

Commit d97dccd

Browse files
ScottFullerton-NHSEdependabot[bot]simonlabareregareth-allan
authored
CCM-14974: Additional failure events in CSV reports (#245)
* CCM-14974: Added 4 new events to the failure report * CCM-14974: Trivy fix * CCM-14974: Trivy fix * CCM-14974: Fix non-null status * CCM-14974: Update sql * CCM-14974: Trivy ignore * CCM-14974: Update test to remove pdm check * Bump h3 from 1.15.5 to 1.15.8 in /src/eventcatalog Bumps [h3](https://github.com/h3js/h3) from 1.15.5 to 1.15.8. - [Release notes](https://github.com/h3js/h3/releases) - [Changelog](https://github.com/h3js/h3/blob/main/CHANGELOG.md) - [Commits](h3js/h3@v1.15.5...v1.15.8) --- updated-dependencies: - dependency-name: h3 dependency-version: 1.15.8 dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> * CCM-14974: Update package lock * CCM-14961: Fix trivy vulnerabilities * CCM-14974: Readd test and change it to message skipped * CCM-14974: Added failure reasons to events * CCM-14974: Fix linting + typecheck * CCM-14974: Fix linting * CCM-14974: Fix unit tests * CCM-14974: Fix sonarcloud * CCM-14974: Add failure reason and code to component test * CCM-14974: Dependency updates * CCM-14974: Remove generate-csv util * CCM-14974: Update package-lock * CCM-14974: Add failure codes to PrintLetterTransitioned events in component tests * CCM-14974: Added failure codes to other failing component test scenarios * CCM-14974: Expand notify-api-client test --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: simonlabarere <simon.labarere1@nhs.net> Co-authored-by: Gareth Allan <157592212+gareth-allan@users.noreply.github.com>
1 parent 2676deb commit d97dccd

29 files changed

Lines changed: 848 additions & 367 deletions

‎infrastructure/terraform/components/dl/data/failure_codes.csv‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
code,description
2+
DL_PDMV_001,Letter rejected by PDM
3+
DL_PDMV_002,Timeout waiting for letter storage
4+
DL_CLIV_003,Attachment contains a virus
5+
DL_INTE_001,Request rejected by Core API

‎infrastructure/terraform/components/dl/glue_catalog_table_failure_code_lookup.tf‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
resource "aws_glue_catalog_table" "failure_code_lookup" {
2+
name = "failure_code_lookup"
3+
description = "Lookup table for failure code descriptions"
4+
database_name = aws_glue_catalog_database.reporting.name
5+
6+
table_type = "EXTERNAL_TABLE"
7+
8+
storage_descriptor {
9+
location = "s3://${module.s3bucket_reporting.bucket}/reference-data/failure_codes/"
10+
11+
input_format = "org.apache.hadoop.mapred.TextInputFormat"
12+
output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
13+
14+
ser_de_info {
15+
name = "csv"
16+
serialization_library = "org.apache.hadoop.hive.serde2.OpenCSVSerde"
17+
18+
parameters = {
19+
"separatorChar" = ","
20+
"skip.header.line.count" = "1"
21+
}
22+
}
23+
24+
columns {
25+
name = "code"
26+
type = "string"
27+
}
28+
29+
columns {
30+
name = "description"
31+
type = "string"
32+
}
33+
}
34+
35+
parameters = {
36+
EXTERNAL = "TRUE"
37+
classification = "csv"
38+
}
39+
}

‎infrastructure/terraform/components/dl/s3_object_failure_codes.tf‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Auto-generated CSV containing failure code definitions
2+
# Source: src/digital-letters-events/failure-codes.ts
3+
# Build: make build / make generate (runs generate-dependencies)
4+
resource "aws_s3_object" "failure_codes" {
5+
bucket = module.s3bucket_reporting.bucket
6+
key = "reference-data/failure_codes/failure_codes.csv"
7+
source = "${path.module}/data/failure_codes.csv"
8+
content_type = "text/csv"
9+
etag = filemd5("${path.module}/data/failure_codes.csv")
10+
11+
tags = merge(
12+
local.default_tags,
13+
{
14+
Name = "${local.csi}-failure-codes-csv"
15+
}
16+
)
17+
}

‎infrastructure/terraform/components/dl/scripts/sql/reports/daily_report.sql‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,35 @@ WITH vars AS (
77
e.time,
88
CASE
99
WHEN e.type LIKE '%.item.dequeued.%'
10-
OR e.type LIKE '%.queue.digital.letter.read.%' THEN 'Digital'
11-
WHEN e.type LIKE '%.print.letter.transitioned.%' THEN 'Print' ELSE NULL
10+
OR e.type LIKE '%.queue.digital.letter.read.%'
11+
OR e.type LIKE '%.pdm.resource.submission.rejected.%'
12+
OR e.type LIKE '%.pdm.resource.retries.exceeded.%'
13+
OR e.type LIKE '%.messages.request.rejected.%' THEN 'Digital'
14+
WHEN e.type LIKE '%.print.letter.transitioned.%'
15+
OR e.type LIKE '%.print.file.quarantined.%' THEN 'Print' ELSE NULL
1216
END as communicationtype,
1317
CASE
1418
WHEN e.type LIKE '%.item.dequeued.%' THEN 'Unread'
1519
WHEN e.type LIKE '%.queue.digital.letter.read.%' THEN 'Read'
20+
WHEN e.type LIKE '%.pdm.resource.submission.rejected.%' THEN 'Failed'
21+
WHEN e.type LIKE '%.pdm.resource.retries.exceeded.%' THEN 'Failed'
22+
WHEN e.type LIKE '%.messages.request.rejected.%' THEN 'Failed'
23+
WHEN e.type LIKE '%.print.file.quarantined.%' THEN 'Failed'
1624
WHEN e.letterstatus = 'RETURNED' THEN 'Returned'
1725
WHEN e.letterstatus = 'FAILED' THEN 'Failed'
1826
WHEN e.letterstatus = 'DISPATCHED' THEN 'Dispatched'
1927
WHEN e.letterstatus = 'REJECTED' THEN 'Rejected' ELSE NULL
20-
END as status
28+
END as status,
29+
e.reasoncode,
30+
COALESCE(
31+
CASE WHEN e.type LIKE '%.messages.request.rejected.%' THEN e.reasontext END,
32+
fcl.description,
33+
e.reasontext,
34+
e.reasoncode
35+
) as reasontext
2136
FROM event_record e
2237
CROSS JOIN vars v
38+
LEFT JOIN failure_code_lookup fcl ON e.reasoncode = fcl.code
2339
WHERE e.senderid = v.senderid
2440
AND e.__year = year(v.dt)
2541
AND e.__month = month(v.dt)
@@ -31,25 +47,31 @@ WITH vars AS (
3147
ORDER BY te.time DESC,
3248
CASE
3349
-- Digital Priority Order
50+
WHEN te.communicationtype = 'Digital' AND te.status = 'Failed' THEN 3
3451
WHEN te.status = 'Read' THEN 2
3552
WHEN te.status = 'Unread' THEN 1
3653
-- Print Priority Order
3754
WHEN te.status = 'Returned' THEN 4
38-
WHEN te.status = 'Failed' THEN 3
55+
WHEN te.communicationtype = 'Print' AND te.status = 'Failed' THEN 3
3956
WHEN te.status = 'Dispatched' THEN 2
4057
WHEN te.status = 'Rejected' THEN 1 ELSE 0
4158
END DESC
4259
) AS "row_number",
4360
te.messagereference,
4461
te.time,
4562
te.communicationtype,
46-
te.status
63+
te.status,
64+
te.reasoncode,
65+
te.reasontext
4766
FROM "translated_events" AS te
48-
where te.status IS NOT NULL
67+
WHERE te.status IS NOT NULL
68+
AND te.communicationtype IS NOT NULL
4969
)
5070
SELECT oe.messagereference as "Message Reference",
5171
oe.time as "Time",
5272
oe.communicationtype as "Communication Type",
53-
oe.status as "Status"
73+
oe.status as "Status",
74+
oe.reasoncode as "Reason Code",
75+
oe.reasontext as "Reason"
5476
FROM "ordered_events" AS oe
5577
WHERE oe.row_number = 1

‎lambdas/core-notifier-lambda/src/__tests__/apis/sqs-handler.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,11 +256,13 @@ describe('createHandler', () => {
256256
const handler = createHandler(dependencies);
257257
const { messageId } = sqsEvent.Records[0];
258258
const errorCode = 'VALIDATION_ERROR';
259+
const failureReason = 'Request validation failed';
259260
const correlationId = 'corr-123';
260261
const error = new RequestNotifyError(
261262
new Error('Validation failed'),
262263
correlationId,
263264
errorCode,
265+
failureReason,
264266
);
265267
// Add messageReference property dynamically to trigger the terminal error path
266268
(error as any).messageReference = messageReference;

‎lambdas/core-notifier-lambda/src/__tests__/app/notify-api-client.test.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,8 +211,11 @@ describe('sendRequest', () => {
211211
mockRequest1.data.attributes.messageReference,
212212
),
213213
).rejects.toMatchObject({
214-
errorCode: 'CM_MISSING_ROUTING_PLAN_TEMPLATE',
214+
cause: error,
215215
correlationId: 'request-item-id_request-item-plan-id',
216+
errorCode: 'CM_MISSING_ROUTING_PLAN_TEMPLATE',
217+
failureReason:
218+
'The templates required to use the routing plan were not found.',
216219
});
217220
},
218221
);

‎lambdas/core-notifier-lambda/src/__tests__/domain/mapper.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,13 +209,15 @@ describe('mapper', () => {
209209
describe('mapPdmEventToMessageRequestRejected', () => {
210210
it('correctly maps PDM event to MessageRequestRejected', () => {
211211
const failureCode = 'INVALID_NHS_NUMBER';
212+
const failureReason = 'NHS number is not valid';
212213
const mockDate = new Date('2024-01-15T12:00:00Z');
213214
jest.spyOn(globalThis, 'Date').mockImplementation(() => mockDate as any);
214215

215216
const result = mapPdmEventToMessageRequestRejected(
216217
mockPdmEvent,
217218
mockSender,
218219
failureCode,
220+
failureReason,
219221
);
220222

221223
expect(result).toEqual({
@@ -232,40 +234,64 @@ describe('mapper', () => {
232234
failureCode: 'INVALID_NHS_NUMBER',
233235
messageUri:
234236
'https://www.nhsapp.service.nhs.uk/digital-letters?letterid=resource-789',
237+
reasonCode: 'DL_INTE_001',
238+
reasonText: 'NHS number is not valid',
235239
},
236240
});
237241

238242
expect(mockRandomUUID).toHaveBeenCalled();
239243
});
240244

245+
it('includes reasonCode and reasonText for reporting', () => {
246+
const failureCode = 'CM_DUPLICATE_REQUEST';
247+
const failureReason = 'This request has already been received';
248+
const result = mapPdmEventToMessageRequestRejected(
249+
mockPdmEvent,
250+
mockSender,
251+
failureCode,
252+
failureReason,
253+
);
254+
255+
expect(result.data.reasonCode).toBe('DL_INTE_001');
256+
expect(result.data.reasonText).toBe(
257+
'This request has already been received',
258+
);
259+
});
260+
241261
it('generates new UUID for event', () => {
242262
const failureCode = 'VALIDATION_ERROR';
263+
const failureReason = 'Request validation failed';
243264
mapPdmEventToMessageRequestRejected(
244265
mockPdmEvent,
245266
mockSender,
246267
failureCode,
268+
failureReason,
247269
);
248270

249271
expect(mockRandomUUID).toHaveBeenCalledTimes(1);
250272
});
251273

252274
it('includes failureCode in data', () => {
253275
const failureCode = 'ROUTING_FAILED';
276+
const failureReason = 'Unable to route message';
254277
const result = mapPdmEventToMessageRequestRejected(
255278
mockPdmEvent,
256279
mockSender,
257280
failureCode,
281+
failureReason,
258282
);
259283

260284
expect(result.data.failureCode).toBe('ROUTING_FAILED');
261285
});
262286

263287
it('includes messageUri with resource ID', () => {
264288
const failureCode = 'TIMEOUT';
289+
const failureReason = 'Request timed out';
265290
const result = mapPdmEventToMessageRequestRejected(
266291
mockPdmEvent,
267292
mockSender,
268293
failureCode,
294+
failureReason,
269295
);
270296

271297
expect(result.data.messageUri).toBe(
@@ -275,32 +301,38 @@ describe('mapper', () => {
275301

276302
it('uses sender senderId in data', () => {
277303
const failureCode = 'UNKNOWN_ERROR';
304+
const failureReason = 'An unknown error occurred';
278305
const result = mapPdmEventToMessageRequestRejected(
279306
mockPdmEvent,
280307
mockSender,
281308
failureCode,
309+
failureReason,
282310
);
283311

284312
expect(result.data.senderId).toBe('test-sender-id');
285313
});
286314

287315
it('uses messageReference from PDM event', () => {
288316
const failureCode = 'DUPLICATE_REQUEST';
317+
const failureReason = 'Duplicate request detected';
289318
const result = mapPdmEventToMessageRequestRejected(
290319
mockPdmEvent,
291320
mockSender,
292321
failureCode,
322+
failureReason,
293323
);
294324

295325
expect(result.data.messageReference).toBe('msg-ref-123');
296326
});
297327

298328
it('sets correct event type', () => {
299329
const failureCode = 'SYSTEM_ERROR';
330+
const failureReason = 'System error occurred';
300331
const result = mapPdmEventToMessageRequestRejected(
301332
mockPdmEvent,
302333
mockSender,
303334
failureCode,
335+
failureReason,
304336
);
305337

306338
expect(result.type).toBe(
@@ -310,10 +342,12 @@ describe('mapper', () => {
310342

311343
it('sets correct dataschema', () => {
312344
const failureCode = 'CONFIG_ERROR';
345+
const failureReason = 'Configuration error';
313346
const result = mapPdmEventToMessageRequestRejected(
314347
mockPdmEvent,
315348
mockSender,
316349
failureCode,
350+
failureReason,
317351
);
318352

319353
expect(result.dataschema).toBe(
@@ -323,10 +357,12 @@ describe('mapper', () => {
323357

324358
it('preserves CloudEvents properties from PDM event', () => {
325359
const failureCode = 'NETWORK_ERROR';
360+
const failureReason = 'Network connection failed';
326361
const result = mapPdmEventToMessageRequestRejected(
327362
mockPdmEvent,
328363
mockSender,
329364
failureCode,
365+
failureReason,
330366
);
331367

332368
expect(result.specversion).toBe('1.0');

‎lambdas/core-notifier-lambda/src/apis/sqs-handler.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ async function processSqsRecord(
125125
incoming,
126126
sender,
127127
error.errorCode,
128+
error.failureReason,
128129
);
129130
} else {
130131
// this might be a transient error so we notify the queue to retry

‎lambdas/core-notifier-lambda/src/app/notify-api-client.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ export class NotifyClient implements INotifyClient {
109109
error,
110110
correlationId,
111111
errorBody?.errors[0].code,
112+
errorBody?.errors[0].detail,
112113
);
113114
}
114115
}

‎lambdas/core-notifier-lambda/src/domain/mapper.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import {
88
} from 'digital-letters-events';
99
import type { SingleMessageRequest } from 'domain/request';
1010

11+
const CORE_API_FAILURE_CODE = 'DL_INTE_001';
12+
1113
const DIGITAL_LETTER_URL =
1214
'https://www.nhsapp.service.nhs.uk/digital-letters?letterid=';
1315

@@ -99,6 +101,7 @@ export function mapPdmEventToMessageRequestRejected(
99101
pdmResourceAvailable: PDMResourceAvailable,
100102
sender: Sender,
101103
notifyFailureCode: string,
104+
failureReason: string,
102105
): MessageRequestRejected {
103106
const { data } = pdmResourceAvailable;
104107
const { messageReference } = data;
@@ -117,6 +120,8 @@ export function mapPdmEventToMessageRequestRejected(
117120
senderId: sender.senderId,
118121
failureCode: notifyFailureCode,
119122
messageUri: `${DIGITAL_LETTER_URL}${data.resourceId}`,
123+
reasonCode: CORE_API_FAILURE_CODE,
124+
reasonText: failureReason,
120125
},
121126
};
122127
}

0 commit comments

Comments
 (0)