diff --git a/infrastructure/terraform/components/dl/README.md b/infrastructure/terraform/components/dl/README.md index f811d47a7..f763d5c0a 100644 --- a/infrastructure/terraform/components/dl/README.md +++ b/infrastructure/terraform/components/dl/README.md @@ -50,8 +50,10 @@ No requirements. | [pdm\_mock](#module\_pdm\_mock) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [pdm\_poll](#module\_pdm\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [pdm\_uploader](#module\_pdm\_uploader) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | +| [print\_analyser](#module\_print\_analyser) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [print\_status\_handler](#module\_print\_status\_handler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [s3bucket\_cf\_logs](#module\_s3bucket\_cf\_logs) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | +| [s3bucket\_file\_safe](#module\_s3bucket\_file\_safe) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | | [s3bucket\_letters](#module\_s3bucket\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | | [s3bucket\_non\_pii\_data](#module\_s3bucket\_non\_pii\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | | [s3bucket\_pii\_data](#module\_s3bucket\_pii\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | @@ -61,6 +63,7 @@ No requirements. | [sqs\_mesh\_download](#module\_sqs\_mesh\_download) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_pdm\_poll](#module\_sqs\_pdm\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_pdm\_uploader](#module\_sqs\_pdm\_uploader) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | +| [sqs\_print\_analyser](#module\_sqs\_print\_analyser) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | | [sqs\_print\_status\_handler](#module\_sqs\_print\_status\_handler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | | [sqs\_ttl](#module\_sqs\_ttl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_ttl\_handle\_expiry\_errors](#module\_sqs\_ttl\_handle\_expiry\_errors) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | diff --git a/infrastructure/terraform/components/dl/cloudwatch_event_rule_file_safe.tf b/infrastructure/terraform/components/dl/cloudwatch_event_rule_file_safe.tf new file mode 100644 index 000000000..c10c14f4a --- /dev/null +++ b/infrastructure/terraform/components/dl/cloudwatch_event_rule_file_safe.tf @@ -0,0 +1,19 @@ +resource "aws_cloudwatch_event_rule" "file_safe" { + name = "${local.csi}-file-safe" + description = "File safe event rule" + event_bus_name = aws_cloudwatch_event_bus.main.name + + event_pattern = jsonencode({ + "detail" : { + "type" : [ + "uk.nhs.notify.digital.letters.print.file.safe.v1" + ] + } + }) +} + +resource "aws_cloudwatch_event_target" "file_safe_print_analyser" { + rule = aws_cloudwatch_event_rule.file_safe.name + arn = module.sqs_print_analyser.sqs_queue_arn + event_bus_name = aws_cloudwatch_event_bus.main.name +} diff --git a/infrastructure/terraform/components/dl/lambda_event_source_mapping_print_analyser.tf b/infrastructure/terraform/components/dl/lambda_event_source_mapping_print_analyser.tf new file mode 100644 index 000000000..49287ef82 --- /dev/null +++ b/infrastructure/terraform/components/dl/lambda_event_source_mapping_print_analyser.tf @@ -0,0 +1,10 @@ +resource "aws_lambda_event_source_mapping" "print_analyser" { + event_source_arn = module.sqs_print_analyser.sqs_queue_arn + function_name = module.print_analyser.function_name + batch_size = var.queue_batch_size + maximum_batching_window_in_seconds = var.queue_batch_window_seconds + + function_response_types = [ + "ReportBatchItemFailures" + ] +} diff --git a/infrastructure/terraform/components/dl/module_lambda_print_analyser.tf b/infrastructure/terraform/components/dl/module_lambda_print_analyser.tf new file mode 100644 index 000000000..fde104c04 --- /dev/null +++ b/infrastructure/terraform/components/dl/module_lambda_print_analyser.tf @@ -0,0 +1,99 @@ +module "print_analyser" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip" + + function_name = "print-analyser" + description = "A function for processing file safe events" + + aws_account_id = var.aws_account_id + component = local.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.print_analyser.json + } + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "print-analyser/dist" + function_include_common = true + handler_function_name = "handler" + runtime = "nodejs22.x" + memory = 128 + timeout = 60 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.log_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = { + "EVENT_PUBLISHER_EVENT_BUS_ARN" = aws_cloudwatch_event_bus.main.arn + "EVENT_PUBLISHER_DLQ_URL" = module.sqs_event_publisher_errors.sqs_queue_url + } +} + +data "aws_iam_policy_document" "print_analyser" { + statement { + sid = "PutEvents" + effect = "Allow" + + actions = [ + "events:PutEvents", + ] + + resources = [ + aws_cloudwatch_event_bus.main.arn, + ] + } + + statement { + sid = "SQSPermissionsDLQs" + effect = "Allow" + + actions = [ + "sqs:SendMessage", + "sqs:SendMessageBatch", + ] + + resources = [ + module.sqs_event_publisher_errors.sqs_queue_arn, + ] + } + + statement { + sid = "SQSPermissionsPrintAnalyserQueue" + effect = "Allow" + + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + ] + + resources = [ + module.sqs_print_analyser.sqs_queue_arn, + ] + } + + statement { + sid = "S3PermissionsPrintAnalyserQueue" + effect = "Allow" + + actions = [ + "s3:GetObject", + ] + + resources = [ + "${module.s3bucket_file_safe.arn}/*", + ] + } +} diff --git a/infrastructure/terraform/components/dl/module_s3bucket_file_safe.tf b/infrastructure/terraform/components/dl/module_s3bucket_file_safe.tf new file mode 100644 index 000000000..4fabd0cc1 --- /dev/null +++ b/infrastructure/terraform/components/dl/module_s3bucket_file_safe.tf @@ -0,0 +1,84 @@ +module "s3bucket_file_safe" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip" + + name = "file-safe" + + aws_account_id = var.aws_account_id + region = var.region + project = var.project + environment = var.environment + component = local.component + + kms_key_arn = module.kms.key_arn + + policy_documents = [data.aws_iam_policy_document.s3bucket_file_safe.json] + + force_destroy = var.force_destroy + + lifecycle_rules = [ + { + enabled = true + + expiration = { + days = "90" + } + + noncurrent_version_transition = [ + { + noncurrent_days = "30" + storage_class = "STANDARD_IA" + } + ] + + noncurrent_version_expiration = { + noncurrent_days = "90" + } + + abort_incomplete_multipart_upload = { + days = "1" + } + } + ] +} + +data "aws_iam_policy_document" "s3bucket_file_safe" { + statement { + sid = "AllowManagedAccountsToList" + effect = "Allow" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + module.s3bucket_file_safe.arn, + ] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${var.aws_account_id}:root" + ] + } + } + + statement { + sid = "AllowManagedAccountsToGet" + effect = "Allow" + + actions = [ + "s3:GetObject", + ] + + resources = [ + "${module.s3bucket_file_safe.arn}/*", + ] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${var.aws_account_id}:root" + ] + } + } +} diff --git a/infrastructure/terraform/components/dl/module_sqs_print_analyser.tf b/infrastructure/terraform/components/dl/module_sqs_print_analyser.tf new file mode 100644 index 000000000..2f1364190 --- /dev/null +++ b/infrastructure/terraform/components/dl/module_sqs_print_analyser.tf @@ -0,0 +1,42 @@ +module "sqs_print_analyser" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip" + + aws_account_id = var.aws_account_id + component = local.component + environment = var.environment + project = var.project + region = var.region + name = "print-analyser" + sqs_kms_key_arn = module.kms.key_arn + visibility_timeout_seconds = 60 + delay_seconds = 5 + create_dlq = true + max_receive_count = 1 + sqs_policy_overload = data.aws_iam_policy_document.sqs_print_analyser.json +} + +data "aws_iam_policy_document" "sqs_print_analyser" { + statement { + sid = "AllowEventBridgeToSendMessage" + effect = "Allow" + + principals { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + + actions = [ + "sqs:SendMessage" + ] + + resources = [ + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-print-analyser-queue" + ] + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_cloudwatch_event_rule.file_safe.arn] + } + } +} diff --git a/lambdas/print-analyser/jest.config.ts b/lambdas/print-analyser/jest.config.ts new file mode 100644 index 000000000..c02601aec --- /dev/null +++ b/lambdas/print-analyser/jest.config.ts @@ -0,0 +1,5 @@ +import { baseJestConfig } from '../../jest.config.base'; + +const config = baseJestConfig; + +export default config; diff --git a/lambdas/print-analyser/package.json b/lambdas/print-analyser/package.json new file mode 100644 index 000000000..fd5650b4d --- /dev/null +++ b/lambdas/print-analyser/package.json @@ -0,0 +1,26 @@ +{ + "dependencies": { + "aws-lambda": "^1.0.7", + "digital-letters-events": "^0.0.1", + "pdf-lib": "^1.17.1", + "utils": "^0.0.1" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.2", + "@types/aws-lambda": "^8.10.155", + "@types/jest": "^29.5.14", + "jest": "^29.7.0", + "jest-mock-extended": "^3.0.7", + "typescript": "^5.9.3" + }, + "name": "nhs-notify-digital-letters-print-analyser", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/print-analyser/src/__tests__/apis/sqs-handler.test.ts b/lambdas/print-analyser/src/__tests__/apis/sqs-handler.test.ts new file mode 100644 index 000000000..442a53129 --- /dev/null +++ b/lambdas/print-analyser/src/__tests__/apis/sqs-handler.test.ts @@ -0,0 +1,207 @@ +import { mock } from 'jest-mock-extended'; +import { randomUUID } from 'node:crypto'; +import { createHandler } from 'apis/sqs-handler'; +import { EventPublisher, Logger } from 'utils'; +import { fileSafeEvent, fivePagePdf, recordEvent } from '__tests__/test-data'; +import { FileSafe } from 'digital-letters-events'; + +const logger = mock(); +const eventPublisher = mock(); + +jest.mock('node:crypto', () => ({ + ...jest.requireActual('node:crypto'), + randomUUID: jest.fn(), +})); + +const mockGetS3ObjectBufferFromUri = jest.fn(); +jest.mock('utils', () => ({ + ...jest.requireActual('utils'), + getS3ObjectBufferFromUri: (...args: any[]) => + mockGetS3ObjectBufferFromUri(...args), +})); + +const mockRandomUUID = randomUUID as jest.MockedFunction; +const mockDate = jest.spyOn(Date.prototype, 'toISOString'); +mockRandomUUID.mockReturnValue('550e8400-e29b-41d4-a716-446655440001'); +mockDate.mockReturnValue('2023-06-20T12:00:00.250Z'); + +const handler = createHandler({ + eventPublisher, + logger, +}); + +describe('SQS Handler', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('file safe', () => { + it('should send pdf.analysed event when file.safe received', async () => { + const testPdf = fivePagePdf(); + mockGetS3ObjectBufferFromUri.mockResolvedValue(testPdf); + + const response = await handler(recordEvent([fileSafeEvent])); + + expect(mockGetS3ObjectBufferFromUri).toHaveBeenCalledWith( + fileSafeEvent.data.letterUri, + ); + expect(eventPublisher.sendEvents).toHaveBeenCalledWith( + [ + { + ...fileSafeEvent, + id: '550e8400-e29b-41d4-a716-446655440001', + time: '2023-06-20T12:00:00.250Z', + recordedtime: '2023-06-20T12:00:00.250Z', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-print-pdf-analysed-data.schema.json', + type: 'uk.nhs.notify.digital.letters.print.pdf.analysed.v1', + source: + '/nhs/england/notify/production/primary/data-plane/digitalletters/print', + data: { + senderId: fileSafeEvent.data.senderId, + messageReference: fileSafeEvent.data.messageReference, + letterUri: fileSafeEvent.data.letterUri, + pageCount: 5, + sha256Hash: + '631b6ef1a936e62277d55a80deb850babdde861152d476489d75b0c9161bd326', + createdAt: fileSafeEvent.data.createdAt, + }, + }, + ], + expect.any(Function), + ); + expect(logger.info).toHaveBeenCalledWith( + 'Received SQS Event of 1 record(s)', + ); + expect(logger.info).toHaveBeenCalledWith( + '1 of 1 records processed successfully', + ); + expect(response).toEqual({ batchItemFailures: [] }); + }); + }); + + describe('errors', () => { + it('should return failed SQS records to the queue if an error occurs while processing them', async () => { + const event = recordEvent([fileSafeEvent]); + event.Records[0].body = 'not-json'; + + const result = await handler(event); + + expect(logger.warn).toHaveBeenCalledWith({ + err: new SyntaxError( + `Unexpected token 'o', "not-json" is not valid JSON`, + ), + description: 'Error parsing SQS record', + }); + + expect(logger.info).toHaveBeenCalledWith( + '0 of 1 records processed successfully', + ); + + expect(result).toEqual({ + batchItemFailures: [{ itemIdentifier: '1' }], + }); + }); + + it('should return failed items to the queue if a mildly invalid file.safe event is received', async () => { + const invalidFileSafeEvent = { + ...fileSafeEvent, + source: 'invalid file.safe source', + }; + const event = recordEvent([invalidFileSafeEvent]); + + const result = await handler(event); + + expect(logger.warn).toHaveBeenCalledWith({ + err: expect.arrayContaining([ + expect.objectContaining({ + instancePath: '/source', + }), + ]), + description: 'Error parsing print analyser queue entry', + messageReference: invalidFileSafeEvent.data.messageReference, + }); + + expect(logger.info).toHaveBeenCalledWith( + '0 of 1 records processed successfully', + ); + + expect(result).toEqual({ + batchItemFailures: [{ itemIdentifier: '1' }], + }); + }); + + it('should return failed items to the queue if a very invalid file.safe event is received', async () => { + const invalidFileSafeEvent = {} as FileSafe; + const event = recordEvent([invalidFileSafeEvent]); + + const result = await handler(event); + + expect(logger.warn).toHaveBeenCalledWith({ + err: expect.arrayContaining([ + expect.objectContaining({ + message: `must have required property 'specversion'`, + }), + ]), + description: 'Error parsing print analyser queue entry', + messageReference: 'not present', + }); + + expect(logger.info).toHaveBeenCalledWith( + '0 of 1 records processed successfully', + ); + + expect(result).toEqual({ + batchItemFailures: [{ itemIdentifier: '1' }], + }); + }); + + it('should return failed items to the queue if event transformation fails', async () => { + const testPdf = fivePagePdf(); + mockGetS3ObjectBufferFromUri.mockResolvedValue(testPdf); + + mockRandomUUID.mockImplementationOnce(() => { + throw new Error('A forced error scenario'); + }); + + const event = recordEvent([fileSafeEvent]); + const result = await handler(event); + + expect(logger.warn).toHaveBeenCalledWith({ + err: 'A forced error scenario', + description: 'Failed processing message', + }); + + expect(logger.info).toHaveBeenCalledWith( + '0 of 1 records processed successfully', + ); + + expect(result).toEqual({ + batchItemFailures: [{ itemIdentifier: '1' }], + }); + }); + + it('should return failed items to the queue if PDF analysis fails', async () => { + mockGetS3ObjectBufferFromUri.mockRejectedValue( + new Error('S3 GetObject failed'), + ); + + const event = recordEvent([fileSafeEvent]); + + const result = await handler(event); + + expect(logger.warn).toHaveBeenCalledWith({ + err: 'S3 GetObject failed', + description: 'Failed processing message', + }); + + expect(logger.info).toHaveBeenCalledWith( + '0 of 1 records processed successfully', + ); + + expect(result).toEqual({ + batchItemFailures: [{ itemIdentifier: '1' }], + }); + }); + }); +}); diff --git a/lambdas/print-analyser/src/__tests__/container.test.ts b/lambdas/print-analyser/src/__tests__/container.test.ts new file mode 100644 index 000000000..64f1a694d --- /dev/null +++ b/lambdas/print-analyser/src/__tests__/container.test.ts @@ -0,0 +1,22 @@ +import { createContainer } from 'container'; + +jest.mock('infra/config', () => ({ + loadConfig: jest.fn(() => ({ + eventPublisherDlqUrl: 'test-url', + eventPublisherEventBusArn: 'test-arn', + })), +})); + +jest.mock('utils', () => ({ + eventBridgeClient: {}, + EventPublisher: jest.fn(() => ({})), + logger: {}, + sqsClient: {}, +})); + +describe('container', () => { + it('should create container', () => { + const container = createContainer(); + expect(container).toBeDefined(); + }); +}); diff --git a/lambdas/print-analyser/src/__tests__/five-page.pdf b/lambdas/print-analyser/src/__tests__/five-page.pdf new file mode 100644 index 000000000..1b0257e6a Binary files /dev/null and b/lambdas/print-analyser/src/__tests__/five-page.pdf differ diff --git a/lambdas/print-analyser/src/__tests__/index.test.ts b/lambdas/print-analyser/src/__tests__/index.test.ts new file mode 100644 index 000000000..b5465321a --- /dev/null +++ b/lambdas/print-analyser/src/__tests__/index.test.ts @@ -0,0 +1,15 @@ +import { handler } from 'index'; + +jest.mock('apis/sqs-handler', () => ({ + createHandler: jest.fn(() => jest.fn()), +})); + +jest.mock('container', () => ({ + createContainer: jest.fn(() => ({})), +})); + +describe('index', () => { + it('should export handler', () => { + expect(handler).toBeDefined(); + }); +}); diff --git a/lambdas/print-analyser/src/__tests__/infra/config.test.ts b/lambdas/print-analyser/src/__tests__/infra/config.test.ts new file mode 100644 index 000000000..2902c80f9 --- /dev/null +++ b/lambdas/print-analyser/src/__tests__/infra/config.test.ts @@ -0,0 +1,15 @@ +import { loadConfig } from 'infra/config'; + +jest.mock('utils', () => ({ + defaultConfigReader: { + getValue: jest.fn(), + getInt: jest.fn(), + }, +})); + +describe('config', () => { + it('should load config', () => { + const config = loadConfig(); + expect(config).toBeDefined(); + }); +}); diff --git a/lambdas/print-analyser/src/__tests__/test-data.ts b/lambdas/print-analyser/src/__tests__/test-data.ts new file mode 100644 index 000000000..420d7a33a --- /dev/null +++ b/lambdas/print-analyser/src/__tests__/test-data.ts @@ -0,0 +1,60 @@ +import { SQSEvent, SQSRecord } from 'aws-lambda'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { FileSafe } from 'digital-letters-events'; + +export const fileSafeEvent: FileSafe = { + id: '550e8400-e29b-41d4-a716-446655440001', + specversion: '1.0', + source: + '/nhs/england/notify/production/primary/data-plane/digitalletters/print', + subject: + 'letter-origin/digital-letters/letter/f47ac10b-58cc-4372-a567-0e02b2c3d479', + type: 'uk.nhs.notify.digital.letters.print.file.safe.v1', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-print-file-safe-data.schema.json', + time: '2023-06-20T12:00:00Z', + recordedtime: '2023-06-20T12:00:00.250Z', + severitynumber: 2, + traceparent: '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', + datacontenttype: 'application/json', + severitytext: 'INFO', + data: { + messageReference: 'ref1', + senderId: 'sender1', + letterUri: 'uri1', + createdAt: '2023-06-20T12:00:00.250Z', + }, +}; + +const busEvent = { + version: '0', + id: 'ab07d406-0797-e919-ff9b-3ad9c5498114', +}; + +const sqsRecord = { + messageId: '1', + receiptHandle: 'abc', + attributes: { + ApproximateReceiveCount: '1', + SentTimestamp: '2025-07-03T14:23:30Z', + SenderId: 'sender-id', + ApproximateFirstReceiveTimestamp: '2025-07-03T14:23:30Z', + }, + messageAttributes: {}, + md5OfBody: '', + eventSource: 'aws:sqs', + eventSourceARN: '', + awsRegion: '', +} as SQSRecord; + +export const recordEvent = (events: FileSafe[]): SQSEvent => ({ + Records: events.map((event, i) => ({ + ...sqsRecord, + messageId: String(i + 1), + body: JSON.stringify({ ...busEvent, detail: event }), + })), +}); + +export const fivePagePdf = () => + readFileSync(path.join(__dirname, 'five-page.pdf')); diff --git a/lambdas/print-analyser/src/apis/sqs-handler.ts b/lambdas/print-analyser/src/apis/sqs-handler.ts new file mode 100644 index 000000000..54bf51502 --- /dev/null +++ b/lambdas/print-analyser/src/apis/sqs-handler.ts @@ -0,0 +1,144 @@ +import type { + SQSBatchItemFailure, + SQSBatchResponse, + SQSEvent, +} from 'aws-lambda'; +import { createHash, randomUUID } from 'node:crypto'; +import { PDFDocument } from 'pdf-lib'; +import { FileSafe, PDFAnalysed } from 'digital-letters-events'; +import fileSafeValidator from 'digital-letters-events/FileSafe.js'; +import pdfAnalysedValidator from 'digital-letters-events/PDFAnalysed.js'; +import { EventPublisher, Logger, getS3ObjectBufferFromUri } from 'utils'; + +export interface HandlerDependencies { + eventPublisher: EventPublisher; + logger: Logger; +} + +type ValidatedRecord = { + messageId: string; + event: FileSafe; +}; + +type PdfInfo = { + pageCount: number; + hash: string; +}; + +function validateRecord( + { body, messageId }: { body: string; messageId: string }, + logger: Logger, +): ValidatedRecord | null { + try { + const sqsEventBody = JSON.parse(body); + const sqsEventDetail = sqsEventBody.detail; + + const isEventValid = fileSafeValidator(sqsEventDetail); + if (!isEventValid) { + logger.warn({ + err: fileSafeValidator.errors, + description: 'Error parsing print analyser queue entry', + messageReference: + sqsEventDetail?.data?.messageReference || 'not present', + }); + + return null; + } + + return { messageId, event: sqsEventDetail }; + } catch (error) { + logger.warn({ + err: error, + description: 'Error parsing SQS record', + }); + + return null; + } +} + +function generateUpdatedEvent(event: FileSafe, pdfInfo: PdfInfo): PDFAnalysed { + const eventTime = new Date().toISOString(); + + const { + data: { createdAt, letterUri, messageReference, senderId }, + } = event; + + return { + ...event, + id: randomUUID(), + time: eventTime, + recordedtime: eventTime, + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-print-pdf-analysed-data.schema.json', + type: 'uk.nhs.notify.digital.letters.print.pdf.analysed.v1', + // NOTE: CCM-13892 Generate event digital letters source property from scratch + source: + '/nhs/england/notify/production/primary/data-plane/digitalletters/print', + data: { + senderId, + messageReference, + letterUri, + pageCount: pdfInfo.pageCount, + sha256Hash: pdfInfo.hash, + createdAt, + }, + }; +} + +async function analysePdf(pdf: Buffer): Promise { + const doc = await PDFDocument.load(pdf); + const pageCount = doc.getPageCount(); + const hash = createHash('sha256').update(pdf).digest('hex'); + + return { pageCount, hash }; +} + +export const createHandler = ({ + eventPublisher, + logger, +}: HandlerDependencies) => + async function handler(sqsEvent: SQSEvent): Promise { + const receivedItemCount = sqsEvent.Records.length; + const batchItemFailures: SQSBatchItemFailure[] = []; + const validatedRecords: ValidatedRecord[] = []; + const validEvents: PDFAnalysed[] = []; + + logger.info(`Received SQS Event of ${receivedItemCount} record(s)`); + + for (const record of sqsEvent.Records) { + const validated = validateRecord(record, logger); + if (validated) { + validatedRecords.push(validated); + } else { + batchItemFailures.push({ itemIdentifier: record.messageId }); + } + } + + await Promise.all( + validatedRecords.map(async (validatedRecord: ValidatedRecord) => { + try { + const { event } = validatedRecord; + const pdfBuffer = await getS3ObjectBufferFromUri( + event.data.letterUri, + ); + const pdfInfo = await analysePdf(pdfBuffer); + validEvents.push(generateUpdatedEvent(event, pdfInfo)); + } catch (error: any) { + logger.warn({ + err: error.message, + description: 'Failed processing message', + }); + batchItemFailures.push({ itemIdentifier: validatedRecord.messageId }); + } + }), + ); + + await eventPublisher.sendEvents(validEvents, pdfAnalysedValidator); + + const processedItemCount = receivedItemCount - batchItemFailures.length; + logger.info( + `${processedItemCount} of ${receivedItemCount} records processed successfully`, + ); + + return { batchItemFailures }; + }; diff --git a/lambdas/print-analyser/src/container.ts b/lambdas/print-analyser/src/container.ts new file mode 100644 index 000000000..7fe49378d --- /dev/null +++ b/lambdas/print-analyser/src/container.ts @@ -0,0 +1,19 @@ +import { HandlerDependencies } from 'apis/sqs-handler'; +import { loadConfig } from 'infra/config'; +import { EventPublisher, eventBridgeClient, logger, sqsClient } from 'utils'; + +export const createContainer = (): HandlerDependencies => { + const { eventPublisherDlqUrl, eventPublisherEventBusArn } = loadConfig(); + + const eventPublisher = new EventPublisher({ + eventBusArn: eventPublisherEventBusArn, + dlqUrl: eventPublisherDlqUrl, + logger, + sqsClient, + eventBridgeClient, + }); + + return { eventPublisher, logger }; +}; + +export default createContainer; diff --git a/lambdas/print-analyser/src/index.ts b/lambdas/print-analyser/src/index.ts new file mode 100644 index 000000000..f25a80861 --- /dev/null +++ b/lambdas/print-analyser/src/index.ts @@ -0,0 +1,6 @@ +import { createHandler } from 'apis/sqs-handler'; +import { createContainer } from 'container'; + +export const handler = createHandler(createContainer()); + +export default handler; diff --git a/lambdas/print-analyser/src/infra/config.ts b/lambdas/print-analyser/src/infra/config.ts new file mode 100644 index 000000000..855e66108 --- /dev/null +++ b/lambdas/print-analyser/src/infra/config.ts @@ -0,0 +1,17 @@ +import { defaultConfigReader } from 'utils'; + +export type Config = { + eventPublisherEventBusArn: string; + eventPublisherDlqUrl: string; +}; + +export function loadConfig(): Config { + return { + eventPublisherEventBusArn: defaultConfigReader.getValue( + 'EVENT_PUBLISHER_EVENT_BUS_ARN', + ), + eventPublisherDlqUrl: defaultConfigReader.getValue( + 'EVENT_PUBLISHER_DLQ_URL', + ), + }; +} diff --git a/lambdas/print-analyser/tsconfig.json b/lambdas/print-analyser/tsconfig.json new file mode 100644 index 000000000..f7bcaa1ff --- /dev/null +++ b/lambdas/print-analyser/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "baseUrl": "./src/", + "isolatedModules": true + }, + "extends": "@tsconfig/node22/tsconfig.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/lambdas/ttl-create-lambda/src/apis/sqs-trigger-lambda.ts b/lambdas/ttl-create-lambda/src/apis/sqs-trigger-lambda.ts index d18219c74..5c9a91ab5 100644 --- a/lambdas/ttl-create-lambda/src/apis/sqs-trigger-lambda.ts +++ b/lambdas/ttl-create-lambda/src/apis/sqs-trigger-lambda.ts @@ -107,6 +107,11 @@ export const createHandler = ({ dataschema: 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-queue-item-enqueued-data.schema.json', source: event.source.replace(/\/mesh$/, '/queue'), + data: { + messageReference: event.data.messageReference, + senderId: event.data.senderId, + messageUri: event.data.messageUri, + }, })), itemEnqueuedValidator, ); diff --git a/package-lock.json b/package-lock.json index 8bb825262..af9f46e08 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "lambdas/pdm-uploader-lambda", "lambdas/core-notifier-lambda", "lambdas/print-status-handler", + "lambdas/print-analyser", "utils/utils", "utils/sender-management", "src/cloudevents", @@ -426,6 +427,77 @@ } } }, + "lambdas/print-analyser": { + "name": "nhs-notify-digital-letters-print-analyser", + "version": "0.0.1", + "dependencies": { + "aws-lambda": "^1.0.7", + "digital-letters-events": "^0.0.1", + "pdf-lib": "^1.17.1", + "utils": "^0.0.1" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.2", + "@types/aws-lambda": "^8.10.155", + "@types/jest": "^29.5.14", + "jest": "^29.7.0", + "jest-mock-extended": "^3.0.7", + "typescript": "^5.9.3" + } + }, + "lambdas/print-analyser/node_modules/@types/jest": { + "version": "29.5.14", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", + "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.0.0", + "pretty-format": "^29.0.0" + } + }, + "lambdas/print-analyser/node_modules/jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "lambdas/print-analyser/node_modules/jest-mock-extended": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/jest-mock-extended/-/jest-mock-extended-3.0.7.tgz", + "integrity": "sha512-7lsKdLFcW9B9l5NzZ66S/yTQ9k8rFtnwYdCNuRU/81fqDWicNDVhitTSPnrGmNeNm0xyw0JHexEOShrIKRCIRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ts-essentials": "^10.0.0" + }, + "peerDependencies": { + "jest": "^24.0.0 || ^25.0.0 || ^26.0.0 || ^27.0.0 || ^28.0.0 || ^29.0.0", + "typescript": "^3.0.0 || ^4.0.0 || ^5.0.0" + } + }, "lambdas/print-sender-lambda": { "extraneous": true }, @@ -4068,6 +4140,36 @@ "node": ">=12.4.0" } }, + "node_modules/@pdf-lib/standard-fonts": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@pdf-lib/standard-fonts/-/standard-fonts-1.0.0.tgz", + "integrity": "sha512-hU30BK9IUN/su0Mn9VdlVKsWBS6GyhVfqjwl1FjZN4TxP6cCw0jP2w7V3Hf5uX7M0AZJ16vey9yE0ny7Sa59ZA==", + "license": "MIT", + "dependencies": { + "pako": "^1.0.6" + } + }, + "node_modules/@pdf-lib/standard-fonts/node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/@pdf-lib/upng": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@pdf-lib/upng/-/upng-1.0.1.tgz", + "integrity": "sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==", + "license": "MIT", + "dependencies": { + "pako": "^1.0.10" + } + }, + "node_modules/@pdf-lib/upng/node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "dev": true, @@ -12474,6 +12576,10 @@ "resolved": "lambdas/print-status-handler", "link": true }, + "node_modules/nhs-notify-digital-letters-print-analyser": { + "resolved": "lambdas/print-analyser", + "link": true + }, "node_modules/nhs-notify-digital-letters-ttl-create-lambda": { "resolved": "lambdas/ttl-create-lambda", "link": true @@ -12951,6 +13057,30 @@ "url": "https://opencollective.com/express" } }, + "node_modules/pdf-lib": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/pdf-lib/-/pdf-lib-1.17.1.tgz", + "integrity": "sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==", + "license": "MIT", + "dependencies": { + "@pdf-lib/standard-fonts": "^1.0.0", + "@pdf-lib/upng": "^1.0.1", + "pako": "^1.0.11", + "tslib": "^1.11.1" + } + }, + "node_modules/pdf-lib/node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/pdf-lib/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "license": "0BSD" + }, "node_modules/picocolors": { "version": "1.1.1", "dev": true, diff --git a/package.json b/package.json index 125f8fb2c..84f4f8a14 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "lambdas/pdm-uploader-lambda", "lambdas/core-notifier-lambda", "lambdas/print-status-handler", + "lambdas/print-analyser", "utils/utils", "utils/sender-management", "src/cloudevents", diff --git a/src/cloudevents/domains/digital-letters/2025-10-draft/data/digital-letters-print-file-safe-data.schema.yaml b/src/cloudevents/domains/digital-letters/2025-10-draft/data/digital-letters-print-file-safe-data.schema.yaml index 23cc96890..7cf833e23 100644 --- a/src/cloudevents/domains/digital-letters/2025-10-draft/data/digital-letters-print-file-safe-data.schema.yaml +++ b/src/cloudevents/domains/digital-letters/2025-10-draft/data/digital-letters-print-file-safe-data.schema.yaml @@ -10,7 +10,10 @@ properties: $ref: ../defs/requests.schema.yaml#/properties/senderId letterUri: $ref: ../defs/print.schema.yaml#/properties/letterUri + createdAt: + $ref: ../defs/print.schema.yaml#/properties/createdAt required: - messageReference - senderId - letterUri + - createdAt diff --git a/tests/playwright/constants/backend-constants.ts b/tests/playwright/constants/backend-constants.ts index ebe9c1f9c..d9d789130 100644 --- a/tests/playwright/constants/backend-constants.ts +++ b/tests/playwright/constants/backend-constants.ts @@ -21,6 +21,7 @@ export const PDM_POLL_DLQ_NAME = `${CSI}-pdm-poll-dlq`; export const CORE_NOTIFIER_DLQ_NAME = `${CSI}-core-notifier-dlq`; export const PRINT_STATUS_HANDLER_DLQ_NAME = `${CSI}-print-status-handler-dlq`; export const HANDLE_TTL_DLQ_NAME = `${CSI}-ttl-handle-expiry-errors-queue`; +export const PRINT_ANALYSER_DLQ_NAME = `${CSI}-print-analyser-dlq`; // Queue Url Prefix export const SQS_URL_PREFIX = `https://sqs.${REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/`; @@ -35,9 +36,11 @@ export const TTL_TABLE_NAME = `${CSI}-ttl`; // S3 export const LETTERS_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-letters`; +export const FILE_SAFE_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-file-safe`; // Cloudwatch export const PDM_UPLOADER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-pdm-uploader`; export const PDM_POLL_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-pdm-poll`; export const CORE_NOTIFIER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-core-notifier`; export const PRINT_STATUS_HANDLER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-status-handler`; +export const PRINT_ANALYSER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-analyser`; diff --git a/tests/playwright/digital-letters-component-tests/print-analyser.component.spec.ts b/tests/playwright/digital-letters-component-tests/print-analyser.component.spec.ts new file mode 100644 index 000000000..ac9060f4c --- /dev/null +++ b/tests/playwright/digital-letters-component-tests/print-analyser.component.spec.ts @@ -0,0 +1,121 @@ +import { expect, test } from '@playwright/test'; +import { + ENV, + FILE_SAFE_S3_BUCKET_NAME, + PRINT_ANALYSER_DLQ_NAME, + PRINT_ANALYSER_LAMBDA_LOG_GROUP_NAME, +} from 'constants/backend-constants'; +import { getLogsFromCloudwatch } from 'helpers/cloudwatch-helpers'; +import eventPublisher from 'helpers/event-bus-helpers'; +import expectToPassEventually from 'helpers/expectations'; +import { fivePagePdf } from 'helpers/pdf-helpers'; +import { v4 as uuidv4 } from 'uuid'; +import fileSafeValidator from 'digital-letters-events/FileSafe.js'; +import { FileSafe } from 'digital-letters-events'; +import { expectMessageContainingString, purgeQueue } from 'helpers/sqs-helpers'; +import { putFileS3 } from 'utils'; + +export const fileSafeEvent: FileSafe = { + id: '550e8400-e29b-41d4-a716-446655440001', + specversion: '1.0', + source: + '/nhs/england/notify/production/primary/data-plane/digitalletters/print', + subject: + 'letter-origin/digital-letters/letter/f47ac10b-58cc-4372-a567-0e02b2c3d479', + type: 'uk.nhs.notify.digital.letters.print.file.safe.v1', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-print-file-safe-data.schema.json', + time: '2023-06-20T12:00:00Z', + recordedtime: '2023-06-20T12:00:00.250Z', + severitynumber: 2, + traceparent: '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', + datacontenttype: 'application/json', + severitytext: 'INFO', + data: { + messageReference: 'ref1', + senderId: 'sender1', + letterUri: 'uri1', + createdAt: '2023-06-20T12:00:00.250Z', + }, +}; + +test.describe('Print analyser', () => { + const pdfFilename = `${uuidv4()}.pdf`; + + test.beforeAll(async () => { + test.setTimeout(150_000); + + await purgeQueue(PRINT_ANALYSER_DLQ_NAME); + + await putFileS3(fivePagePdf(), { + Bucket: FILE_SAFE_S3_BUCKET_NAME, + Key: pdfFilename, + }); + }); + + test(`should create pdf.analysed event for a file.safe event`, async () => { + const messageReference = uuidv4(); + const event: FileSafe = { + ...fileSafeEvent, + data: { + ...fileSafeEvent.data, + letterUri: `s3://${FILE_SAFE_S3_BUCKET_NAME}/${pdfFilename}`, + messageReference, + }, + }; + + await eventPublisher.sendEvents([event], fileSafeValidator); + + await expectToPassEventually(async () => { + const eventLogEntry = await getLogsFromCloudwatch( + `/aws/vendedlogs/events/event-bus/nhs-${ENV}-dl`, + [ + '$.message_type = "EVENT_RECEIPT"', + '$.details.detail_type = "uk.nhs.notify.digital.letters.print.pdf.analysed.v1"', + `$.details.event_detail = "*\\"messageReference\\":\\"${messageReference}\\"*"`, + `$.details.event_detail = "*\\"senderId\\":\\"${event.data.senderId}\\"*"`, + `$.details.event_detail = "*\\"letterUri\\":\\"${event.data.letterUri}\\"*"`, + `$.details.event_detail = "*\\"pageCount\\":5*"`, + `$.details.event_detail = "*\\"sha256Hash\\":\\"631b6ef1a936e62277d55a80deb850babdde861152d476489d75b0c9161bd326\\"*"`, + `$.details.event_detail = "*\\"createdAt\\":\\"${event.data.createdAt}\\"*"`, + ], + ); + + expect(eventLogEntry.length).toEqual(1); + }, 120); + }); + + test('should send invalid event to print analyser dlq', async () => { + test.setTimeout(250_000); + + // Send file.safe event with missing data properties + const messageReference = uuidv4(); + const event = { + ...fileSafeEvent, + data: { + messageReference, + }, + } as FileSafe; + + await eventPublisher.sendEvents([event], () => true); + + await expectToPassEventually(async () => { + const eventLogEntry = await getLogsFromCloudwatch( + PRINT_ANALYSER_LAMBDA_LOG_GROUP_NAME, + [ + '$.message.description = "Error parsing print analyser queue entry"', + `$.message.err[0].message = "must have required property 'senderId'"`, + `$.message.messageReference = "${messageReference}"`, + ], + ); + + expect(eventLogEntry.length).toEqual(1); + }, 120); + + await expectMessageContainingString( + PRINT_ANALYSER_DLQ_NAME, + messageReference, + 120, + ); + }); +}); diff --git a/tests/playwright/helpers/five-page.pdf b/tests/playwright/helpers/five-page.pdf new file mode 100644 index 000000000..1b0257e6a Binary files /dev/null and b/tests/playwright/helpers/five-page.pdf differ diff --git a/tests/playwright/helpers/pdf-helpers.ts b/tests/playwright/helpers/pdf-helpers.ts new file mode 100644 index 000000000..aa23a07fc --- /dev/null +++ b/tests/playwright/helpers/pdf-helpers.ts @@ -0,0 +1,5 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; + +export const fivePagePdf = () => + readFileSync(path.join(__dirname, 'five-page.pdf')); diff --git a/utils/utils/src/__tests__/s3-utils/get-object-s3.test.ts b/utils/utils/src/__tests__/s3-utils/get-object-s3.test.ts index 20a03f4a5..d9cce7e03 100644 --- a/utils/utils/src/__tests__/s3-utils/get-object-s3.test.ts +++ b/utils/utils/src/__tests__/s3-utils/get-object-s3.test.ts @@ -1,5 +1,10 @@ import { Readable } from 'node:stream'; -import { getS3Object, getS3ObjectFromUri, s3Client } from '../../s3-utils'; +import { + getS3Object, + getS3ObjectBufferFromUri, + getS3ObjectFromUri, + s3Client, +} from '../../s3-utils'; describe('getS3Object', () => { afterEach(jest.resetAllMocks); @@ -86,6 +91,7 @@ describe('getS3Object', () => { expect(data).toEqual(defaultValue); }); }); + describe('getS3ObjectFromUri', () => { afterEach(jest.resetAllMocks); @@ -167,3 +173,89 @@ describe('getS3ObjectFromUri', () => { ); }); }); + +describe('getS3ObjectBufferFromUri', () => { + afterEach(jest.resetAllMocks); + + it('Should throw an error for invalid S3 URI format', async () => { + await expect(getS3ObjectBufferFromUri('invalid-uri')).rejects.toThrow( + 'Invalid S3 URI format: invalid-uri', + ); + }); + + it('Should throw an error for S3 URI without bucket', async () => { + await expect(getS3ObjectBufferFromUri('s3://')).rejects.toThrow( + 'Invalid S3 URI format: s3://', + ); + }); + + it('Should throw an error for S3 URI without key', async () => { + await expect(getS3ObjectBufferFromUri('s3://bucket-name/')).rejects.toThrow( + 'Invalid S3 URI format: s3://bucket-name/', + ); + }); + + it('Should parse valid S3 URI and retrieve object', async () => { + const result = JSON.stringify({ + featureFlags: { + testFlag: true, + }, + }); + + s3Client.send = jest + .fn() + .mockReturnValueOnce({ Body: Readable.from([result]) }); + + const uri = 's3://bucket-name/config.test.json'; + const data = await getS3ObjectBufferFromUri(uri); + + expect(s3Client.send).toHaveBeenCalledWith( + expect.objectContaining({ + input: { + Bucket: 'bucket-name', + Key: 'config.test.json', + VersionId: undefined, + }, + }), + ); + + const expectedBuffer = Buffer.from(result); + expect(data).toEqual(expectedBuffer); + }); + + it('Should parse S3 URI with nested path', async () => { + const result = 'test content'; + + s3Client.send = jest + .fn() + .mockReturnValueOnce({ Body: Readable.from([result]) }); + + const uri = 's3://bucket-name/path/to/nested/file.json'; + const data = await getS3ObjectBufferFromUri(uri); + + expect(s3Client.send).toHaveBeenCalledWith( + expect.objectContaining({ + input: { + Bucket: 'bucket-name', + Key: 'path/to/nested/file.json', + VersionId: undefined, + }, + }), + ); + + const expectedBuffer = Buffer.from(result); + expect(data).toEqual(expectedBuffer); + }); + + it('Should throw an error if object not found', async () => { + s3Client.send = jest.fn().mockImplementationOnce(() => { + throw new Error('No file found'); + }); + + await expect( + getS3ObjectBufferFromUri('s3://bucket-name/config.test.json'), + ).rejects.toThrow( + "Could not retrieve from bucket 's3://bucket-name/config.test.json' from S3", + ); + }); +}); diff --git a/utils/utils/src/__tests__/s3-utils/put-data-s3.test.ts b/utils/utils/src/__tests__/s3-utils/put-data-s3.test.ts index 790a61892..1263479ee 100644 --- a/utils/utils/src/__tests__/s3-utils/put-data-s3.test.ts +++ b/utils/utils/src/__tests__/s3-utils/put-data-s3.test.ts @@ -23,6 +23,7 @@ describe('putDataS3', () => { Body: '{\n "value1": "1a",\n "value2": "2a"\n}', }); }); + it('throws an error when there is an issue puts data in S3', async () => { const s3Client = mockClient(S3Client); s3Client.rejectsOnce(new Error('It broke!')); diff --git a/utils/utils/src/__tests__/s3-utils/put-file-s3.test.ts b/utils/utils/src/__tests__/s3-utils/put-file-s3.test.ts new file mode 100644 index 000000000..854574999 --- /dev/null +++ b/utils/utils/src/__tests__/s3-utils/put-file-s3.test.ts @@ -0,0 +1,82 @@ +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { putFileS3 } from '../../s3-utils'; + +describe('putFileS3', () => { + it('puts buffer in S3', async () => { + const s3Client = mockClient(S3Client); + const testBuffer = Buffer.from('test pdf content'); + + await putFileS3(testBuffer, { + Bucket: 'bucket-name', + Key: 'file.pdf', + }); + + expect(s3Client).toHaveReceivedCommandWith(PutObjectCommand, { + Bucket: 'bucket-name', + Key: 'file.pdf', + Body: testBuffer, + Metadata: {}, + }); + }); + + it('puts buffer in S3 with ContentType', async () => { + const s3Client = mockClient(S3Client); + const testBuffer = Buffer.from('test pdf content'); + + await putFileS3( + testBuffer, + { + Bucket: 'bucket-name', + Key: 'file.pdf', + }, + {}, + 'application/pdf', + ); + + expect(s3Client).toHaveReceivedCommandWith(PutObjectCommand, { + Bucket: 'bucket-name', + Key: 'file.pdf', + Body: testBuffer, + Metadata: {}, + ContentType: 'application/pdf', + }); + }); + + it('puts buffer in S3 with metadata', async () => { + const s3Client = mockClient(S3Client); + const testBuffer = Buffer.from('test pdf content'); + + await putFileS3( + testBuffer, + { + Bucket: 'bucket-name', + Key: 'file.pdf', + }, + { 'x-custom-metadata': 'value' }, + ); + + expect(s3Client).toHaveReceivedCommandWith(PutObjectCommand, { + Bucket: 'bucket-name', + Key: 'file.pdf', + Body: testBuffer, + Metadata: { 'x-custom-metadata': 'value' }, + }); + }); + + it('throws an error when there is an issue putting buffer in S3', async () => { + const s3Client = mockClient(S3Client); + s3Client.rejectsOnce(new Error('It broke!')); + const testBuffer = Buffer.from('test pdf content'); + + await expect( + putFileS3(testBuffer, { + Bucket: 'bucket-name', + Key: 'file.pdf', + }), + ).rejects.toThrow( + 'Upload to bucket-name/file.pdf failed, error: Error: It broke!', + ); + }); +}); diff --git a/utils/utils/src/s3-utils/get-object-s3.ts b/utils/utils/src/s3-utils/get-object-s3.ts index 4e35cc86c..9430b4729 100644 --- a/utils/utils/src/s3-utils/get-object-s3.ts +++ b/utils/utils/src/s3-utils/get-object-s3.ts @@ -30,7 +30,7 @@ export interface S3Location { VersionId?: string; } -export async function streamToString(Body: Readable) { +async function streamToString(Body: Readable) { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; Body.on('data', (chunk: ArrayBuffer | SharedArrayBuffer) => @@ -41,6 +41,17 @@ export async function streamToString(Body: Readable) { }); } +async function streamToBuffer(Body: Readable) { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + Body.on('data', (chunk: ArrayBuffer | SharedArrayBuffer) => + chunks.push(Buffer.from(chunk)), + ); + Body.on('error', (err) => reject(err)); + Body.on('end', () => resolve(Buffer.concat(chunks))); + }); +} + export async function getS3ObjectStream( location: S3Location, ): Promise { @@ -93,3 +104,21 @@ export async function getS3ObjectFromUri(uri: string): Promise { const [, Bucket, Key] = match; return getS3Object({ Bucket, Key }); } + +export async function getS3ObjectBufferFromUri(uri: string): Promise { + const regex = /^s3:\/\/([^/]+)\/(.+)$/; + const match = regex.exec(uri); + if (!match) { + throw new Error(`Invalid S3 URI format: ${uri}`); + } + const [, Bucket, Key] = match; + + try { + return await streamToBuffer(await getS3ObjectStream({ Bucket, Key })); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + throw new Error( + `Could not retrieve from bucket 's3://${Bucket}/${Key}' from S3: ${msg}`, + ); + } +} diff --git a/utils/utils/src/s3-utils/index.ts b/utils/utils/src/s3-utils/index.ts index d46b10bb0..867e20cfc 100644 --- a/utils/utils/src/s3-utils/index.ts +++ b/utils/utils/src/s3-utils/index.ts @@ -1,3 +1,4 @@ export * from './get-object-s3'; export * from './s3-client'; export * from './put-data-s3'; +export * from './put-file-s3'; diff --git a/utils/utils/src/s3-utils/put-data-s3.ts b/utils/utils/src/s3-utils/put-data-s3.ts index 5d81c4b9d..5aacbfe06 100644 --- a/utils/utils/src/s3-utils/put-data-s3.ts +++ b/utils/utils/src/s3-utils/put-data-s3.ts @@ -1,4 +1,3 @@ -/* eslint-disable no-console */ import { PutObjectCommand, PutObjectCommandOutput } from '@aws-sdk/client-s3'; import type { S3Location } from './get-object-s3'; import { s3Client } from './s3-client'; @@ -17,7 +16,7 @@ export async function putDataS3( }; const data = await s3Client.send(new PutObjectCommand(params)); - console.log(`Data uploaded to ${Bucket}/${Key}`); + return data; } catch (error) { throw new Error(`Upload to ${Bucket}/${Key} failed, error: ${error}`); diff --git a/utils/utils/src/s3-utils/put-file-s3.ts b/utils/utils/src/s3-utils/put-file-s3.ts new file mode 100644 index 000000000..d60a2f1be --- /dev/null +++ b/utils/utils/src/s3-utils/put-file-s3.ts @@ -0,0 +1,26 @@ +import { PutObjectCommand, PutObjectCommandOutput } from '@aws-sdk/client-s3'; +import type { S3Location } from './get-object-s3'; +import { s3Client } from './s3-client'; + +export async function putFileS3( + buffer: Buffer, + { Bucket, Key }: S3Location, + Metadata: Record = {}, + ContentType?: string, +): Promise { + try { + const params = { + Bucket, + Key, + Body: buffer, + Metadata, + ...(ContentType && { ContentType }), + }; + + const data = await s3Client.send(new PutObjectCommand(params)); + + return data; + } catch (error) { + throw new Error(`Upload to ${Bucket}/${Key} failed, error: ${error}`); + } +}