From 0fb43e455ecf9a0032e59a0c9ff1212869ddaa1c Mon Sep 17 00:00:00 2001 From: Thomas-Boyle Date: Tue, 7 Apr 2026 10:54:27 +0100 Subject: [PATCH 01/10] Refactor Lambda deployment workflow to optimize image build process. Introduced a build metadata preparation step to check for existing images before building and pushing, enhancing efficiency. Updated ECR login and Docker build steps to conditionally execute based on the presence of existing images, improving deployment reliability. --- .github/workflows/deploy-lambda-artifact.yml | 90 ++++++++++++++------ 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/.github/workflows/deploy-lambda-artifact.yml b/.github/workflows/deploy-lambda-artifact.yml index 57c90b89c..13dbb0e39 100644 --- a/.github/workflows/deploy-lambda-artifact.yml +++ b/.github/workflows/deploy-lambda-artifact.yml @@ -190,16 +190,9 @@ jobs: echo "deployment_mode=${deployment_mode}" >> "$GITHUB_OUTPUT" - - name: Login to Amazon ECR - id: login-ecr - if: ${{ steps.decide.outputs.deployment_mode == 'build' }} - uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 - - - name: Build, publish and emit digest manifest - id: build + - name: Prepare build metadata + id: build-check if: ${{ steps.decide.outputs.deployment_mode == 'build' }} - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} run: | set -euo pipefail @@ -208,10 +201,18 @@ jobs: GIT_TAG="${TAG_PREFIX}git-${SHORT_SHA}" REL_TAG="${TAG_PREFIX}rel-${RELEASE_STAMP}" - IMAGE_URI_GIT="${ECR_REGISTRY}/${ECR_REPOSITORY}:${GIT_TAG}" - IMAGE_URI_REL="${ECR_REGISTRY}/${ECR_REPOSITORY}:${REL_TAG}" + REPOSITORY_URI="$( + aws ecr describe-repositories \ + --repository-names "${ECR_REPOSITORY}" \ + --region "${AWS_REGION}" \ + --query 'repositories[0].repositoryUri' \ + --output text + )" - IMAGE_DIGEST="$( + IMAGE_URI_GIT="${REPOSITORY_URI}:${GIT_TAG}" + IMAGE_URI_REL="${REPOSITORY_URI}:${REL_TAG}" + + EXISTING_IMAGE_DIGEST="$( aws ecr describe-images \ --repository-name "${ECR_REPOSITORY}" \ --region "${AWS_REGION}" \ @@ -220,19 +221,58 @@ jobs: --output text 2>/dev/null || true )" - if [ -z "${IMAGE_DIGEST}" ] || [ "${IMAGE_DIGEST}" = "None" ]; then - docker build -f "${DOCKERFILE_PATH}" -t "${IMAGE_URI_GIT}" -t "${IMAGE_URI_REL}" "${DOCKER_CONTEXT_PATH}" - docker push "${IMAGE_URI_GIT}" - docker push "${IMAGE_URI_REL}" + SHOULD_BUILD="false" + if [ -z "${EXISTING_IMAGE_DIGEST}" ] || [ "${EXISTING_IMAGE_DIGEST}" = "None" ]; then + SHOULD_BUILD="true" + fi - IMAGE_DIGEST="$( - aws ecr describe-images \ - --repository-name "${ECR_REPOSITORY}" \ - --region "${AWS_REGION}" \ - --image-ids imageTag="${GIT_TAG}" \ - --query 'imageDetails[0].imageDigest' \ - --output text - )" + echo "git_tag=${GIT_TAG}" >> "$GITHUB_OUTPUT" + echo "release_tag=${REL_TAG}" >> "$GITHUB_OUTPUT" + echo "repository_uri=${REPOSITORY_URI}" >> "$GITHUB_OUTPUT" + echo "image_uri_git=${IMAGE_URI_GIT}" >> "$GITHUB_OUTPUT" + echo "image_uri_rel=${IMAGE_URI_REL}" >> "$GITHUB_OUTPUT" + echo "existing_image_digest=${EXISTING_IMAGE_DIGEST}" >> "$GITHUB_OUTPUT" + echo "should_build=${SHOULD_BUILD}" >> "$GITHUB_OUTPUT" + + - name: Login to Amazon ECR + id: login-ecr + if: ${{ steps.decide.outputs.deployment_mode == 'build' && steps.build-check.outputs.should_build == 'true' }} + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 + + - name: Set up Docker Buildx + if: ${{ steps.decide.outputs.deployment_mode == 'build' && steps.build-check.outputs.should_build == 'true' }} + uses: docker/setup-buildx-action@v3 + + - name: Build and publish image with layer caching + id: build-image + if: ${{ steps.decide.outputs.deployment_mode == 'build' && steps.build-check.outputs.should_build == 'true' }} + uses: docker/build-push-action@v6 + with: + context: ${{ env.DOCKER_CONTEXT_PATH }} + file: ${{ env.DOCKERFILE_PATH }} + push: true + tags: | + ${{ steps.build-check.outputs.image_uri_git }} + ${{ steps.build-check.outputs.image_uri_rel }} + cache-from: type=gha,scope=${{ env.ECR_REPOSITORY }} + cache-to: type=gha,mode=max,scope=${{ env.ECR_REPOSITORY }} + + - name: Emit build digest manifest + id: build + if: ${{ steps.decide.outputs.deployment_mode == 'build' }} + env: + REPOSITORY_URI: ${{ steps.build-check.outputs.repository_uri }} + GIT_TAG: ${{ steps.build-check.outputs.git_tag }} + REL_TAG: ${{ steps.build-check.outputs.release_tag }} + EXISTING_IMAGE_DIGEST: ${{ steps.build-check.outputs.existing_image_digest }} + SHOULD_BUILD: ${{ steps.build-check.outputs.should_build }} + BUILT_IMAGE_DIGEST: ${{ steps.build-image.outputs.digest }} + run: | + set -euo pipefail + + IMAGE_DIGEST="${EXISTING_IMAGE_DIGEST}" + if [ "${SHOULD_BUILD}" = "true" ]; then + IMAGE_DIGEST="${BUILT_IMAGE_DIGEST}" else echo "Immutable tag '${GIT_TAG}' already exists. Reusing existing image digest." fi @@ -242,7 +282,7 @@ jobs: exit 1 fi - IMAGE_URI_PINNED="${ECR_REGISTRY}/${ECR_REPOSITORY}@${IMAGE_DIGEST}" + IMAGE_URI_PINNED="${REPOSITORY_URI}@${IMAGE_DIGEST}" echo "image_version=${GIT_TAG}" >> "$GITHUB_OUTPUT" echo "image_digest=${IMAGE_DIGEST}" >> "$GITHUB_OUTPUT" echo "image_uri=${IMAGE_URI_PINNED}" >> "$GITHUB_OUTPUT" From 8fd24c5bee1e68351392ad73985685ef978c9408 Mon Sep 17 00:00:00 2001 From: Thomas-Boyle Date: Tue, 7 Apr 2026 11:02:51 +0100 Subject: [PATCH 02/10] Update Docker Buildx action version in Lambda deployment workflow to improve build consistency. --- .github/workflows/deploy-lambda-artifact.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-lambda-artifact.yml b/.github/workflows/deploy-lambda-artifact.yml index 13dbb0e39..216dcf8a4 100644 --- a/.github/workflows/deploy-lambda-artifact.yml +++ b/.github/workflows/deploy-lambda-artifact.yml @@ -241,7 +241,7 @@ jobs: - name: Set up Docker Buildx if: ${{ steps.decide.outputs.deployment_mode == 'build' && steps.build-check.outputs.should_build == 'true' }} - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f - name: Build and publish image with layer caching id: build-image From b997584c81e5a5b2730c3abd3b25ee7fc8d61251 Mon Sep 17 00:00:00 2001 From: Thomas-Boyle Date: Tue, 7 Apr 2026 11:04:15 +0100 Subject: [PATCH 03/10] Update Docker Buildx action version in Lambda deployment workflow to ensure compatibility and improve build reliability. --- .github/workflows/deploy-lambda-artifact.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-lambda-artifact.yml b/.github/workflows/deploy-lambda-artifact.yml index 216dcf8a4..b8f7b07f9 100644 --- a/.github/workflows/deploy-lambda-artifact.yml +++ b/.github/workflows/deploy-lambda-artifact.yml @@ -246,7 +246,7 @@ jobs: - name: Build and publish image with layer caching id: build-image if: ${{ steps.decide.outputs.deployment_mode == 'build' && steps.build-check.outputs.should_build == 'true' }} - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: context: ${{ env.DOCKER_CONTEXT_PATH }} file: ${{ env.DOCKERFILE_PATH }} From 0a6597f4332097692610343e64762f41f3a4f5ab Mon Sep 17 00:00:00 2001 From: Thomas-Boyle Date: Tue, 7 Apr 2026 11:40:03 +0100 Subject: [PATCH 04/10] Update description for build_image parameter in Lambda deployment workflow to improve clarity. --- .github/workflows/deploy-lambda-artifact.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-lambda-artifact.yml b/.github/workflows/deploy-lambda-artifact.yml index b8f7b07f9..08440736b 100644 --- a/.github/workflows/deploy-lambda-artifact.yml +++ b/.github/workflows/deploy-lambda-artifact.yml @@ -16,7 +16,7 @@ on: required: true type: string build_image: - description: Force a fresh build and publish + description: Force a fresh build and publish. required: false type: boolean default: false From 67649455cb179782f734ea02784207acfe1fe4e4 Mon Sep 17 00:00:00 2001 From: Thomas-Boyle Date: Tue, 7 Apr 2026 11:46:32 +0100 Subject: [PATCH 05/10] Change to test build --- lambdas/recordprocessor/src/batch_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/recordprocessor/src/batch_processor.py b/lambdas/recordprocessor/src/batch_processor.py index 7ceeb249e..da5c6631e 100644 --- a/lambdas/recordprocessor/src/batch_processor.py +++ b/lambdas/recordprocessor/src/batch_processor.py @@ -126,7 +126,7 @@ def process_rows( total_rows_processed_count: int = 0, ) -> tuple[int, Exception | None]: """ - Processes each row in the csv_reader starting from start_row. + Processes each row in the csv_reader starting from start_row """ row_count = 0 start_row = total_rows_processed_count From e70b864244309b60192697e2c5e1dd347d861c0c Mon Sep 17 00:00:00 2001 From: Thomas-Boyle Date: Wed, 8 Apr 2026 11:50:18 +0100 Subject: [PATCH 06/10] Update Lambda deployment workflow to disable provenance for Docker image builds and improve docstring clarity in batch processor. --- .github/workflows/deploy-lambda-artifact.yml | 1 + lambdas/recordprocessor/src/batch_processor.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-lambda-artifact.yml b/.github/workflows/deploy-lambda-artifact.yml index 08440736b..13a68a03b 100644 --- a/.github/workflows/deploy-lambda-artifact.yml +++ b/.github/workflows/deploy-lambda-artifact.yml @@ -251,6 +251,7 @@ jobs: context: ${{ env.DOCKER_CONTEXT_PATH }} file: ${{ env.DOCKERFILE_PATH }} push: true + provenance: false tags: | ${{ steps.build-check.outputs.image_uri_git }} ${{ steps.build-check.outputs.image_uri_rel }} diff --git a/lambdas/recordprocessor/src/batch_processor.py b/lambdas/recordprocessor/src/batch_processor.py index da5c6631e..7ceeb249e 100644 --- a/lambdas/recordprocessor/src/batch_processor.py +++ b/lambdas/recordprocessor/src/batch_processor.py @@ -126,7 +126,7 @@ def process_rows( total_rows_processed_count: int = 0, ) -> tuple[int, Exception | None]: """ - Processes each row in the csv_reader starting from start_row + Processes each row in the csv_reader starting from start_row. """ row_count = 0 start_row = total_rows_processed_count From b2091bf4262a270fb58fe90b5166853eba03cd1b Mon Sep 17 00:00:00 2001 From: Thomas-Boyle Date: Wed, 8 Apr 2026 14:11:20 +0100 Subject: [PATCH 07/10] Refactor Lambda deployment workflow to streamline image build conditions. Removed unnecessary variables and simplified logic for determining whether to build a new image based on existing image digest. Updated Docker build and ECR login steps to enhance efficiency and reliability. --- .github/workflows/deploy-lambda-artifact.yml | 31 +++++++------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/.github/workflows/deploy-lambda-artifact.yml b/.github/workflows/deploy-lambda-artifact.yml index 13a68a03b..1ed34e16f 100644 --- a/.github/workflows/deploy-lambda-artifact.yml +++ b/.github/workflows/deploy-lambda-artifact.yml @@ -209,9 +209,6 @@ jobs: --output text )" - IMAGE_URI_GIT="${REPOSITORY_URI}:${GIT_TAG}" - IMAGE_URI_REL="${REPOSITORY_URI}:${REL_TAG}" - EXISTING_IMAGE_DIGEST="$( aws ecr describe-images \ --repository-name "${ECR_REPOSITORY}" \ @@ -221,31 +218,27 @@ jobs: --output text 2>/dev/null || true )" - SHOULD_BUILD="false" - if [ -z "${EXISTING_IMAGE_DIGEST}" ] || [ "${EXISTING_IMAGE_DIGEST}" = "None" ]; then - SHOULD_BUILD="true" + if [ "${EXISTING_IMAGE_DIGEST}" = "None" ]; then + EXISTING_IMAGE_DIGEST="" fi echo "git_tag=${GIT_TAG}" >> "$GITHUB_OUTPUT" echo "release_tag=${REL_TAG}" >> "$GITHUB_OUTPUT" echo "repository_uri=${REPOSITORY_URI}" >> "$GITHUB_OUTPUT" - echo "image_uri_git=${IMAGE_URI_GIT}" >> "$GITHUB_OUTPUT" - echo "image_uri_rel=${IMAGE_URI_REL}" >> "$GITHUB_OUTPUT" echo "existing_image_digest=${EXISTING_IMAGE_DIGEST}" >> "$GITHUB_OUTPUT" - echo "should_build=${SHOULD_BUILD}" >> "$GITHUB_OUTPUT" - name: Login to Amazon ECR id: login-ecr - if: ${{ steps.decide.outputs.deployment_mode == 'build' && steps.build-check.outputs.should_build == 'true' }} + if: ${{ steps.decide.outputs.deployment_mode == 'build' && !steps.build-check.outputs.existing_image_digest }} uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 - name: Set up Docker Buildx - if: ${{ steps.decide.outputs.deployment_mode == 'build' && steps.build-check.outputs.should_build == 'true' }} + if: ${{ steps.decide.outputs.deployment_mode == 'build' && !steps.build-check.outputs.existing_image_digest }} uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f - name: Build and publish image with layer caching id: build-image - if: ${{ steps.decide.outputs.deployment_mode == 'build' && steps.build-check.outputs.should_build == 'true' }} + if: ${{ steps.decide.outputs.deployment_mode == 'build' && !steps.build-check.outputs.existing_image_digest }} uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: context: ${{ env.DOCKER_CONTEXT_PATH }} @@ -253,8 +246,8 @@ jobs: push: true provenance: false tags: | - ${{ steps.build-check.outputs.image_uri_git }} - ${{ steps.build-check.outputs.image_uri_rel }} + ${{ steps.build-check.outputs.repository_uri }}:${{ steps.build-check.outputs.git_tag }} + ${{ steps.build-check.outputs.repository_uri }}:${{ steps.build-check.outputs.release_tag }} cache-from: type=gha,scope=${{ env.ECR_REPOSITORY }} cache-to: type=gha,mode=max,scope=${{ env.ECR_REPOSITORY }} @@ -266,19 +259,17 @@ jobs: GIT_TAG: ${{ steps.build-check.outputs.git_tag }} REL_TAG: ${{ steps.build-check.outputs.release_tag }} EXISTING_IMAGE_DIGEST: ${{ steps.build-check.outputs.existing_image_digest }} - SHOULD_BUILD: ${{ steps.build-check.outputs.should_build }} BUILT_IMAGE_DIGEST: ${{ steps.build-image.outputs.digest }} run: | set -euo pipefail - IMAGE_DIGEST="${EXISTING_IMAGE_DIGEST}" - if [ "${SHOULD_BUILD}" = "true" ]; then - IMAGE_DIGEST="${BUILT_IMAGE_DIGEST}" - else + IMAGE_DIGEST="${BUILT_IMAGE_DIGEST:-${EXISTING_IMAGE_DIGEST}}" + + if [ -n "${EXISTING_IMAGE_DIGEST}" ] && [ -z "${BUILT_IMAGE_DIGEST}" ]; then echo "Immutable tag '${GIT_TAG}' already exists. Reusing existing image digest." fi - if [ -z "${IMAGE_DIGEST}" ] || [ "${IMAGE_DIGEST}" = "None" ]; then + if [ -z "${IMAGE_DIGEST}" ]; then echo "Unable to resolve image digest for tag '${GIT_TAG}'." exit 1 fi From 4eb09591139ca9cc393644058de902df5e85ef92 Mon Sep 17 00:00:00 2001 From: Thomas-Boyle Date: Thu, 9 Apr 2026 10:56:18 +0100 Subject: [PATCH 08/10] chore: empty commit From 9539f15fa6e93dfaf81225a21a9e0b5bcf6abb7e Mon Sep 17 00:00:00 2001 From: Thomas-Boyle Date: Mon, 13 Apr 2026 10:48:18 +0100 Subject: [PATCH 09/10] Update AWS actions in Lambda deployment workflow to latest versions for improved security and functionality. --- .github/workflows/deploy-lambda-artifact.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-lambda-artifact.yml b/.github/workflows/deploy-lambda-artifact.yml index 1ed34e16f..64a859982 100644 --- a/.github/workflows/deploy-lambda-artifact.yml +++ b/.github/workflows/deploy-lambda-artifact.yml @@ -123,7 +123,7 @@ jobs: fetch-depth: 0 - name: Connect to AWS - uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 with: aws-region: eu-west-2 role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops @@ -230,7 +230,7 @@ jobs: - name: Login to Amazon ECR id: login-ecr if: ${{ steps.decide.outputs.deployment_mode == 'build' && !steps.build-check.outputs.existing_image_digest }} - uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 + uses: aws-actions/amazon-ecr-login@f2e9fc6c2b355c1890b65e6f6f0e2ac3e6e22f78 - name: Set up Docker Buildx if: ${{ steps.decide.outputs.deployment_mode == 'build' && !steps.build-check.outputs.existing_image_digest }} From 311919dda193b812789c5adda640c2c6fb4824d9 Mon Sep 17 00:00:00 2001 From: Thomas-Boyle Date: Mon, 13 Apr 2026 12:19:38 +0100 Subject: [PATCH 10/10] Add support for multi-platform builds by specifying target platform in Docker deployment --- .github/workflows/deploy-lambda-artifact.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-lambda-artifact.yml b/.github/workflows/deploy-lambda-artifact.yml index 64a859982..98b460b04 100644 --- a/.github/workflows/deploy-lambda-artifact.yml +++ b/.github/workflows/deploy-lambda-artifact.yml @@ -243,6 +243,7 @@ jobs: with: context: ${{ env.DOCKER_CONTEXT_PATH }} file: ${{ env.DOCKERFILE_PATH }} + platforms: linux/amd64 push: true provenance: false tags: |