VED-806 Refactor batch_processor_filter (#952)

* init I

* root

* makefile

* common

* dependabot

* lint

* aws_dynamodb

* common.clients

* ruff

* rm comment

* REGION_NAME

Author: JamesW1-NHS

.github/dependabot.yml

@@ -7,9 +7,9 @@ version: 2
 updates:
   - package-ecosystem: "docker"
     directories:
-      - "/batch_processor_filter"
       - "/infrastructure/grafana/non-prod/docker"
       - "/lambdas/ack_backend"
+      - "/lambdas/batch_processor_filter"
       - "/lambdas/delta_backend"
       - "/lambdas/filenameprocessor"
       - "/lambdas/mesh_processor"
@@ -50,8 +50,8 @@ updates:
     directories:
       - "/"
       - "/backend"
-      - "/batch_processor_filter"
       - "/lambdas/ack_backend"
+      - "/lambdas/batch_processor_filter"
       - "/lambdas/delta_backend"
       - "/lambdas/filenameprocessor"
       - "/lambdas/id_sync"

.github/workflows/quality-checks.yml

@@ -97,17 +97,6 @@ jobs:
           aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
           aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
 
-      - name: Run unittest with batchprocessorfilter-coverage
-        working-directory: batch_processor_filter
-        id: batchprocessorfilter
-        env:
-          PYTHONPATH: ${{ github.workspace }}/batch_processor_filter/src:${{ github.workspace }}/batch_processor_filter/tests
-        continue-on-error: true
-        run: |
-          poetry install
-          poetry run coverage run -m unittest discover || echo "batchprocessorfilter tests failed" >> ../failed_tests.txt
-          poetry run coverage xml -o ../batchprocessorfilter-coverage.xml
-
       - name: Run unittest with recordprocessor-coverage
         working-directory: lambdas/recordprocessor
         id: recordprocessor
@@ -153,6 +142,17 @@ jobs:
           poetry run coverage run --source=src -m unittest discover || echo "ack-lambda tests failed" >> ../../failed_tests.txt
           poetry run coverage xml -o ../../ack-lambda-coverage.xml
 
+      - name: Run unittest with batchprocessorfilter-coverage
+        working-directory: lambdas/batch_processor_filter
+        id: batchprocessorfilter
+        env:
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/batch_processor_filter/src:${{ env.LAMBDA_PATH }}/batch_processor_filter/tests:${{ env.SHARED_PATH }}/src
+        continue-on-error: true
+        run: |
+          poetry install
+          poetry run coverage run -m unittest discover || echo "batchprocessorfilter tests failed" >> ../../failed_tests.txt
+          poetry run coverage xml -o ../../batchprocessorfilter-coverage.xml
+
       - name: Run unittest with coverage-delta
         working-directory: lambdas/delta_backend
         id: delta

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
-PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter lambdas/ack_backend lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mesh_processor lambdas/mns_subscription lambdas/recordprocessor lambdas/redis_sync lambdas/shared
+PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend lambdas/ack_backend lambdas/batch_processor_filter lambdas/delta_backend lambdas/filenameprocessor lambdas/id_sync lambdas/mesh_processor lambdas/mns_subscription lambdas/recordprocessor lambdas/redis_sync lambdas/shared
 PYTHON_PROJECT_DIRS = tests/e2e tests/e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 .PHONY: install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images

batch_processor_filter/src/logger.py

@@ -9,6 +9,9 @@
         {
             "path": "lambdas/ack_backend",
         },
+        {
+            "path": "lambdas/batch_processor_filter",
+        },
         {
             "path": "lambdas/delta_backend",
         },

batch_processor_filter/src/send_log_to_firehose.py

@@ -1,6 +1,6 @@
 # Define the directory containing the Docker image and calculate its SHA-256 hash for triggering redeployments
 locals {
-  batch_processor_filter_lambda_dir     = abspath("${path.root}/../../batch_processor_filter")
+  batch_processor_filter_lambda_dir     = abspath("${path.root}/../../lambdas/batch_processor_filter")
   batch_processor_filter_lambda_files   = fileset(local.batch_processor_filter_lambda_dir, "**")
   batch_processor_filter_lambda_dir_sha = sha1(join("", [for f in local.batch_processor_filter_lambda_files : filesha1("${local.batch_processor_filter_lambda_dir}/${f}")]))
 }
@@ -15,8 +15,9 @@ resource "aws_ecr_repository" "batch_processor_filter_lambda_repository" {
 
 # Module for building and pushing Docker image to ECR
 module "batch_processor_filter_docker_image" {
-  source  = "terraform-aws-modules/lambda/aws//modules/docker-build"
-  version = "8.1.2"
+  source           = "terraform-aws-modules/lambda/aws//modules/docker-build"
+  version          = "8.1.2"
+  docker_file_path = "./batch_processor_filter/Dockerfile"
 
   create_ecr_repo = false
   ecr_repo        = aws_ecr_repository.batch_processor_filter_lambda_repository.name
@@ -39,9 +40,10 @@ module "batch_processor_filter_docker_image" {
 
   platform      = "linux/amd64"
   use_image_tag = false
-  source_path   = local.batch_processor_filter_lambda_dir
+  source_path   = abspath("${path.root}/../../lambdas")
   triggers = {
-    dir_sha = local.batch_processor_filter_lambda_dir_sha
+    dir_sha        = local.batch_processor_filter_lambda_dir_sha
+    shared_dir_sha = local.shared_dir_sha
   }
 }
 

immunisation-fhir-api.code-workspace

@@ -5,12 +5,31 @@ RUN mkdir -p /home/appuser && \
     echo 'appuser:x:1001:' >> /etc/group && \
     chown -R 1001:1001 /home/appuser && pip install "poetry~=2.1.4"
 
-COPY poetry.lock pyproject.toml README.md ./
+# Install Poetry dependencies
+# Copy batch_processor_filter Poetry files
+COPY ./batch_processor_filter/poetry.lock ./batch_processor_filter/pyproject.toml ./
+
+# Install batch_processor_filter dependencies
+WORKDIR /var/task
 RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main
+
 # -----------------------------
 FROM base AS build
-COPY src .
+
+# Set working directory back to Lambda task root
+WORKDIR /var/task
+
+# Copy shared source code
+COPY ./shared/src/common ./common
+
+# Copy batch_processor_filter source code
+COPY ./batch_processor_filter/src .
+
+# Set correct permissions
 RUN chmod 644 $(find . -type f) && chmod 755 $(find . -type d)
+
 # Switch to the non-root user for running the container
 USER 1001:1001
+
+# Set the Lambda handler
 CMD ["lambda_handler.lambda_handler"]

infrastructure/instance/batch_processor_filter_lambda.tf

@@ -1,3 +1,5 @@
+TEST_ENV := @PYTHONPATH=src:tests:../shared/src
+
 build:
 	docker build -t imms-lambda-build -f Dockerfile .
 
@@ -6,15 +8,15 @@ package: build
 	docker run --rm -v $(shell pwd)/build:/build imms-lambda-build
 
 test:
-	@PYTHONPATH=src:tests python -m unittest
+	$(TEST_ENV) python -m unittest
 
 coverage-run:
-	@PYTHONPATH=src:tests coverage run -m unittest discover
+	$(TEST_ENV) coverage run -m unittest discover
 
 coverage-report:
-	coverage report -m
+	$(TEST_ENV) coverage report -m
 
 coverage-html:
-	coverage html
+	$(TEST_ENV) coverage html
 
 .PHONY: build package test