VED-738 refactoring

.github/dependabot.yml

@@ -10,11 +10,10 @@ updates:
       - "/lambdas/ack_backend"
       - "/delta_backend"
       - "/filenameprocessor"
-      - "/grafana/non-prod/docker"
+      - "/infrastructure/grafana/non-prod/docker"
       - "/mesh_processor"
       - "/recordprocessor"
       - "/sandbox"
-      - "/temporary_sandbox"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1
@@ -52,8 +51,8 @@ updates:
       - "/backend"
       - "/batch_processor_filter"
       - "/delta_backend"
-      - "/e2e"
-      - "/e2e_batch"
+      - "/tests/e2e"
+      - "/tests/e2e_batch"
       - "/filenameprocessor"
       - "/mesh_processor"
       - "/recordprocessor"
@@ -73,10 +72,10 @@ updates:
 
   - package-ecosystem: "terraform"
     directories:
-      - "/grafana/non-prod/terraform"
-      - "/infra"
-      - "/terraform"
-      - "/terraform_aws_backup/**"
+      - "/infrastructure/grafana/non-prod/terraform"
+      - "/infrastructure/account"
+      - "/infrastructure/instance"
+      - "/infrastructure/terraform_aws_backup/**"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1

.github/workflows/create-release-tag.yml

@@ -29,7 +29,7 @@ jobs:
         run: pip install semver
 
       - name: Set SPEC_VERSION env var
-        run: echo ::set-env name=SPEC_VERSION::$(python scripts/calculate_version.py)
+        run: echo ::set-env name=SPEC_VERSION::$(python utilities/scripts/calculate_version.py)
         env:
           ACTIONS_ALLOW_UNSECURE_COMMANDS: true
 

.github/workflows/deploy-backend.yml

@@ -72,18 +72,18 @@ jobs:
           terraform_version: "1.12.2"
 
       - name: Terraform Init
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: make init
 
       - name: Terraform Plan
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: make plan-ci
 
       - name: Save Terraform Plan
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         with:
           name: ${{ env.ENVIRONMENT }}-${{ env.SUB_ENVIRONMENT }}-tfplan
-          path: terraform/tfplan
+          path: infrastructure/instance/tfplan
 
   terraform-apply:
     needs: terraform-plan
@@ -108,14 +108,14 @@ jobs:
         uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0
         with:
           name: ${{ env.ENVIRONMENT }}-${{ env.SUB_ENVIRONMENT }}-tfplan
-          path: terraform
+          path: infrastructure/instance
 
       - name: Terraform Init
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: make init
 
       - name: Terraform Apply
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: |
           make apply-ci
           echo "ID_SYNC_QUEUE_ARN=$(make -s output name=id_sync_queue_arn)" >> $GITHUB_ENV

.github/workflows/pr-teardown.yml

@@ -43,7 +43,7 @@ jobs:
           terraform_version: "1.12.2"
 
       - name: Terraform Init and extract MNS SQS QUEUE ARN
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: |
           make init apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
           make workspace apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
@@ -71,6 +71,6 @@ jobs:
           make unsubscribe
 
       - name: Terraform Destroy
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: |
           make destroy apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT

.github/workflows/quality-checks.yml

@@ -175,7 +175,7 @@ jobs:
         working-directory: lambdas/ack_backend
         id: acklambda
         env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/ack_backend/src:tests:${{ env.SHARED_PATH }}/src
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/ack_backend/src:${{ env.LAMBDA_PATH }}/ack_backend/tests:${{ env.SHARED_PATH }}/src
         continue-on-error: true
         run: |
           poetry install
@@ -186,7 +186,7 @@ jobs:
         working-directory: lambdas/mns_subscription
         id: mns_subscription
         env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/mns_subscription/src:${{ env.SHARED_PATH }}/src
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/mns_subscription/src:${{ env.LAMBDA_PATH }}/mns_subscription/tests:${{ env.SHARED_PATH }}/src
         continue-on-error: true
         run: |
           poetry install
@@ -198,7 +198,7 @@ jobs:
         working-directory: lambdas/redis_sync
         id: redis_sync
         env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/redis_sync/src:${{ env.SHARED_PATH }}/src
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/redis_sync/src:${{ env.LAMBDA_PATH }}/redis_sync/tests:${{ env.SHARED_PATH }}/src
         continue-on-error: true
         run: |
           poetry install
@@ -220,7 +220,7 @@ jobs:
         working-directory: lambdas/id_sync
         id: id_sync
         env:
-          PYTHONPATH: ${{ env.LAMBDA_PATH }}/id_sync/src:${{ env.SHARED_PATH }}/src
+          PYTHONPATH: ${{ env.LAMBDA_PATH }}/id_sync/src:${{ env.LAMBDA_PATH }}/id_sync/tests:${{ env.SHARED_PATH }}/src
         continue-on-error: true
         run: |
           poetry install

.github/workflows/run-e2e-tests.yml

@@ -132,17 +132,17 @@ jobs:
 
       - name: Terraform Init
         if: ${{ env.TF_OUTPUTS_REQUIRED == 'true' }}
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: make init
 
       - name: Set Terraform workspace
         if: ${{ env.TF_OUTPUTS_REQUIRED == 'true' }}
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: make workspace
 
       - name: Read Terraform outputs
         if: ${{ env.TF_OUTPUTS_REQUIRED == 'true' }}
-        working-directory: terraform
+        working-directory: infrastructure/instance
         run: |
           echo "IMMS_DELTA_TABLE_NAME=$(make -s output name=imms_delta_table_name)" >> $GITHUB_ENV
           echo "AWS_DOMAIN_NAME=$(make -s output name=service_domain_name)" >> $GITHUB_ENV
@@ -157,15 +157,15 @@ jobs:
         with:
           python-version: 3.11
           cache: "poetry"
-          cache-dependency-path: "e2e/poetry.lock"
+          cache-dependency-path: tests/e2e/poetry.lock
 
       - name: Install e2e test dependencies
-        working-directory: e2e
+        working-directory: tests/e2e
         run: poetry install --no-root
 
       - name: Get Apigee access token
         if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
-        working-directory: e2e
+        working-directory: tests/e2e
         env:
           APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }}
           APIGEE_BASIC_AUTH_TOKEN: ${{ secrets.APIGEE_BASIC_AUTH_TOKEN }}
@@ -187,17 +187,17 @@ jobs:
 
       - name: Run proxy e2e test suite
         if: ${{ vars.RUN_PROXY_E2E_TESTS == 'true' }}
-        working-directory: e2e
+        working-directory: tests/e2e
         run: poetry run python -m unittest test_proxy
 
       - name: Run sandbox e2e test suite
         if: ${{ vars.RUN_SANDBOX_E2E_TESTS == 'true' }}
-        working-directory: e2e
+        working-directory: tests/e2e
         run: poetry run python -m unittest test_proxy.TestProxyHealthcheck
 
       - name: Run full e2e test suite
         if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
-        working-directory: e2e
+        working-directory: tests/e2e
         run: poetry run python -m unittest
 
   batch-e2e-tests:
@@ -223,14 +223,14 @@ jobs:
         with:
           python-version: 3.11
           cache: "poetry"
-          cache-dependency-path: "e2e_batch/poetry.lock"
+          cache-dependency-path: tests/e2e_batch/poetry.lock
 
       - name: Install e2e test dependencies
-        working-directory: e2e_batch
+        working-directory: tests/e2e_batch
         run: poetry install --no-root
 
       - name: Run batch e2e test suite
-        working-directory: e2e_batch
+        working-directory: tests/e2e_batch
         env:
           ENVIRONMENT: ${{ inputs.sub_environment }}
         run: poetry run python -m unittest -c -v

.gitignore

@@ -17,7 +17,7 @@ __pycache__/
 .envrc
 smoketest-report.xml
 lambda_typescript/**/*.js
-terraform/zips
+instance/zips
 .dir-locals.el
 *.pyc
 .python-version

.prettierignore

@@ -3,5 +3,5 @@
 
 # We hit compile errors in Apigee if these are auto formatted
 # TODO - investigate
-/proxies/live/apiproxy/resources/jsc/
-/proxies/sandbox/apiproxy/resources/jsc/
+/infrastructure/proxies/live/apiproxy/resources/jsc/
+/infrastructure/proxies/sandbox/apiproxy/resources/jsc/

Makefile

@@ -1,7 +1,7 @@
 SHELL=/usr/bin/env bash -euo pipefail
 
 PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS = backend batch_processor_filter delta_backend filenameprocessor mesh_processor recordprocessor lambdas/ack_backend lambdas/redis_sync lambdas/id_sync lambdas/mns_subscription lambdas/shared
-PYTHON_PROJECT_DIRS = e2e e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
+PYTHON_PROJECT_DIRS = tests/e2e tests/e2e_batch quality_checks $(PYTHON_PROJECT_DIRS_WITH_UNIT_TESTS)
 
 .PHONY: install lint format format-check clean publish build-proxy release initialise-all-python-venvs update-all-python-dependencies run-all-python-unit-tests build-all-docker-images
 
@@ -33,10 +33,10 @@ publish: clean
 
 #Runs build proxy script
 build-proxy:
-	scripts/build_proxy.sh
+	utilities/scripts/build_proxy.sh
 
 #Files to loop over in release
-_dist_include="poetry.toml Makefile build/. specification sandbox scripts"
+_dist_include="poetry.toml Makefile build/. specification sandbox utilities/scripts"
 
 #Create /dist/ sub-directory and copy files into directory
 #Ensure full dir structure is preserved for Lambdas

README.md

@@ -42,10 +42,9 @@ See https://nhsd-confluence.digital.nhs.uk/display/APM/Glossary.
 
 | Folder                 | Description                                                                                            |
 | ---------------------- | ------------------------------------------------------------------------------------------------------ |
-| `infra`                | Base infrastructure components.                                                                        |
+| `account`              | Base infrastructure components.                                                                        |
 | `grafana`              | Terraform configuration for Grafana, built on top of core infra.                                       |
-| `terraform`            | Core Terraform infrastructure code. This is run in each PR and sets up lambdas associated with the PR. |
-| `terraform_sandbox`    | Sandbox environment for testing infrastructure changes.                                                |
+| `instance`             | Core Terraform infrastructure code. This is run in each PR and sets up lambdas associated with the PR. |
 | `terraform_aws_backup` | Streamlined backup processing with AWS.                                                                |
 | `proxies`              | Apigee API proxy definitions.                                                                          |
 
@@ -251,7 +250,8 @@ run a different set of tests. To do this:
 - In Project Structure add an existing virtualenv SDK for `.direnv/python-x.x.x/bin/python`.
 - Set the project SDK and the default root module SDK to the one created above.
     - Add `tests` as sources.
-    - Add `.direnv`, `terraform/.terraform`, and `terraform/build` as exclusions if they're not already.
+    - Add `.direnv`, `infrastructure/instance/.terraform`, and `infrastructure/instance/build`
+      as exclusions if they're not already.
 - Add another existing virtualenv SDK for `backend/.direnv/python-x.x.x/bin/python`.
 - Import a module pointed at the `backend` directory, set the SDK created above.
     - Add the `src` and `tests` directories as sources.

README.specification.md

@@ -6,8 +6,8 @@ This is a specification for the _immunisation-fhir-api_ API.
 
 - `specification/` This [Open API Specification](https://swagger.io/docs/specification/about/) describes the endpoints, methods and messages exchanged by the API. Use it to generate interactive documentation; the contract between the API and its consumers.
 - `sandbox/` This NodeJS application implements a mock implementation of the service. Use it as a back-end service to the interactive documentation to illustrate interactions and concepts. It is not intended to provide an exhaustive/faithful environment suitable for full development and testing.
-- `scripts/` Utilities helpful to developers of this specification.
-- `proxies/` Live (connecting to another service) and sandbox (using the sandbox container) Apigee API Proxy definitions.
+- `utilities/scripts/` Utilities helpful to developers of this specification.
+- `infrastructure/proxies/` Live (connecting to another service) and sandbox (using the sandbox container) Apigee API Proxy definitions.
 
 Consumers of the API will find developer documentation on the [NHS Digital Developer Hub](https://digital.nhs.uk/developer).
 
@@ -129,7 +129,7 @@ The `project.yml` file needs to be populated with your service names to make the
 
 `/azure/templates`: Here you can define reusable actions, such as running tests, and call these actions during Azure Devops pipelines. 
 
-#### `/proxies`:
+#### `infrastructure/proxies`:
 
 This folder contains files relating to your Apigee API proxy.
 
@@ -146,16 +146,16 @@ See the APM confluence for more information on how the [\_ping](https://nhsd-con
 
 `/targets`: The XMLs within these folders set up target definitions which allow connections to external target servers. The sandbox target definition is implemented to route to the sandbox target server (code for this sandbox is found under /sandbox of this template repo). For more info on setting up a target server see the [API Producer Zone confluence](https://nhsd-confluence.digital.nhs.uk/display/APM/Setting+up+a+target+server)
 
-#### `/sandbox`:
+#### `sandbox`:
 
 This folder contains a template for a sandbox API. This example is a NodeJs application running in Docker. The application handles a few simple endpoints such as: /\_ping, /health, /\_status, /hello and some logging logic.
 For more information about building sandbox APIs see the [API Producer Zone confluence](https://nhsd-confluence.digital.nhs.uk/display/APM/Setting+up+your+API+sandbox).
 
-#### `/scripts`:
+#### `utilities/scripts`:
 
 Contains useful scripts that are used throughout the project, for example in Makefile and Github workflows
 
-#### `/specification`:
+#### `specification`:
 
 Create an OpenAPI Specification to document your API. For more information about developing specifications see the [API Producer Zone confluence](https://nhsd-confluence.digital.nhs.uk/display/APM/Documenting+your+API).
 

immunisation-fhir-api.code-workspace

@@ -19,10 +19,10 @@
             "path": "mesh_processor",
         },
         {
-            "path": "e2e",
+            "path": "tests/e2e",
         },
         {
-            "path": "e2e_batch",
+            "path": "tests/e2e_batch",
         },
         {
             "path": "lambdas/ack_backend",

terraform/Makefile → infrastructure/instance/Makefile

@@ -56,13 +56,13 @@ destroy: workspace
 output:
 	$(tf_cmd) output -raw $(name)
 
-#Make lambda zip file in /terraform/zips directory. Whenever code gets changed in lamdba_typescript directory , new zip file gets uploaded to s3. For local,you can you this make target
+#Make lambda zip file in /infrastructure/instance/zips directory. Whenever code gets changed in lamdba_typescript directory , new zip file gets uploaded to s3. For local,you can you this make target
 lambda-zip:
 	cd ../lambda_typescript && \
 	chmod +x ./deploy.sh && \
 	./deploy.sh
 
-#Make catch-all-lambda zip file in /terraform/zips directory. Whenever code gets changed in lamdba_typescript directory , new zip file gets uploaded to s3. For local,you can you this make target
+#Make catch-all-lambda zip file in /infrastructure/instance/zips directory. Whenever code gets changed in lamdba_typescript directory , new zip file gets uploaded to s3. For local,you can you this make target
 catch-all-zip:
 	cd ../catch_all_lambda && \
 	chmod +x ./deploy.sh && \

terraform/ack_lambda.tf → infrastructure/instance/ack_lambda.tf

@@ -1,6 +1,6 @@
 # Define the directory containing the Docker image and calculate its SHA-256 hash for triggering redeployments
 locals {
-  ack_lambda_dir = abspath("${path.root}/../lambdas/ack_backend")
+  ack_lambda_dir = abspath("${path.root}/../../lambdas/ack_backend")
 
   ack_lambda_files = fileset(local.ack_lambda_dir, "**")
 
@@ -43,7 +43,7 @@ module "ack_processor_docker_image" {
 
   platform      = "linux/amd64"
   use_image_tag = false
-  source_path   = abspath("${path.root}/../lambdas")
+  source_path   = abspath("${path.root}/../../lambdas")
   triggers = {
     dir_sha        = local.ack_lambda_dir_sha
     shared_dir_sha = local.shared_dir_sha

terraform/batch_processor_filter_lambda.tf → infrastructure/instance/batch_processor_filter_lambda.tf

@@ -1,6 +1,6 @@
 # Define the directory containing the Docker image and calculate its SHA-256 hash for triggering redeployments
 locals {
-  batch_processor_filter_lambda_dir     = abspath("${path.root}/../batch_processor_filter")
+  batch_processor_filter_lambda_dir     = abspath("${path.root}/../../batch_processor_filter")
   batch_processor_filter_lambda_files   = fileset(local.batch_processor_filter_lambda_dir, "**")
   batch_processor_filter_lambda_dir_sha = sha1(join("", [for f in local.batch_processor_filter_lambda_files : filesha1("${local.batch_processor_filter_lambda_dir}/${f}")]))
 }

terraform/delta.tf → infrastructure/instance/delta.tf

@@ -1,5 +1,5 @@
 locals {
-  delta_lambda_dir = abspath("${path.root}/../delta_backend")
+  delta_lambda_dir = abspath("${path.root}/../../delta_backend")
   delta_files      = fileset(local.delta_lambda_dir, "**")
   delta_dir_sha    = sha1(join("", [for f in local.delta_files : filesha1("${local.delta_lambda_dir}/${f}")]))
   function_name    = "delta"