Update: [AEA-5871] - modify eps-assist-me deployment pipeline to run regression tests (#97)

## Summary

🎫 [AEA-5871](https://nhsd-jira.digital.nhs.uk/browse/AEA-5871) modify
eps-assist-me deployment pipeline to run regression tests

- Routine Change
- 🤖 Operational or Infrastructure Change

### Details

This pull request updates the eps-assist-me deployment pipeline to run
regression tests.

some refactors:
- makes devcontainer arm macos happy 🥳 
- L1 to L2 infra change of OpenSearch
- makes asdf versioning a repo level change
- use common release tagging workflow

---------

Co-authored-by: Anthony Brown <121869075+anthony-nhs@users.noreply.github.com>
Co-authored-by: Bence Gadanyi <bence.gadanyi1@nhs.net>
Co-authored-by: tstephen-nhs <231503406+tstephen-nhs@users.noreply.github.com>

Author: 4 people

.devcontainer/Dockerfile

@@ -1,5 +1,17 @@
 FROM mcr.microsoft.com/devcontainers/base:ubuntu
 
+# provide DOCKER_GID via build args if you need to force group id to match host
+ARG DOCKER_GID
+ARG TARGETARCH
+ENV TARGETARCH=${TARGETARCH}
+
+ARG ASDF_VERSION
+COPY .tool-versions.asdf /tmp/.tool-versions.asdf
+
+RUN if [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then dpkg --add-architecture arm64; fi
+
+# Anticipate and resolve potential permission issues with apt
+RUN mkdir -p /tmp && chmod 1777 /tmp
 # Install system dependencies
 RUN apt-get update \
     && export DEBIAN_FRONTEND=noninteractive \
@@ -8,49 +20,76 @@ RUN apt-get update \
     libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev libbz2-dev \
     zlib1g-dev unixodbc unixodbc-dev libsecret-1-0 libsecret-1-dev libsqlite3-dev \
     jq apt-transport-https ca-certificates gnupg-agent \
-    software-properties-common bash-completion python3-pip make libbz2-dev \
+    software-properties-common bash-completion python3-pip make \
     libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev \
-    xz-utils tk-dev liblzma-dev netcat-openbsd libyaml-dev
+    xz-utils tk-dev liblzma-dev netcat-traditional libyaml-dev
 
 # Install aws stuff
-RUN wget -O /tmp/awscliv2.zip "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" && \
+RUN if [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then \
+      wget -O /tmp/awscliv2.zip "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip"; \
+    else \
+      wget -O /tmp/awscliv2.zip "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"; \
+    fi && \
     unzip /tmp/awscliv2.zip -d /tmp/aws-cli && \
     /tmp/aws-cli/aws/install && \
-    rm tmp/awscliv2.zip && \
-    rm -rf /tmp/aws-cli
+    rm /tmp/awscliv2.zip && rm -rf /tmp/aws-cli
 
-# Set user to vscode
-USER vscode
+RUN if [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then \
+      wget -O /tmp/aws-sam-cli.zip "https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-arm64.zip"; \
+    else \
+      wget -O /tmp/aws-sam-cli.zip "https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip"; \
+    fi && \
+    unzip /tmp/aws-sam-cli.zip -d /tmp/aws-sam-cli && \
+    /tmp/aws-sam-cli/install && \
+    rm /tmp/aws-sam-cli.zip && rm -rf /tmp/aws-sam-cli
 
 # Install ASDF
-RUN git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch v0.11.3; \
-    echo '. $HOME/.asdf/asdf.sh' >> ~/.bashrc; \
-    echo '. $HOME/.asdf/completions/asdf.bash' >> ~/.bashrc; \
+RUN ASDF_VERSION=$(awk '!/^#/ && NF {print $1; exit}' /tmp/.tool-versions.asdf) && \
+    if [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" == "aarch64" ]; then \
+      wget -O /tmp/asdf.tar.gz "https://github.com/asdf-vm/asdf/releases/download/v${ASDF_VERSION}/asdf-v${ASDF_VERSION}-linux-arm64.tar.gz"; \
+    else \
+      wget -O /tmp/asdf.tar.gz "https://github.com/asdf-vm/asdf/releases/download/v${ASDF_VERSION}/asdf-v${ASDF_VERSION}-linux-amd64.tar.gz"; \
+    fi && \
+    tar -xzf /tmp/asdf.tar.gz -C /tmp && \
+    mkdir -p /usr/bin && \
+    mv /tmp/asdf /usr/bin/asdf && \
+    chmod +x /usr/bin/asdf && \
+    rm -rf /tmp/asdf.tar.gz 
+
+# specify DOCKER_GID to force container docker group id to match host
+RUN if [ -n "${DOCKER_GID}" ]; then \
+      if ! getent group docker; then \
+        groupadd -g ${DOCKER_GID} docker; \
+      else \
+        groupmod -g ${DOCKER_GID} docker; \
+      fi && \
+      usermod -aG docker vscode; \
+    fi
+
+USER vscode
+
+ENV PATH="/home/vscode/.asdf/shims/:$PATH"
+RUN \
+    echo 'PATH="/home/vscode/.asdf/shims/:$PATH"' >> ~/.bashrc; \
     echo '# Install Ruby Gems to ~/gems' >> ~/.bashrc; \
     echo 'export GEM_HOME="$HOME/gems"' >> ~/.bashrc; \
     echo 'export PATH="$HOME/gems/bin:$PATH"' >> ~/.bashrc;
 
-ENV PATH="$PATH:/home/vscode/.asdf/bin/:/workspaces/eps-assist-me/node_modules/.bin"
+ENV PATH="$PATH:/workspaces/eps-assist-me/node_modules/.bin"
 
 # Install ASDF plugins
-RUN asdf plugin add python; \
-    asdf plugin add poetry https://github.com/asdf-community/asdf-poetry.git; \
-    asdf plugin add shellcheck https://github.com/luizm/asdf-shellcheck.git; \
-    asdf plugin add nodejs https://github.com/asdf-vm/asdf-nodejs.git; \
-    asdf plugin add direnv; \
-    asdf plugin add actionlint; \
+RUN asdf plugin add python && \
+    asdf plugin add poetry https://github.com/asdf-community/asdf-poetry.git && \
+    asdf plugin add nodejs https://github.com/asdf-vm/asdf-nodejs.git && \
+    asdf plugin add shellcheck https://github.com/luizm/asdf-shellcheck.git && \
+    asdf plugin add direnv && \
+    asdf plugin add actionlint && \
     asdf plugin add ruby https://github.com/asdf-vm/asdf-ruby.git
 
-# Set working directory
 WORKDIR /workspaces/eps-assist-me
-
-# Copy .tool-versions
 ADD .tool-versions /workspaces/eps-assist-me/.tool-versions
 ADD .tool-versions /home/vscode/.tool-versions
 
-# Install tools
-RUN asdf install; \
-    asdf reshim python; \
-    asdf reshim poetry; \
-    asdf reshim nodejs; \
-    asdf direnv setup --shell bash --version 2.32.2;
+# install python before poetry to ensure correct python version is used
+RUN asdf install python && \
+    asdf install

.devcontainer/devcontainer.json

@@ -2,17 +2,19 @@
 // README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu
 {
   "name": "Ubuntu",
-  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
   "build": {
     "dockerfile": "Dockerfile",
     "context": "..",
-    "args": {}
+    "args": {
+      "DOCKER_GID": "${env:DOCKER_GID:}"
+    }
   },
   "mounts": [
     "source=${env:HOME}${env:USERPROFILE}/.aws,target=/home/vscode/.aws,type=bind",
     "source=${env:HOME}${env:USERPROFILE}/.ssh,target=/home/vscode/.ssh,type=bind",
     "source=${env:HOME}${env:USERPROFILE}/.gnupg,target=/home/vscode/.gnupg,type=bind",
-    "source=${env:HOME}${env:USERPROFILE}/.npmrc,target=/home/vscode/.npmrc,type=bind"
+    "source=${env:HOME}${env:USERPROFILE}/.npmrc,target=/home/vscode/.npmrc,type=bind",
+    "source=${env:HOME}${env:USERPROFILE}/.gitconfig,target=/home/vscode/.gitconfig,type=bind"
   ],
   "features": {
     "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
@@ -46,7 +48,7 @@
         "github.vscode-github-actions"
       ],
       "settings": {
-        "python.defaultInterpreterPath": "/workspaces/eps-assist-me/.venv/bin/python",
+        "python.defaultInterpreterPath": "/home/vscode/.asdf/shims/python",
         "python.analysis.autoSearchPaths": true,
         "python.analysis.extraPaths": [],
         "python.testing.unittestEnabled": false,

.github/workflows/cdk_package_code.yml

@@ -26,10 +26,10 @@ jobs:
         with:
           ref: ${{ env.BRANCH_NAME }}
 
-      # using git commit sha for version of action to ensure we have stable version
       - name: Get asdf version
         id: asdf-version
-        run: echo "version=0.18.0" >> "$GITHUB_OUTPUT"
+        run: echo "version=$(awk '!/^#/ && NF {print $1; exit}' .tool-versions.asdf)" >> "$GITHUB_OUTPUT"
+
 
       # using git commit sha for version of action to ensure we have stable version
       - name: Install asdf

.github/workflows/ci.yml

@@ -32,18 +32,8 @@ jobs:
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
-  get_commit_id:
-    runs-on: ubuntu-22.04
-    outputs:
-      commit_id: ${{ steps.commit_id.outputs.commit_id }}
-    steps:
-      - name: Get Commit ID
-        id: commit_id
-        run: |
-          echo "commit_id=${{ github.sha }}" >> "$GITHUB_OUTPUT"
-
   tag_release:
-    needs: [quality_checks, get_commit_id, get_asdf_version]
+    needs: [quality_checks, get_asdf_version]
     uses: NHSDigital/eps-workflow-semantic-release/.github/workflows/tag-release.yml@361957c147279f5f0f68b64fde9927833363d5f7
     with:
       dry_run: true
@@ -54,21 +44,21 @@ jobs:
     secrets: inherit
 
   package_code:
-    needs: [get_commit_id, tag_release]
+    needs: [tag_release]
     uses: ./.github/workflows/cdk_package_code.yml
     with:
       STACK_NAME: epsam
       VERSION_NUMBER: ${{ needs.tag_release.outputs.version_tag }}
-      COMMIT_ID: ${{ needs.get_commit_id.outputs.commit_id }}
+      COMMIT_ID: ${{ github.sha }}
 
   release_dev:
-    needs: [get_commit_id, tag_release, package_code]
+    needs: [tag_release, package_code]
     uses: ./.github/workflows/release_all_stacks.yml
     with:
       STACK_NAME: epsam
       TARGET_ENVIRONMENT: dev
       VERSION_NUMBER: ${{ needs.tag_release.outputs.version_tag }}
-      COMMIT_ID: ${{ needs.get_commit_id.outputs.commit_id }}
+      COMMIT_ID: ${{ github.sha }}
       CDK_APP_NAME: EpsAssistMeApp
       DEPLOY_CODE: true
       LOG_RETENTION_IN_DAYS: 30
@@ -78,6 +68,7 @@ jobs:
       MARK_JIRA_RELEASED: false
       CREATE_INT_RC_RELEASE_NOTES: false
       IS_PULL_REQUEST: false
+      RUN_REGRESSION_TESTS: true
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.DEV_CLOUD_FORMATION_DEPLOY_ROLE }}
       CDK_PULL_IMAGE_ROLE: ${{ secrets.DEV_CDK_PULL_IMAGE_ROLE }}
@@ -90,22 +81,23 @@ jobs:
       SLACK_SIGNING_SECRET: ${{ secrets.SLACK_SIGNING_SECRET }}
 
   release_qa:
-    needs: [get_commit_id, tag_release, package_code, release_dev]
+    needs: [tag_release, package_code, release_dev]
     uses: ./.github/workflows/release_all_stacks.yml
     with:
       STACK_NAME: epsam
       TARGET_ENVIRONMENT: qa
       VERSION_NUMBER: ${{ needs.tag_release.outputs.version_tag }}
-      COMMIT_ID: ${{ needs.get_commit_id.outputs.commit_id }}
+      COMMIT_ID: ${{ github.sha }}
       CDK_APP_NAME: EpsAssistMeApp
       DEPLOY_CODE: true
       LOG_RETENTION_IN_DAYS: 30
       LOG_LEVEL: "DEBUG"
-      CREATE_INT_RELEASE_NOTES: true
-      CREATE_PROD_RELEASE_NOTES: true
+      CREATE_INT_RELEASE_NOTES: false
+      CREATE_PROD_RELEASE_NOTES: false
       MARK_JIRA_RELEASED: false
       CREATE_INT_RC_RELEASE_NOTES: false
       IS_PULL_REQUEST: false
+      RUN_REGRESSION_TESTS: true
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.QA_CLOUD_FORMATION_DEPLOY_ROLE }}
       CDK_PULL_IMAGE_ROLE: ${{ secrets.QA_CDK_PULL_IMAGE_ROLE }}

.github/workflows/pull_request.yml

@@ -19,8 +19,7 @@ jobs:
 
       - name: Get asdf version
         id: asdf-version
-        run: echo "version=0.18.0" >> "$GITHUB_OUTPUT"
-
+        run: echo "version=$(awk '!/^#/ && NF {print $1; exit}' .tool-versions.asdf)" >> "$GITHUB_OUTPUT"
       - name: Load config value
         id: load-config
         run: |
@@ -62,31 +61,22 @@ jobs:
             }
           result-encoding: string
 
-  get_commit_id:
-    runs-on: ubuntu-22.04
-    outputs:
-      commit_id: ${{ steps.commit_id.outputs.commit_id }}
-    steps:
-      - name: Get Commit ID
-        id: commit_id
-        run: |
-          echo "commit_id=${{ github.sha }}" >> "$GITHUB_OUTPUT"
   package_code:
-    needs: [get_issue_number, get_commit_id]
+    needs: [get_issue_number]
     uses: ./.github/workflows/cdk_package_code.yml
     with:
       STACK_NAME: epsam-pr-${{needs.get_issue_number.outputs.issue_number}}
       VERSION_NUMBER: PR-${{ needs.get_issue_number.outputs.issue_number }}
-      COMMIT_ID: ${{ needs.get_commit_id.outputs.commit_id }}
+      COMMIT_ID: ${{ github.sha }}
 
   release_code:
-    needs: [get_issue_number, package_code, get_commit_id]
+    needs: [get_issue_number, package_code]
     uses: ./.github/workflows/release_all_stacks.yml
     with:
       STACK_NAME: epsam-pr-${{needs.get_issue_number.outputs.issue_number}}
       TARGET_ENVIRONMENT: dev-pr
       VERSION_NUMBER: PR-${{ needs.get_issue_number.outputs.issue_number }}
-      COMMIT_ID: ${{ needs.get_commit_id.outputs.commit_id }}
+      COMMIT_ID: ${{ github.sha }}
       CDK_APP_NAME: EpsAssistMeApp
       DEPLOY_CODE: true
       LOG_RETENTION_IN_DAYS: 30
@@ -96,6 +86,7 @@ jobs:
       MARK_JIRA_RELEASED: false
       CREATE_INT_RC_RELEASE_NOTES: false
       IS_PULL_REQUEST: true
+      RUN_REGRESSION_TESTS: true
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.DEV_CLOUD_FORMATION_DEPLOY_ROLE }}
       CDK_PULL_IMAGE_ROLE: ${{ secrets.DEV_CDK_PULL_IMAGE_ROLE }}