Skip to content

Commit 65706eb

Browse files
MacMur85MacMur85
authored
feat: [DTOSS-11798] devtest testing environment (#1785)
feat: DevTest workflows set, triggered by a commit with devtest tag
1 parent 118bb39 commit 65706eb

3 files changed

Lines changed: 468 additions & 10 deletions

File tree

‎.github/workflows/cicd-1-pull-request-devtest.yaml‎

Lines changed: 162 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,172 @@
1-
name: CI/CD pull request - devtest
1+
name: 'CI/CD pull request - devtest'
22

33
on:
44
push:
55
tags:
66
- 'devtest'
77

8+
permissions:
9+
contents: read
10+
811
jobs:
9-
print-debug-info:
12+
metadata:
13+
name: "Set CI/CD metadata"
1014
runs-on: ubuntu-latest
15+
timeout-minutes: 1
16+
permissions:
17+
pull-requests: read
18+
outputs:
19+
build_datetime_london: ${{ steps.variables.outputs.build_datetime_london }}
20+
build_datetime: ${{ steps.variables.outputs.build_datetime }}
21+
build_timestamp: ${{ steps.variables.outputs.build_timestamp }}
22+
build_epoch: ${{ steps.variables.outputs.build_epoch }}
23+
nodejs_version: ${{ steps.variables.outputs.nodejs_version }}
24+
python_version: ${{ steps.variables.outputs.python_version }}
25+
terraform_version: ${{ steps.variables.outputs.terraform_version }}
26+
environment_tag: ${{ steps.variables.outputs.environment_tag }}
27+
version: ${{ steps.variables.outputs.version }}
28+
does_pull_request_exist: ${{ steps.pr_exists.outputs.does_pull_request_exist }}
1129
steps:
12-
- name: Display Information
30+
- name: "Checkout code"
31+
uses: actions/checkout@v4
32+
with:
33+
submodules: 'true'
34+
- name: "Set CI/CD variables"
35+
id: variables
36+
run: |
37+
datetime=$(date -u +'%Y-%m-%dT%H:%M:%S%z')
38+
BUILD_DATETIME=$datetime make version-create-effective-file
39+
echo "build_datetime_london=$(TZ=Europe/London date --date=$datetime +'%Y-%m-%dT%H:%M:%S%z')" >> $GITHUB_OUTPUT
40+
echo "build_datetime=$datetime" >> $GITHUB_OUTPUT
41+
echo "build_timestamp=$(date --date=$datetime -u +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT
42+
echo "build_epoch=$(date --date=$datetime -u +'%s')" >> $GITHUB_OUTPUT
43+
echo "nodejs_version=$(grep "^nodejs" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
44+
echo "python_version=$(grep "^nodejs" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
45+
echo "terraform_version=$(grep "^terraform" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
46+
echo "version=$(head -n 1 .version 2> /dev/null || echo unknown)" >> $GITHUB_OUTPUT
47+
echo "environment_tag=development" >> $GITHUB_OUTPUT
48+
- name: "Check if pull request exists for this branch"
49+
id: pr_exists
50+
env:
51+
GH_TOKEN: ${{ github.token }}
52+
run: |
53+
branch_name=${GITHUB_HEAD_REF:-$(echo $GITHUB_REF | sed 's#refs/heads/##')}
54+
echo "Current branch is '$branch_name'"
55+
if gh pr list --head $branch_name | grep -q .; then
56+
echo "Pull request exists"
57+
echo "does_pull_request_exist=true" >> $GITHUB_OUTPUT
58+
else
59+
echo "Pull request doesn't exist"
60+
echo "does_pull_request_exist=false" >> $GITHUB_OUTPUT
61+
fi
62+
- name: "List variables"
1363
run: |
14-
echo "Workflow triggered by tag!"
15-
echo "--------------------------------------"
16-
echo "Tag/Ref Name : $GITHUB_REF"
17-
echo "Commit Hash : $GITHUB_SHA"
18-
echo "Triggered By : $GITHUB_ACTOR"
19-
echo "Event Name : $GITHUB_EVENT_NAME"
20-
echo "--------------------------------------"
64+
export BUILD_DATETIME_LONDON="${{ steps.variables.outputs.build_datetime_london }}"
65+
export BUILD_DATETIME="${{ steps.variables.outputs.build_datetime }}"
66+
export BUILD_TIMESTAMP="${{ steps.variables.outputs.build_timestamp }}"
67+
export BUILD_EPOCH="${{ steps.variables.outputs.build_epoch }}"
68+
export NODEJS_VERSION="${{ steps.variables.outputs.nodejs_version }}"
69+
export PYTHON_VERSION="${{ steps.variables.outputs.python_version }}"
70+
export TERRAFORM_VERSION="${{ steps.variables.outputs.terraform_version }}"
71+
export ENVIRONMENT_TAG="${{ steps.variables.outputs.environment_tag }}"
72+
export VERSION="${{ steps.variables.outputs.version }}"
73+
export DOES_PULL_REQUEST_EXIST="${{ steps.pr_exists.outputs.does_pull_request_exist }}"
74+
make list-variables
75+
commit-stage: # Recommended maximum execution time is 2 minutes
76+
name: "Commit stage"
77+
needs: [metadata]
78+
permissions:
79+
contents: read
80+
id-token: write
81+
uses: ./.github/workflows/stage-1-commit.yaml
82+
with:
83+
build_datetime: "${{ needs.metadata.outputs.build_datetime }}"
84+
build_timestamp: "${{ needs.metadata.outputs.build_timestamp }}"
85+
build_epoch: "${{ needs.metadata.outputs.build_epoch }}"
86+
nodejs_version: "${{ needs.metadata.outputs.nodejs_version }}"
87+
python_version: "${{ needs.metadata.outputs.python_version }}"
88+
terraform_version: "${{ needs.metadata.outputs.terraform_version }}"
89+
version: "${{ needs.metadata.outputs.version }}"
90+
test-stage: # Recommended maximum execution time is 5 minutes
91+
name: 'Test stage'
92+
needs: [metadata]
93+
permissions:
94+
pull-requests: write
95+
uses: ./.github/workflows/stage-2-test.yaml
96+
with:
97+
unit_test_dir: tests/UnitTests
98+
app_dir: application/CohortManager
99+
build_datetime: '${{ needs.metadata.outputs.build_datetime }}'
100+
build_timestamp: '${{ needs.metadata.outputs.build_timestamp }}'
101+
build_epoch: '${{ needs.metadata.outputs.build_epoch }}'
102+
nodejs_version: '${{ needs.metadata.outputs.nodejs_version }}'
103+
python_version: '${{ needs.metadata.outputs.python_version }}'
104+
terraform_version: '${{ needs.metadata.outputs.terraform_version }}'
105+
version: '${{ needs.metadata.outputs.version }}'
106+
analysis-stage: # Recommended maximum execution time is 5 minutes
107+
name: "Analysis stage"
108+
needs: [metadata, commit-stage, test-stage]
109+
uses: ./.github/workflows/stage-2-analyse.yaml
110+
permissions:
111+
contents: read
112+
id-token: write
113+
pull-requests: read
114+
secrets:
115+
sonar_token: ${{ secrets.SONAR_TOKEN }}
116+
with:
117+
unit_test_dir: tests/UnitTests
118+
build_datetime: "${{ needs.metadata.outputs.build_datetime }}"
119+
build_timestamp: "${{ needs.metadata.outputs.build_timestamp }}"
120+
build_epoch: "${{ needs.metadata.outputs.build_epoch }}"
121+
nodejs_version: "${{ needs.metadata.outputs.nodejs_version }}"
122+
python_version: "${{ needs.metadata.outputs.python_version }}"
123+
terraform_version: "${{ needs.metadata.outputs.terraform_version }}"
124+
version: "${{ needs.metadata.outputs.version }}"
125+
build-image-stage: # Recommended maximum execution time is 3 minutes
126+
name: "Image build stage"
127+
needs: [metadata, test-stage, analysis-stage]
128+
uses: ./.github/workflows/stage-3-build-images-devtest.yaml
129+
permissions:
130+
contents: read
131+
id-token: write
132+
pull-requests: read
133+
secrets:
134+
client_id: ${{ secrets.AZURE_CLIENT_ID }}
135+
tenant_id: ${{ secrets.AZURE_TENANT_ID }}
136+
subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
137+
acr_name: ${{ secrets.ACR_NAME }}
138+
with:
139+
docker_compose_file: application/CohortManager/compose.yaml
140+
excluded_containers_csv_list: azurite,azurite-setup,sql-server
141+
environment_tag: ${{ needs.metadata.outputs.environment_tag }}
142+
function_app_source_code_path: application/CohortManager/src
143+
project_name: cohort-manager
144+
build_all_images: true
145+
acceptance-stage: # Recommended maximum execution time is 10 minutes
146+
name: "Acceptance stage"
147+
needs: [metadata, build-image-stage]
148+
uses: ./.github/workflows/stage-4-acceptance.yaml
149+
if: needs.metadata.outputs.does_pull_request_exist == 'true' || (github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'reopened'))
150+
with:
151+
build_datetime: "${{ needs.metadata.outputs.build_datetime }}"
152+
build_timestamp: "${{ needs.metadata.outputs.build_timestamp }}"
153+
build_epoch: "${{ needs.metadata.outputs.build_epoch }}"
154+
nodejs_version: "${{ needs.metadata.outputs.nodejs_version }}"
155+
python_version: "${{ needs.metadata.outputs.python_version }}"
156+
terraform_version: "${{ needs.metadata.outputs.terraform_version }}"
157+
version: "${{ needs.metadata.outputs.version }}"
158+
deploy-stage:
159+
if: github.event_name == 'push'
160+
name: Deploy DevTest environment for commit ${{ github.sha }}
161+
needs: [metadata, build-image-stage]
162+
permissions:
163+
id-token: write
164+
contents: read
165+
uses: ./.github/workflows/stage-4-deploy-devtest.yaml
166+
secrets:
167+
client_id: ${{ secrets.AZURE_CLIENT_ID }}
168+
tenant_id: ${{ secrets.AZURE_TENANT_ID }}
169+
subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
170+
with:
171+
environments: "[\"development\"]"
172+
commit_sha: ${{ github.sha }}

‎.github/workflows/stage-3-build-images-devtest.yaml‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
name: 'Docker Image CI - devtest'
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
environment_tag:
7+
description: Environment of the deployment
8+
required: true
9+
type: string
10+
default: development
11+
docker_compose_file:
12+
description: The path of the compose.yaml file needed to build docker images
13+
required: true
14+
type: string
15+
function_app_source_code_path:
16+
description: The source path of the function app source code for the docker builds
17+
required: true
18+
type: string
19+
project_name:
20+
description: The name of the project
21+
required: true
22+
type: string
23+
excluded_containers_csv_list:
24+
description: Excluded containers in a comma separated list
25+
required: true
26+
type: string
27+
build_all_images:
28+
description: Build all images (true) or only changed ones (false)
29+
required: false
30+
type: boolean
31+
default: false
32+
33+
secrets:
34+
client_id:
35+
description: 'The Azure Client ID.'
36+
required: true
37+
tenant_id:
38+
description: 'The Azure Tenant ID.'
39+
required: true
40+
subscription_id:
41+
description: 'The Azure Subscription ID.'
42+
required: true
43+
acr_name:
44+
description: 'The name of the Azure Container Registry.'
45+
required: true
46+
47+
jobs:
48+
get-functions:
49+
runs-on: ubuntu-latest
50+
permissions:
51+
contents: read
52+
pull-requests: read
53+
id-token: write
54+
outputs:
55+
FUNC_NAMES: ${{ steps.get-function-names.outputs.FUNC_NAMES }}
56+
DOCKER_COMPOSE_DIR: ${{ steps.get-function-names.outputs.DOCKER_COMPOSE_DIR }}
57+
steps:
58+
- uses: actions/checkout@v4
59+
with:
60+
fetch-depth: 2
61+
token: ${{ secrets.GITHUB_TOKEN }}
62+
63+
- name: Checkout dtos-devops-templates repository
64+
uses: actions/checkout@v4
65+
with:
66+
repository: NHSDigital/dtos-devops-templates
67+
path: templates
68+
ref: main
69+
70+
- name: Determine which Docker container(s) to build
71+
id: get-function-names
72+
env:
73+
COMPOSE_FILES_CSV: ${{ inputs.docker_compose_file }}
74+
EXCLUDED_CONTAINERS_CSV: ${{ inputs.excluded_containers_csv_list }}
75+
SOURCE_CODE_PATH: ${{ inputs.function_app_source_code_path }}
76+
MANUAL_BUILD_ALL: ${{ inputs.build_all_images || false }}
77+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
78+
run: bash scripts/deployment/get-docker-names.sh
79+
80+
build-and-push:
81+
runs-on: ubuntu-latest
82+
permissions:
83+
id-token: write
84+
contents: read
85+
pull-requests: read
86+
needs: get-functions
87+
strategy:
88+
matrix:
89+
function: ${{ fromJSON(needs.get-functions.outputs.FUNC_NAMES) }}
90+
if: needs.get-functions.outputs.FUNC_NAMES != '[]'
91+
outputs:
92+
short_commit_hash: ${{ env.COMMIT_HASH_TAG }}
93+
devtest_pr_num_tag: ${{ env.DEVTEST_PR_NUM_TAG }}
94+
steps:
95+
- uses: actions/checkout@v4
96+
with:
97+
token: ${{ secrets.GITHUB_TOKEN }}
98+
fetch-depth: 1
99+
submodules: 'true'
100+
101+
- name: Checkout dtos-devops-templates repository
102+
uses: actions/checkout@v4
103+
with:
104+
repository: NHSDigital/dtos-devops-templates
105+
path: templates
106+
ref: main
107+
108+
- name: Az CLI login
109+
uses: azure/login@v2
110+
with:
111+
client-id: ${{ secrets.client_id }}
112+
tenant-id: ${{ secrets.tenant_id }}
113+
subscription-id: ${{ secrets.subscription_id }}
114+
115+
- name: Azure Container Registry login
116+
env:
117+
ACR_NAME: ${{ secrets.acr_name }}
118+
run: az acr login --name ${ACR_NAME}
119+
120+
- name: Create Tags
121+
env:
122+
GH_TOKEN: ${{ github.token }}
123+
ENVIRONMENT_TAG: ${{ inputs.environment_tag }}
124+
continue-on-error: false
125+
run: |
126+
echo "The branch is: ${GITHUB_REF}"
127+
128+
PR_NUM=$(gh api repos/${{ github.repository }}/commits/${{ github.sha }}/pulls --jq '.[0].number')
129+
PR_NUM_TAG="pr${PR_NUM}"
130+
131+
DEVTEST_PR_NUM_TAG="devtest_${PR_NUM_TAG}"
132+
echo "DEVTEST_PR_NUM_TAG=${DEVTEST_PR_NUM_TAG}" >> ${GITHUB_ENV}
133+
134+
SHORT_COMMIT_HASH=$(git rev-parse --short ${GITHUB_SHA})
135+
echo "COMMIT_HASH_TAG=${SHORT_COMMIT_HASH}" >> ${GITHUB_ENV}
136+
137+
echo "ENVIRONMENT_TAG=${ENVIRONMENT_TAG}" >> ${GITHUB_ENV}
138+
139+
- name: Build and Push Image
140+
working-directory: ${{ steps.get-function-names.outputs.DOCKER_COMPOSE_DIR }}
141+
continue-on-error: false
142+
env:
143+
COMPOSE_FILE: ${{ inputs.docker_compose_file }}
144+
PROJECT_NAME: ${{ inputs.project_name }}
145+
ACR_NAME: ${{ secrets.acr_name }}
146+
run: |
147+
function=${{ matrix.function }}
148+
149+
echo PROJECT_NAME: ${PROJECT_NAME}
150+
151+
if [ -z "${function}" ]; then
152+
echo "Function variable is empty. Skipping Docker build."
153+
exit 0
154+
fi
155+
156+
# Build the image
157+
docker compose -f ${COMPOSE_FILE//,/ -f } -p ${PROJECT_NAME} --profile "*" build --no-cache --pull ${function}
158+
159+
repo_name="${ACR_NAME}.azurecr.io/${PROJECT_NAME}-${function}"
160+
echo $(repo_name)
161+
162+
# Tag the image
163+
echo "Tag the image:"
164+
docker tag ${PROJECT_NAME}-${function}:latest "$repo_name:${DEVTEST_PR_NUM_TAG}"
165+
166+
# If this variable is set, the create-sbom-report.sh script will scan this docker image instead.
167+
export CHECK_DOCKER_IMAGE=${PROJECT_NAME}-${function}:latest
168+
export FORCE_USE_DOCKER=true
169+
170+
# Push the image to the repository
171+
docker push "${repo_name}:${DEVTEST_PR_NUM_TAG}"
172+
173+
- name: Cleanup the docker images
174+
env:
175+
PROJECT_NAME: ${{ inputs.project_name }}
176+
ACR_NAME: ${{ secrets.acr_name }}
177+
run: |
178+
function=${{ matrix.function }}
179+
repo_name="${ACR_NAME}.azurecr.io/${PROJECT_NAME}-${function}"
180+
181+
# Remove the images
182+
docker rmi "${repo_name}:${DEVTEST_PR_NUM_TAG}"
183+
docker rmi ${PROJECT_NAME}-${function}:latest

0 commit comments

Comments
 (0)