Skip to content

Commit 6a62bce

Browse files
TheOneFromNorwayTheOneFromNorway
authored
Merge pull request #6362 from NHSDigital/e2e-tests-on-fork-pr
E2E tests on fork PR
2 parents 08c20a3 + eb43b37 commit 6a62bce

4 files changed

Lines changed: 501 additions & 381 deletions

File tree

‎.github/workflows/end-to-end-tests-aws.yml‎

Lines changed: 377 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,377 @@
1+
name: AWS deployment E2E tests
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
git_reference_for_application_image:
7+
description: The git reference for deploying containerized mavis application
8+
type: string
9+
required: true
10+
git_reference_for_database_image:
11+
description: The environment to build the base image against
12+
type: string
13+
required: false
14+
default: next
15+
secrets:
16+
HTTP_AUTH_TOKEN_FOR_TESTS:
17+
description: HTTP Basic Auth token for the environment under test
18+
required: true
19+
MAVIS_TESTING_REPO_ACCESS_TOKEN:
20+
description: Access token for the manage-vaccinations-in-schools-testing repository
21+
required: true
22+
23+
permissions: {}
24+
25+
jobs:
26+
check-development-image-presence:
27+
name: Check if mavis docker image already exists
28+
runs-on: ubuntu-latest
29+
permissions:
30+
id-token: write
31+
outputs:
32+
build-needed: ${{ steps.check-image.outputs.build-needed }}
33+
application-image-git-ref: ${{ steps.check-image.outputs.GIT_REF_SHA }}
34+
steps:
35+
- name: Configure AWS Credentials
36+
uses: aws-actions/configure-aws-credentials@v6
37+
with:
38+
role-to-assume: arn:aws:iam::393416225559:role/GitHubAssuranceTestRole
39+
aws-region: eu-west-2
40+
- uses: actions/checkout@v6
41+
with:
42+
fetch-depth: 0
43+
repository: nhsuk/manage-vaccinations-in-schools
44+
- name: Check if image exists
45+
id: check-image
46+
env:
47+
GIT_REF: >-
48+
${{ format('origin/{0}', inputs.git_reference_for_application_image) }}
49+
run: |
50+
GIT_REF_SHA=$(git rev-parse "$GIT_REF")
51+
echo "GIT_REF_SHA=$GIT_REF_SHA" >> "$GITHUB_OUTPUT"
52+
if aws ecr describe-images \
53+
--repository-name mavis/development \
54+
--image-ids "imageTag=$GIT_REF_SHA" > /dev/null 2>&1; then
55+
echo "Docker image with given tag already exists"
56+
echo "build-needed=false" >> "$GITHUB_OUTPUT"
57+
else
58+
echo "Docker image does not exist. Build needed"
59+
echo "build-needed=true" >> "$GITHUB_OUTPUT"
60+
fi
61+
build-and-push-development-image:
62+
needs: [check-development-image-presence]
63+
if: ${{ !cancelled() && needs.check-development-image-presence.outputs.build-needed == 'true' }}
64+
runs-on: ubuntu-latest
65+
permissions:
66+
id-token: write
67+
steps:
68+
- name: Checkout code
69+
uses: actions/checkout@v6
70+
with:
71+
ref: ${{ needs.check-development-image-presence.outputs.application-image-git-ref }}
72+
repository: nhsuk/manage-vaccinations-in-schools
73+
- name: Configure AWS Credentials
74+
uses: aws-actions/configure-aws-credentials@v6
75+
with:
76+
role-to-assume: arn:aws:iam::393416225559:role/GitHubAssuranceTestRole
77+
aws-region: eu-west-2
78+
- name: Login to ECR
79+
id: login-ecr
80+
uses: aws-actions/amazon-ecr-login@v2
81+
- name: Build and push mavis/development docker image
82+
# yamllint disable rule:line-length
83+
run: |
84+
docker build \
85+
--build-arg BUNDLE_WITHOUT=test \
86+
--build-arg RAILS_ENV=development \
87+
-t "393416225559.dkr.ecr.eu-west-2.amazonaws.com/mavis/development:${{ needs.check-development-image-presence.outputs.application-image-git-ref }}" \
88+
.
89+
docker push "393416225559.dkr.ecr.eu-west-2.amazonaws.com/mavis/development:${{ needs.check-development-image-presence.outputs.application-image-git-ref }}"
90+
# yamllint enable rule:line-length
91+
check-database-image-presence:
92+
name: Check if database docker image already exists
93+
runs-on: ubuntu-latest
94+
permissions:
95+
id-token: write
96+
outputs:
97+
build-needed: ${{ steps.check-image.outputs.build-needed }}
98+
db_git_ref_sha: ${{ steps.check-image.outputs.GIT_REF_SHA }}
99+
steps:
100+
- name: Configure AWS Credentials
101+
uses: aws-actions/configure-aws-credentials@v6
102+
with:
103+
role-to-assume: arn:aws:iam::393416225559:role/GitHubAssuranceTestRole
104+
aws-region: eu-west-2
105+
- uses: actions/checkout@v6
106+
with:
107+
fetch-depth: 0
108+
repository: nhsuk/manage-vaccinations-in-schools
109+
- name: Check if image exists
110+
id: check-image
111+
env:
112+
GIT_REF: >-
113+
${{ format('origin/{0}', inputs.git_reference_for_database_image) }}
114+
run: |
115+
GIT_REF_SHA=$(git rev-parse "$GIT_REF")
116+
echo "GIT_REF_SHA=$GIT_REF_SHA" >> "$GITHUB_OUTPUT"
117+
if aws ecr describe-images \
118+
--repository-name mavis/development/postgres_db \
119+
--image-ids "imageTag=$GIT_REF_SHA" > /dev/null 2>&1; then
120+
echo "Docker image with given tag already exists"
121+
echo "build-needed=false" >> "$GITHUB_OUTPUT"
122+
else
123+
echo "Docker image does not exist. Build needed"
124+
echo "build-needed=true" >> "$GITHUB_OUTPUT"
125+
fi
126+
build-and-push-database-image:
127+
needs: [check-database-image-presence]
128+
if: ${{ !cancelled() && needs.check-database-image-presence.outputs.build-needed == 'true' }}
129+
permissions:
130+
id-token: write
131+
contents: read
132+
uses: ./.github/workflows/create_dockerized_db.yml
133+
with:
134+
github_ref: ${{ needs.check-database-image-presence.outputs.db_git_ref_sha }}
135+
launch-dockerized-devimage:
136+
needs:
137+
- check-development-image-presence
138+
- build-and-push-development-image
139+
- check-database-image-presence
140+
- build-and-push-database-image
141+
if: >-
142+
${{ !cancelled() && (needs.build-and-push-development-image.result == 'success' ||
143+
(needs.check-development-image-presence.result == 'success' &&
144+
needs.build-and-push-development-image.result == 'skipped')) &&
145+
(needs.build-and-push-database-image.result == 'success' ||
146+
(needs.check-database-image-presence.result == 'success' &&
147+
needs.build-and-push-database-image.result == 'skipped')) }}
148+
runs-on: ubuntu-latest
149+
permissions:
150+
id-token: write
151+
outputs:
152+
run_task_arn: ${{ steps.run-task.outputs.run-task-arn }}
153+
steps:
154+
- name: Configure AWS credentials
155+
uses: aws-actions/configure-aws-credentials@v6
156+
with:
157+
role-to-assume: arn:aws:iam::393416225559:role/GitHubAssuranceTestRole
158+
aws-region: eu-west-2
159+
- name: Render task definition web
160+
id: render-task-definition-web
161+
uses: aws-actions/amazon-ecs-render-task-definition@v1
162+
with:
163+
task-definition-family: assurance-testing-mavis-development-task-definition-template
164+
container-name: mavis-development-web
165+
# yamllint disable-line rule:line-length
166+
image:
167+
393416225559.dkr.ecr.eu-west-2.amazonaws.com/mavis/development:${{
168+
needs.check-development-image-presence.outputs.application-image-git-ref }}
169+
- name: Render task definition database
170+
id: render-task-definition-database
171+
uses: aws-actions/amazon-ecs-render-task-definition@v1
172+
with:
173+
task-definition: ${{ steps.render-task-definition-web.outputs.task-definition }}
174+
container-name: mavis-development-db
175+
# yamllint disable-line rule:line-length
176+
image:
177+
393416225559.dkr.ecr.eu-west-2.amazonaws.com/mavis/development/postgres_db:${{
178+
needs.check-database-image-presence.outputs.db_git_ref_sha }}
179+
- name: Render task definition sidekiq
180+
id: render-task-definition-sidekiq
181+
uses: aws-actions/amazon-ecs-render-task-definition@v1
182+
with:
183+
task-definition: ${{ steps.render-task-definition-database.outputs.task-definition }}
184+
container-name: mavis-development-sidekiq
185+
# yamllint disable-line rule:line-length
186+
image:
187+
393416225559.dkr.ecr.eu-west-2.amazonaws.com/mavis/development:${{
188+
needs.check-development-image-presence.outputs.application-image-git-ref }}
189+
- name: Prepare deployment
190+
id: prepare-deployment
191+
run: |
192+
file_path="assurance-testing-mavis-development-task-definition.json"
193+
family_name="assurance-testing-mavis-development-task-definition"
194+
jq --arg f "$family_name" '.family = $f' \
195+
"${{ steps.render-task-definition-sidekiq.outputs.task-definition }}" > "$file_path"
196+
197+
subnet_id=$(aws ec2 describe-subnets \
198+
--filters Name=tag:Name,Values=assurance-testing-subnet \
199+
--query 'Subnets[0].SubnetId' --output text)
200+
security_group_id=$(aws ec2 describe-security-groups \
201+
--filters Name=group-name,Values=assurance-testing-mavis-development-sg \
202+
--query 'SecurityGroups[0].GroupId' --output text)
203+
204+
echo "run-task-subnets=$subnet_id" >> "$GITHUB_OUTPUT"
205+
echo "run-task-security-groups=$security_group_id" >> "$GITHUB_OUTPUT"
206+
- name: Deploy task definition
207+
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
208+
id: run-task
209+
with:
210+
task-definition: "assurance-testing-mavis-development-task-definition.json"
211+
cluster: assurance-testing
212+
run-task: true
213+
run-task-subnets: ${{ steps.prepare-deployment.outputs.run-task-subnets }}
214+
run-task-launch-type: "FARGATE"
215+
run-task-security-groups: ${{ steps.prepare-deployment.outputs.run-task-security-groups }}
216+
run-task-assign-public-IP: ENABLED
217+
wait-for-task-stability:
218+
needs: launch-dockerized-devimage
219+
if: ${{ !cancelled() && needs.launch-dockerized-devimage.result == 'success'}}
220+
runs-on: ubuntu-latest
221+
permissions:
222+
id-token: write
223+
outputs:
224+
task_arn: ${{ steps.compile-outputs.outputs.task_arn }}
225+
container_ip: ${{ steps.compile-outputs.outputs.container_ip }}
226+
steps:
227+
- name: Configure AWS credentials
228+
uses: aws-actions/configure-aws-credentials@v6
229+
with:
230+
role-to-assume: arn:aws:iam::393416225559:role/GitHubAssuranceTestRole
231+
aws-region: eu-west-2
232+
# yamllint disable rule:line-length
233+
- name: Get container logs
234+
run: |
235+
TASK_ARN=$(echo '${{ needs.launch-dockerized-devimage.outputs.run_task_arn }}' | jq -r '.[0]')
236+
# shellcheck disable=SC2001
237+
TASK_ID=$(sed 's:^.*/::' <<< "$TASK_ARN")
238+
WEB_CLOUDWATCH_URL="https://eu-west-2.console.aws.amazon.com/cloudwatch/home?region=eu-west-2#logsV2:log-groups/log-group/assurance-testing-ecs/log-events/assurance-testing-logs\$252Fmavis-development-web\$252F${TASK_ID}\$3Fstart\$3D$(date +%s)000\$26end\$3D$(date -d '+30 minutes' +%s)000"
239+
SIDEKIQ_CLOUDWATCH_URL="https://eu-west-2.console.aws.amazon.com/cloudwatch/home?region=eu-west-2#logsV2:log-groups/log-group/assurance-testing-ecs/log-events/assurance-testing-logs\$252Fmavis-development-sidekiq\$252F${TASK_ID}\$3Fstart\$3D$(date +%s)000\$26end\$3D$(date -d '+30 minutes' +%s)000"
240+
241+
{
242+
echo "**Task ID:** $TASK_ID"
243+
echo "**Container Logs:** "
244+
echo " * WEB logs: $WEB_CLOUDWATCH_URL"
245+
echo " * SIDEKIQ logs: $SIDEKIQ_CLOUDWATCH_URL"
246+
} >> "$GITHUB_STEP_SUMMARY"
247+
248+
echo "Logs for server: [web](${WEB_CLOUDWATCH_URL}) and [sidekiq](${SIDEKIQ_CLOUDWATCH_URL})"
249+
- name: Wait for task to stabilise
250+
run: |
251+
set -euo pipefail
252+
253+
ELAPSED=0
254+
POLL_INTERVAL=10
255+
TASK_ARN=$(echo '${{ needs.launch-dockerized-devimage.outputs.run_task_arn }}' | jq -r '.[0]')
256+
echo "Waiting for ECS task $TASK_ARN to become HEALTHY"
257+
258+
while (( ELAPSED < 300 )); do
259+
HEALTH_STATUS=$(aws ecs describe-tasks \
260+
--cluster "assurance-testing" \
261+
--tasks "$TASK_ARN" \
262+
--query 'tasks[0].healthStatus' \
263+
--output text 2>/dev/null || echo "UNKNOWN")
264+
265+
LAST_EVENT=$(aws ecs describe-tasks \
266+
--cluster "assurance-testing" \
267+
--tasks "$TASK_ARN" \
268+
--query 'tasks[0].lastStatus' \
269+
--output text 2>/dev/null || echo "UNKNOWN")
270+
271+
case "$HEALTH_STATUS" in
272+
"HEALTHY")
273+
echo "Task is HEALTHY"
274+
exit 0
275+
;;
276+
"UNHEALTHY")
277+
echo "Task reported UNHEALTHY"
278+
aws ecs describe-tasks --cluster "assurance-testing" --tasks "$TASK_ARN" --query 'tasks[0].containers[].{Name:name,Health:health,Reason:lastStatusReason}' --output table
279+
exit 1
280+
;;
281+
"UNKNOWN"|"")
282+
# Task might not exist or API error
283+
if [[ "$LAST_EVENT" == "STOPPED" || "$LAST_EVENT" == "DEACTIVATED" ]]; then
284+
echo "Task is no longer running (lastStatus: $LAST_EVENT)"
285+
exit 1
286+
fi
287+
echo "Task not found or not reporting health yet (healthStatus: UNKNOWN, lastStatus: $LAST_EVENT). Retrying..."
288+
;;
289+
*)
290+
# HEALTHY/UNHEALTHY not reported yet (common in early lifecycle)
291+
echo "Task health check in progress... (healthStatus: $HEALTH_STATUS, lastStatus: $LAST_EVENT)"
292+
;;
293+
esac
294+
295+
sleep "$POLL_INTERVAL"
296+
ELAPSED=$((ELAPSED + POLL_INTERVAL))
297+
done
298+
299+
echo "Timeout reached (300 seconds) while waiting for task to become HEALTHY"
300+
echo "Final status:"
301+
aws ecs describe-tasks \
302+
--cluster "assurance-testing" \
303+
--tasks "$TASK_ARN" \
304+
--query 'tasks[0]' \
305+
--output json || true
306+
exit 1
307+
- name: Compile outputs
308+
id: compile-outputs
309+
run: |
310+
TASK_ARN=$(echo '${{ needs.launch-dockerized-devimage.outputs.run_task_arn }}' | jq -r '.[0]')
311+
NETWORK_INTERFACE=$(aws ecs describe-tasks \
312+
--cluster assurance-testing \
313+
--tasks "$TASK_ARN" \
314+
| jq -r '.tasks[0].attachments[0].details[] | select(.name == "networkInterfaceId") | .value')
315+
CONTAINER_IP=$(aws ec2 describe-network-interfaces \
316+
--network-interface-ids "$NETWORK_INTERFACE" \
317+
--query 'NetworkInterfaces[0].Association.PublicIp' \
318+
--output text)
319+
echo "container_ip=$CONTAINER_IP" >> "$GITHUB_OUTPUT"
320+
echo "task_arn=$TASK_ARN" >> "$GITHUB_OUTPUT"
321+
echo "Started task: $TASK_ARN"
322+
# yamllint enable rule:line-length
323+
find-correct-test-branch:
324+
needs: [wait-for-task-stability]
325+
if: ${{ !cancelled() && needs.wait-for-task-stability.result == 'success' }}
326+
runs-on: ubuntu-latest
327+
permissions:
328+
contents: read
329+
outputs:
330+
test_branch: ${{ steps.check-branch.outputs.test_branch }}
331+
steps:
332+
- name: Check if head branch or base branch exists
333+
id: check-branch
334+
run: |
335+
if git ls-remote --exit-code \
336+
--heads https://github.com/NHSDigital/manage-vaccinations-in-schools-testing.git \
337+
"$HEAD_REF" > /dev/null 2>&1; then
338+
echo "test_branch=$HEAD_REF" >> "$GITHUB_OUTPUT"
339+
elif git ls-remote --exit-code \
340+
--heads https://github.com/NHSDigital/manage-vaccinations-in-schools-testing.git \
341+
"$BASE_REF" > /dev/null 2>&1; then
342+
echo "test_branch=$BASE_REF" >> "$GITHUB_OUTPUT"
343+
else
344+
echo "test_branch=main" >> "$GITHUB_OUTPUT"
345+
fi
346+
env:
347+
HEAD_REF: ${{ github.head_ref }}
348+
BASE_REF: ${{ github.base_ref }}
349+
call-end-to-end-tests:
350+
needs: [launch-dockerized-devimage, wait-for-task-stability, find-correct-test-branch]
351+
if:
352+
${{ !cancelled() && needs.launch-dockerized-devimage.result == 'success' &&
353+
needs.wait-for-task-stability.result == 'success'}}
354+
uses: ./.github/workflows/call-end-to-end-tests.yml
355+
permissions:
356+
contents: write
357+
id-token: write
358+
with:
359+
cross_service_tests: false
360+
github_ref: ${{ needs.find-correct-test-branch.outputs.test_branch }}
361+
endpoint: http://${{ needs.wait-for-task-stability.outputs.container_ip }}:4000
362+
stop-docker-environment:
363+
needs: [call-end-to-end-tests, launch-dockerized-devimage, wait-for-task-stability]
364+
if: ${{ always() && needs.launch-dockerized-devimage.result != 'skipped'}}
365+
runs-on: ubuntu-latest
366+
permissions:
367+
id-token: write
368+
steps:
369+
- name: Configure AWS credentials
370+
uses: aws-actions/configure-aws-credentials@v6
371+
with:
372+
role-to-assume: arn:aws:iam::393416225559:role/GitHubAssuranceTestRole
373+
aws-region: eu-west-2
374+
- name: Stop dockerized dev image
375+
run: >-
376+
aws ecs stop-task --cluster assurance-testing --task ${{
377+
needs.wait-for-task-stability.outputs.task_arn }}

0 commit comments

Comments
 (0)