Skip to content

Commit f3c1bc1

Browse files
bogsi17bogsi17
authored
Create configuration for data replication (#3449)
- This produces a replicate DB with a single ECS task connected - Useful for testing migrations/sql/etc on a production data without actually modifying the production system - The replicate system is completely ignorant of source db/service - The replicate DB will be spawned from a snapshot which is passed as a variable - Some modification was done to the ecs module to ensure that task definitions are distinct
2 parents 8bf4bba + 6cfbc47 commit f3c1bc1

26 files changed

Lines changed: 705 additions & 7 deletions

‎.github/workflows/data-replication-pipeline.yml‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
name: Data replication pipeline
2+
run-name: ${{ inputs.action }} data replication resources for ${{ inputs.environment }}
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
environment:
8+
description: Deployment environment
9+
required: true
10+
type: choice
11+
options:
12+
- training
13+
- production
14+
- test
15+
- qa
16+
- sandbox-alpha
17+
- sandbox-beta
18+
image_tag:
19+
description: Docker image tag to deploy
20+
required: false
21+
type: string
22+
action:
23+
description: Action to perform on data replication env
24+
required: true
25+
type: choice
26+
options:
27+
- Destroy
28+
- Recreate
29+
default: Recreate
30+
db_snapshot_arn:
31+
description: ARN of the DB snapshot to use (optional)
32+
required: false
33+
type: string
34+
35+
env:
36+
aws_role: ${{ inputs.environment == 'production'
37+
&& 'arn:aws:iam::820242920762:role/GithubDeployDataReplicationInfrastructure'
38+
|| 'arn:aws:iam::393416225559:role/GithubDeployDataReplicationInfrastructure' }}
39+
40+
defaults:
41+
run:
42+
working-directory: terraform/data_replication
43+
44+
concurrency:
45+
group: deploy-data-replica-${{ inputs.environment }}
46+
47+
jobs:
48+
prepare:
49+
name: Prepare data replica
50+
runs-on: ubuntu-latest
51+
permissions:
52+
id-token: write
53+
steps:
54+
- name: Checkout code
55+
uses: actions/checkout@v4
56+
- name: Configure AWS Credentials
57+
uses: aws-actions/configure-aws-credentials@v4
58+
with:
59+
role-to-assume: ${{ env.aws_role }}
60+
aws-region: eu-west-2
61+
- name: get latest snapshot
62+
id: get-latest-snapshot
63+
run: |
64+
set -e
65+
if [ -z "${{ inputs.db_snapshot_arn }}" ]; then
66+
echo "No snapshot ARN provided, fetching the latest snapshot"
67+
SNAPSHOT_ARN=$(aws rds describe-db-cluster-snapshots \
68+
--query 'DBClusterSnapshots[?contains(DBClusterSnapshotIdentifier, `${{ inputs.environment}}`)].[DBClusterSnapshotArn, SnapshotCreateTime]' \
69+
--output text | sort -k2 -r | head -n 1 | cut -f1)
70+
else
71+
echo "Using provided snapshot ARN: ${{ inputs.db_snapshot_arn }}"
72+
SNAPSHOT_ARN="${{ inputs.db_snapshot_arn }}"
73+
fi
74+
echo "SNAPSHOT_ARN=$SNAPSHOT_ARN" >> $GITHUB_OUTPUT
75+
- name: Install terraform
76+
uses: hashicorp/setup-terraform@v3
77+
with:
78+
terraform_version: 1.10.5
79+
- name: Get db secret arn
80+
id: get-db-secret-arn
81+
working-directory: terraform/app
82+
run: |
83+
terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
84+
DB_SECRET_ARN=$(terraform output --raw db_secret_arn)
85+
echo "DB_SECRET_ARN=$DB_SECRET_ARN" >> $GITHUB_OUTPUT
86+
- name: ECR login
87+
id: login-ecr
88+
uses: aws-actions/amazon-ecr-login@v2
89+
- name: Get docker image digest
90+
id: get-docker-image-digest
91+
run: |
92+
set -e
93+
DOCKER_IMAGE="${{ steps.login-ecr.outputs.registry }}/mavis/webapp:${{ inputs.image_tag || github.sha }}"
94+
docker pull "$DOCKER_IMAGE"
95+
DOCKER_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$DOCKER_IMAGE")
96+
DIGEST="${DOCKER_DIGEST#*@}"
97+
echo "DIGEST=$DIGEST" >> $GITHUB_OUTPUT
98+
outputs:
99+
SNAPSHOT_ARN: ${{ steps.get-latest-snapshot.outputs.SNAPSHOT_ARN }}
100+
DB_SECRET_ARN: ${{ steps.get-db-secret-arn.outputs.DB_SECRET_ARN }}
101+
DOCKER_DIGEST: ${{ steps.get-docker-image-digest.outputs.DIGEST }}
102+
103+
destroy:
104+
name: Destroy data replication infrastructure
105+
runs-on: ubuntu-latest
106+
environment: ${{ inputs.environment }}
107+
permissions:
108+
id-token: write
109+
steps:
110+
- name: Checkout code
111+
uses: actions/checkout@v4
112+
- name: Configure AWS Credentials
113+
uses: aws-actions/configure-aws-credentials@v4
114+
with:
115+
role-to-assume: ${{ env.aws_role }}
116+
aws-region: eu-west-2
117+
- name: Install terraform
118+
uses: hashicorp/setup-terraform@v3
119+
with:
120+
terraform_version: 1.10.5
121+
- name: Terraform Destroy
122+
id: destroy
123+
run: |
124+
set -e
125+
terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
126+
terraform destroy -var-file="env/${{ inputs.environment }}.tfvars" -var="image_digest=filler_value" \
127+
-var="db_secret_arn=filler_value" -var="imported_snapshot=filler_value" -auto-approve
128+
129+
plan:
130+
if: ${{ inputs.action == 'Recreate' }}
131+
name: Terraform plan
132+
runs-on: ubuntu-latest
133+
needs:
134+
- prepare
135+
- destroy
136+
env:
137+
SNAPSHOT_ARN: ${{ needs.prepare.outputs.SNAPSHOT_ARN }}
138+
DB_SECRET_ARN: ${{ needs.prepare.outputs.DB_SECRET_ARN }}
139+
DOCKER_DIGEST: ${{ needs.prepare.outputs.DOCKER_DIGEST }}
140+
permissions:
141+
id-token: write
142+
steps:
143+
- name: Checkout code
144+
uses: actions/checkout@v4
145+
- name: Configure AWS Credentials
146+
uses: aws-actions/configure-aws-credentials@v4
147+
with:
148+
role-to-assume: ${{ env.aws_role }}
149+
aws-region: eu-west-2
150+
- name: Install terraform
151+
uses: hashicorp/setup-terraform@v3
152+
with:
153+
terraform_version: 1.10.5
154+
- name: Terraform Plan
155+
id: plan
156+
run: |
157+
set -e
158+
terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
159+
terraform plan -var="image_digest=${{ env.DOCKER_DIGEST }}" -var="db_secret_arn=${{ env.DB_SECRET_ARN }}" \
160+
-var="imported_snapshot=${{ env.SNAPSHOT_ARN }}" -var-file="env/${{ inputs.environment }}.tfvars" \
161+
-out ${{ runner.temp }}/tfplan | tee ${{ runner.temp }}/tf_stdout
162+
- name: Upload artifact
163+
uses: actions/upload-artifact@v4
164+
with:
165+
name: tfplan_infrastructure-${{ inputs.environment }}
166+
path: ${{ runner.temp }}/tfplan
167+
168+
apply:
169+
name: Terraform apply
170+
runs-on: ubuntu-latest
171+
needs: plan
172+
environment: ${{ inputs.environment }}
173+
permissions:
174+
id-token: write
175+
steps:
176+
- name: Checkout code
177+
uses: actions/checkout@v4
178+
- name: Configure AWS Credentials
179+
uses: aws-actions/configure-aws-credentials@v4
180+
with:
181+
role-to-assume: ${{ env.aws_role }}
182+
aws-region: eu-west-2
183+
- name: Download artifact
184+
uses: actions/download-artifact@v4
185+
with:
186+
name: tfplan_infrastructure-${{ inputs.environment }}
187+
path: ${{ runner.temp }}
188+
- name: Install terraform
189+
uses: hashicorp/setup-terraform@v3
190+
with:
191+
terraform_version: 1.10.5
192+
- name: Apply the changes
193+
run: |
194+
set -e
195+
terraform init -backend-config="env/${{ inputs.environment }}-backend.hcl" -upgrade
196+
terraform apply ${{ runner.temp }}/tfplan

‎bin/docker-start‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@ if [ "$SERVER_TYPE" == "web" ]; then
88
elif [ "$SERVER_TYPE" == "good-job" ]; then
99
echo "Starting good-job server..."
1010
exec "$BIN_DIR"/good_job start
11+
elif [ "$SERVER_TYPE" == "none" ]; then
12+
echo "No server started"
13+
exec tail -f /dev/null # Keep container running
1114
else
12-
echo "SERVER_TYPE variable: '$SERVER_TYPE' unknown. Allowed values ['web','good-job']"
15+
echo "SERVER_TYPE variable: '$SERVER_TYPE' unknown. Allowed values ['web','good-job', 'none']"
1316
exit 1
1417
fi

‎terraform/app/modules/ecs_service/autoscaling.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ resource "aws_appautoscaling_target" "this" {
1010

1111
resource "aws_appautoscaling_policy" "this" {
1212
for_each = local.autoscaling_enabled ? var.autoscaling_policies : {}
13-
name = "${var.server_type}-${each.key}-scaling-${var.environment}"
13+
name = "${local.server_type_name}-${each.key}-scaling-${var.environment}"
1414
policy_type = "TargetTrackingScaling"
1515
resource_id = aws_appautoscaling_target.this[0].resource_id
1616
scalable_dimension = aws_appautoscaling_target.this[0].scalable_dimension

‎terraform/app/modules/ecs_service/main.tf‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ terraform {
99
}
1010

1111
resource "aws_security_group" "this" {
12-
name = "${var.server_type}-service-${var.environment}"
12+
name = "${local.server_type_name}-service-${var.environment}"
1313
description = "Security Group for communication with ECS Service"
1414
vpc_id = var.network_params.vpc_id
1515
lifecycle {
@@ -27,7 +27,7 @@ resource "aws_security_group_rule" "egress_all" {
2727
}
2828

2929
resource "aws_ecs_service" "this" {
30-
name = "mavis-${var.environment}-${var.server_type}"
30+
name = "mavis-${var.environment}-${local.server_type_name}"
3131
cluster = var.cluster_id
3232
task_definition = aws_ecs_task_definition.this.arn
3333
desired_count = var.minimum_replica_count
@@ -70,7 +70,7 @@ resource "aws_ecs_service" "this" {
7070
}
7171

7272
resource "aws_ecs_task_definition" "this" {
73-
family = "mavis-${var.server_type}-task-definition-${var.environment}"
73+
family = "mavis-${local.server_type_name}-task-definition-${var.environment}"
7474
requires_compatibilities = ["FARGATE"]
7575
network_mode = "awsvpc"
7676
cpu = var.task_config.cpu
@@ -95,7 +95,7 @@ resource "aws_ecs_task_definition" "this" {
9595
options = {
9696
awslogs-group = var.task_config.log_group_name
9797
awslogs-region = var.task_config.region
98-
awslogs-stream-prefix = "${var.environment}-${var.server_type}-logs"
98+
awslogs-stream-prefix = "${var.environment}-${local.server_type_name}-logs"
9999
}
100100
}
101101
healthCheck = {

‎terraform/app/modules/ecs_service/variables.tf‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,17 @@ variable "environment" {
66

77
variable "server_type" {
88
type = string
9-
description = "Type of server to be deployed. This is set as an environment variable in the main container, and is used to determine how the application is launched"
9+
description = "Type of server to be deployed. This is set as an environment variable in the main container, and is used to determine how the application is launched."
1010
nullable = false
1111
}
1212

13+
variable "server_type_name" {
14+
type = string
15+
description = "Name of the server type to be deployed."
16+
default = null
17+
nullable = true
18+
}
19+
1320
variable "minimum_replica_count" {
1421
type = number
1522
description = "Minimum amount of allowed replicas for the service. Also the replica count when creating th service."
@@ -109,4 +116,5 @@ variable "container_name" {
109116

110117
locals {
111118
autoscaling_enabled = var.maximum_replica_count > var.minimum_replica_count
119+
server_type_name = var.server_type_name != null ? var.server_type_name : var.server_type
112120
}

‎terraform/data_replication/README.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Data replication module
2+
3+
## Overview
4+
5+
This module can be used to verify data migration tasks on a copy of actual production data before running them on production.
6+
7+
It creates
8+
9+
- A replication of a given database based on a provided snapshot
10+
- A dedicated ECS service connected to the database
11+
12+
## Setup
13+
14+
This module is managed via a GitHub Actions workflow. To separate it from the rest of the infrastructure, the workflow uses a dedicated IAM role. To set up everything from scratch, manually create the role
15+
`GithubDeployDataReplicationInfrastructure` based on the policy template `github_data_replication_actions_policy.json` and the trust policy `github_role_<ENVIRONMENT>_trust_policy.json`.
16+
17+
## Usage
18+
19+
### Manage the database replication infrastructure
20+
21+
To create the infrastructure, run the `data-replication-pipeline.yml` workflow and select the 'Recreate' option.
22+
This will destroy any existing replication infrastructure and create a new replicated database from the latest snapshot.
23+
24+
To destroy the resources, run the `data-replication-pipeline.yml` with the 'Destroy' option.
25+
26+
### Connect to the dedicated ECS task
27+
28+
To connect to the dedicated ECS task, run
29+
30+
```
31+
./script/shell.sh <ENV>-data-replication
32+
```

‎terraform/data_replication/ecs.tf‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
2+
resource "aws_ecs_cluster" "cluster" {
3+
name = local.name_prefix
4+
5+
setting {
6+
name = "containerInsights"
7+
value = "enabled"
8+
}
9+
}
10+
11+
resource "aws_cloudwatch_log_group" "ecs_log_group" {
12+
name = "${local.name_prefix}-ecs"
13+
retention_in_days = 14
14+
skip_destroy = false
15+
}
16+
17+
18+
module "db_access_service" {
19+
source = "../app/modules/ecs_service"
20+
cluster_id = aws_ecs_cluster.cluster.id
21+
cluster_name = aws_ecs_cluster.cluster.name
22+
environment = var.environment
23+
maximum_replica_count = 1
24+
minimum_replica_count = 1
25+
network_params = {
26+
subnets = local.subnet_list
27+
vpc_id = aws_vpc.vpc.id
28+
}
29+
server_type = "none"
30+
server_type_name = "data-replication"
31+
task_config = {
32+
environment = local.task_envs
33+
secrets = local.task_secrets
34+
cpu = 1024
35+
memory = 2048
36+
docker_image = "${var.account_id}.dkr.ecr.eu-west-2.amazonaws.com/${var.docker_image}@${var.image_digest}"
37+
execution_role_arn = aws_iam_role.ecs_task_execution_role.arn
38+
task_role_arn = aws_iam_role.ecs_task_role.arn
39+
log_group_name = aws_cloudwatch_log_group.ecs_log_group.name
40+
region = var.region
41+
health_check_command = ["CMD-SHELL", "echo 'alive' || exit 1"]
42+
}
43+
depends_on = [aws_rds_cluster_instance.instance]
44+
}

‎terraform/data_replication/env/production-backend.hcl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
bucket = "nhse-mavis-terraform-state-production"
2+
key = "terraform-data-replication-production.tfstate"
3+
region = "eu-west-2"
4+
dynamodb_table = "mavis-terraform-state-lock"

‎terraform/data_replication/env/production.tfvars‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
environment = "production"
2+
rails_env = "production"
3+
rails_master_key_path = "/copilot/mavis/production/secrets/RAILS_MASTER_KEY"
4+
max_aurora_capacity_units = 16

‎terraform/data_replication/env/qa-backend.hcl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
bucket = "nhse-mavis-terraform-state"
2+
key = "terraform-data-replication-qa.tfstate"
3+
region = "eu-west-2"
4+
dynamodb_table = "mavis-terraform-state-lock"

0 commit comments

Comments
 (0)