Skip to content

Commit e8499d5

Browse files
rfk-ncrfk-nc
authored
feat: DTOSS-9884 Schedule Immutable Backups in Prod (#1535)
* Add Immutable backup and restore container jobs * Add retry limit * Resolve SonarCloud root user warnings * Resolve SonarCloud warnings * Resolve SonarCloud warnings
1 parent 0c189e0 commit e8499d5

16 files changed

Lines changed: 673 additions & 27 deletions

File tree

‎.azuredevops/pipelines/task_azure_sql_backup_prod.yaml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
---
2+
3+
name: $(Build.SourceBranchName)-$(Date:yyyyMMdd)_$(Rev:r)
4+
trigger: none
5+
pr: none
6+
7+
schedules:
8+
- cron: "0 2 * * *" # Run daily at 2:00 AM UTC
9+
displayName: 'Run Backup'
10+
branches:
11+
include:
12+
- main
13+
always: true
14+
15+
resources:
16+
repositories:
17+
- repository: dtos-devops-templates
18+
type: github
19+
name: NHSDigital/dtos-devops-templates
20+
ref: 34a7e5c3072bddaa350804905c60b9c8e7ae7191
21+
endpoint: NHSDigital
22+
23+
variables:
24+
- name: hostPoolName
25+
value: private-pool-prod-uks
26+
- group: PROD_core_backend
27+
- group: PROD_image_pipelines
28+
- name: TF_VERSION
29+
value: 1.11.4
30+
- name: TF_PLAN_ARTIFACT
31+
value: tf_plan_core_PROD
32+
- name: TF_DIRECTORY
33+
value: $(System.DefaultWorkingDirectory)/$(System.TeamProject)/infrastructure/tf-core
34+
- name: ENVIRONMENT
35+
value: production
36+
37+
stages:
38+
- stage: db_backup_stage
39+
displayName: Database backup stage
40+
pool:
41+
name: $(hostPoolName)
42+
jobs:
43+
- job: db_changes
44+
displayName: Create Database Backup and Upload to Storage Account
45+
steps:
46+
- checkout: self
47+
- checkout: dtos-devops-templates
48+
- template: .azuredevops/templates/steps/app-container-job-start.yaml@dtos-devops-templates
49+
parameters:
50+
serviceConnection: $(SERVICE_CONNECTION)
51+
targetSubscriptionId: $(TF_VAR_TARGET_SUBSCRIPTION_ID)
52+
resourceGroupName: $(RESOURCE_GROUP_NAME_SQL)
53+
jobName: ca-db-backup-uksouth

‎infrastructure/tf-audit/environments/production.tfvars‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,4 +72,24 @@ storage_accounts = {
7272
}
7373
}
7474
}
75+
sqlbackups = {
76+
name_suffix = "sqlbackups"
77+
account_tier = "Standard"
78+
replication_type = "GRS"
79+
public_network_access_enabled = false
80+
blob_properties_delete_retention_policy = 28
81+
blob_properties_versioning_enabled = true
82+
containers = {
83+
sql-backups-immutable = {
84+
container_name = "sql-backups-immutable"
85+
container_access_type = "private"
86+
immutability_policy = {
87+
is_locked = false
88+
immutability_period_in_days = 1
89+
protected_append_writes_all_enabled = false
90+
protected_append_writes_enabled = false
91+
}
92+
}
93+
}
94+
}
7595
}

‎infrastructure/tf-audit/environments/sandbox.tfvars‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
application = "cohman"
22
application_full_name = "cohort-manager"
3-
environment = "SBX"
3+
environment = "SBRK"
44

55
tags = {
66
Environment = "sandbox"
@@ -17,7 +17,7 @@ features = {
1717
regions = {
1818
uksouth = {
1919
is_primary_region = true
20-
address_space = "10.127.0.0/16"
20+
address_space = "10.129.0.0/16"
2121
connect_peering = true
2222
subnets = {
2323
pep = {
@@ -52,7 +52,16 @@ storage_accounts = {
5252
container_name = "vulnerability-assessment"
5353
container_access_type = "private"
5454
}
55+
sql-backups-immutable = {
56+
container_name = "sql-backups-immutable"
57+
container_access_type = "private"
58+
immutability_policy = {
59+
is_locked = false
60+
immutability_period_in_days = 1
61+
protected_append_writes_all_enabled = false
62+
protected_append_writes_enabled = false
63+
}
64+
}
5565
}
56-
5766
}
5867
}

‎infrastructure/tf-audit/variables.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,12 @@ variable "storage_accounts" {
113113
containers = optional(map(object({
114114
container_name = string
115115
container_access_type = optional(string, "private")
116+
immutability_policy = optional(object({
117+
is_locked = optional(bool, false)
118+
immutability_period_in_days = optional(number, 0)
119+
protected_append_writes_all_enabled = optional(bool, false)
120+
protected_append_writes_enabled = optional(bool, false)
121+
}), null)
116122
})), {})
117123
}))
118124
}

‎infrastructure/tf-core/container_app_job.tf‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,24 @@ locals {
88
region = region # 1st iterator
99
container_app_job = container_app_job # 2nd iterator
1010
},
11-
config # the rest of the key/value pairs for a specific container_app_job
11+
config, # the rest of the key/value pairs for a specific container_app_job
12+
{
13+
env_vars = merge(
14+
# Add environment variables defined specifically for this container app job:
15+
config.env_vars_static,
16+
17+
# Add in the database connection string if the name of the variable is provided:
18+
config.add_user_assigned_identity != null && length(config.db_connection_string_name) > 0 ? {
19+
(config.db_connection_string_name) = "Server=tcp:${module.regions_config[region].names.sql-server}.database.windows.net,1433;Initial Catalog=${var.sqlserver.dbs.cohman.db_name_suffix};Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;Authentication='Active Directory Managed Identity';User ID=${module.user_assigned_managed_identity_sql["${container_app_job}-${region}"].client_id};"
20+
} : {},
21+
22+
# Add in the MANAGED_IDENTITY_CLIENT_ID environment variable if using a user assigned managed identity:
23+
config.add_user_assigned_identity != null ? {
24+
"MANAGED_IDENTITY_CLIENT_ID" = "${module.user_assigned_managed_identity_sql["${container_app_job}-${region}"].client_id}",
25+
"TARGET_SUBSCRIPTION_ID" = var.TARGET_SUBSCRIPTION_ID
26+
} : {}
27+
)
28+
}
1229
)
1330
]
1431
])
@@ -29,17 +46,16 @@ module "container-app-job" {
2946
location = each.value.region
3047

3148
container_app_environment_id = module.container-app-environment["${each.value.container_app_environment_key}-${each.value.region}"].id
32-
user_assigned_identity_ids = [module.managed_identity_sql_db_management[each.value.region].id]
49+
user_assigned_identity_ids = each.value.add_user_assigned_identity ? [module.user_assigned_managed_identity_sql["${each.key}"].id] : []
3350

3451
acr_login_server = data.azurerm_container_registry.acr.login_server
3552
acr_managed_identity_id = each.value.container_registry_use_mi ? data.azurerm_user_assigned_identity.acr_mi.id : null
3653
docker_image = "${data.azurerm_container_registry.acr.login_server}/${each.value.docker_image}:${each.value.docker_env_tag != "" ? each.value.docker_env_tag : var.docker_image_tag}"
54+
replica_retry_limit = each.value.replica_retry_limit != null ? each.value.replica_retry_limit : 1
3755

38-
environment_variables = {
39-
"DtOsDatabaseConnectionString" = "Server=tcp:${module.regions_config[each.value.region].names.sql-server}.database.windows.net,1433;Initial Catalog=${var.sqlserver.dbs.cohman.db_name_suffix};Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;Authentication='Active Directory Managed Identity';User ID=${module.managed_identity_sql_db_management[each.value.region].client_id};"
40-
}
56+
environment_variables = each.value.env_vars != null ? each.value.env_vars : {}
4157

4258
depends_on = [
43-
module.managed_identity_sql_db_management
59+
module.azure_sql_server
4460
]
4561
}

‎infrastructure/tf-core/environments/production.tfvars‎

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,6 @@ routes = {
109109
app_service_plan = {
110110
os_type = "Linux"
111111
vnet_integration_enabled = true
112-
zone_balancing_enabled = true
113112

114113
autoscale = {
115114
scaling_rule = {
@@ -247,6 +246,38 @@ container_app_jobs = {
247246
container_app_environment_key = "db-management"
248247
docker_image = "cohort-manager-db-migration"
249248
container_registry_use_mi = true
249+
db_connection_string_name = "DtOsDatabaseConnectionString"
250+
add_user_assigned_identity = true
251+
replica_retry_limit = 1
252+
}
253+
db-backup = {
254+
container_app_environment_key = "db-management"
255+
docker_image = "db-immutable-backup"
256+
docker_env_tag = "latest"
257+
container_registry_use_mi = true
258+
add_user_assigned_identity = true
259+
replica_retry_limit = 1
260+
env_vars_static = {
261+
SQL_SERVER_NAME = "sqlsvr-cohman-prod-uks"
262+
SQL_DATABASE_NAME = "DToSDB"
263+
STORAGE_ACCOUNT_NAME = "stcohmanprodukssqlbackup"
264+
STORAGE_CONTAINER_NAME = "sql-backups-immutable"
265+
}
266+
}
267+
db-restore = {
268+
container_app_environment_key = "db-management"
269+
docker_image = "db-immutable-backup-restore"
270+
docker_env_tag = "latest"
271+
container_registry_use_mi = true
272+
add_user_assigned_identity = true
273+
replica_retry_limit = 1
274+
env_vars_static = {
275+
SQL_SERVER_NAME = "sqlsvr-cohman-prod-uks"
276+
SQL_DATABASE_NAME = "DToSDB_RESTORE"
277+
STORAGE_ACCOUNT_NAME = "stcohmanprodukssqlbackup"
278+
STORAGE_CONTAINER_NAME = "sql-backups-immutable"
279+
BACKUP_FILE_NAME = "filename.bacpac"
280+
}
250281
}
251282
}
252283
}
@@ -1003,10 +1034,11 @@ function_apps = {
10031034
}
10041035

10051036
RetrievePDSDemographic = {
1006-
name_suffix = "retrieve-pds-demographic"
1007-
function_endpoint_name = "RetrievePDSDemographic"
1008-
app_service_plan_key = "NonScaling"
1009-
key_vault_url = "KeyVaultConnectionString"
1037+
name_suffix = "retrieve-pds-demographic"
1038+
function_endpoint_name = "RetrievePDSDemographic"
1039+
app_service_plan_key = "NonScaling"
1040+
service_bus_connections = ["internal"]
1041+
key_vault_url = "KeyVaultConnectionString"
10101042
app_urls = [
10111043
{
10121044
env_var_name = "ExceptionFunctionURL"
@@ -1246,7 +1278,11 @@ sqlserver = {
12461278
ad_auth_only = true
12471279
auditing_policy_retention_in_days = 30
12481280
security_alert_policy_retention_days = 30
1249-
db_management_mi_name_prefix = "mi-cohort-manager-db-management"
1281+
user_assigned_identities = [
1282+
"db-management",
1283+
"db-backup",
1284+
"db-restore"
1285+
]
12501286

12511287
server = {
12521288
sqlversion = "12.0"
@@ -1262,7 +1298,7 @@ sqlserver = {
12621298
licence_type = "LicenseIncluded"
12631299
max_gb = 100
12641300
read_scale = false
1265-
sku = "S12"
1301+
sku = "S2"
12661302
storage_account_type = "GeoZone"
12671303
zone_redundant = false
12681304

‎infrastructure/tf-core/environments/sandbox.tfvars‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,8 @@ container_app_jobs = {
299299
container_app_environment_key = "db-management"
300300
docker_image = "cohort-manager-db-migration"
301301
container_registry_use_mi = true
302+
db_connection_string_name = "DtOsDatabaseConnectionString"
303+
add_user_assigned_identity = true
302304
}
303305
}
304306
}
@@ -1300,7 +1302,9 @@ sqlserver = {
13001302
ad_auth_only = true
13011303
auditing_policy_retention_in_days = 30
13021304
security_alert_policy_retention_days = 30
1303-
db_management_mi_name_prefix = "mi-cohort-manager-db-management"
1305+
user_assigned_identities = [
1306+
"db-management"
1307+
]
13041308

13051309
server = {
13061310
sqlversion = "12.0"

‎infrastructure/tf-core/network_routing.tf‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ module "route_table_core" {
4444

4545
subnet_ids = [
4646
module.subnets["${module.regions_config[each.key].names.subnet}-apps"].id,
47-
module.subnets["${module.regions_config[each.key].names.subnet}-pep"].id
47+
module.subnets["${module.regions_config[each.key].names.subnet}-pep"].id,
48+
module.subnets["${module.regions_config[each.key].names.subnet}-container-app-db-management"].id
4849
]
4950

5051
tags = var.tags

‎infrastructure/tf-core/sql_server.tf‎

Lines changed: 69 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -68,25 +68,87 @@ module "azure_sql_server" {
6868
tags = var.tags
6969
}
7070

71-
module "managed_identity_sql_db_management" {
72-
for_each = var.sqlserver != {} ? var.regions : {}
71+
# Create User Assigned Managed Identities for Azure SQL access by other resources
72+
73+
locals {
74+
managed_identities = flatten([
75+
for region, _ in var.regions : [
76+
for mi_name in var.sqlserver.user_assigned_identities : {
77+
region = region
78+
mi_name = mi_name
79+
}
80+
]
81+
])
82+
83+
managed_identities_map = {
84+
for object in local.managed_identities : "${object.mi_name}-${object.region}" => object
85+
}
86+
}
87+
88+
module "user_assigned_managed_identity_sql" {
89+
for_each = local.managed_identities_map
7390

7491
source = "../../../dtos-devops-templates/infrastructure/modules/managed-identity"
7592

76-
uai_name = "${var.sqlserver.db_management_mi_name_prefix}-${lower(var.environment)}-${lower(each.key)}"
77-
resource_group_name = azurerm_resource_group.core[each.key].name
78-
location = each.key
93+
uai_name = "${module.regions_config[each.value.region].names.managed-identity}-${lower(each.value.mi_name)}"
94+
resource_group_name = azurerm_resource_group.core[each.value.region].name
95+
location = each.value.region
7996

8097
tags = var.tags
8198
}
8299

100+
# Assign RBAC roles to the User Assigned Managed Identities for Azure SQL access by other resources
101+
# DB-MANAGEMENT needs Contributor on the SQL Server to be able to run migrations
83102
module "sql_db_management_rbac_assignment" {
84-
for_each = var.sqlserver != {} ? var.regions : {}
103+
for_each = contains(var.sqlserver.user_assigned_identities, "db-management") && var.sqlserver != {} ? var.regions : {}
85104

86105
source = "../../../dtos-devops-templates/infrastructure/modules/rbac-assignment"
87106

88-
principal_id = module.managed_identity_sql_db_management[each.key].principal_id
107+
principal_id = module.user_assigned_managed_identity_sql["db-management-${each.key}"].principal_id
89108
role_definition_name = "Contributor"
90109
scope = module.azure_sql_server[each.key].sql_server_id
91110

92111
}
112+
113+
# DB-BACKUP needs SQL DB Contributor on the SQL Server to be able to read the database, and Storage Blob Data Contributor on the Storage Account to write the backups
114+
module "sql_db_backup_rbac_assignment_sql_contributor" {
115+
for_each = contains(var.sqlserver.user_assigned_identities, "db-backup") && var.sqlserver != {} ? var.regions : {}
116+
117+
source = "../../../dtos-devops-templates/infrastructure/modules/rbac-assignment"
118+
119+
principal_id = module.user_assigned_managed_identity_sql["db-backup-${each.key}"].principal_id
120+
role_definition_name = "SQL DB Contributor"
121+
scope = module.azure_sql_server[each.key].sql_server_id
122+
}
123+
124+
module "sql_db_backup_rbac_assignment_storage_contributor" {
125+
for_each = contains(var.sqlserver.user_assigned_identities, "db-backup") && var.sqlserver != {} ? var.regions : {}
126+
127+
source = "../../../dtos-devops-templates/infrastructure/modules/rbac-assignment"
128+
129+
principal_id = module.user_assigned_managed_identity_sql["db-backup-${each.key}"].principal_id
130+
role_definition_name = "Storage Blob Data Contributor"
131+
scope = data.terraform_remote_state.audit.outputs.storage_account_audit["sqlbackups-${local.primary_region}"].id
132+
}
133+
134+
135+
# DB-RESTORE needs SQL DB Contributor on the SQL Server to be able to write to the database, and Storage Blob Data Reader on the Storage Account to read the backups
136+
module "sql_db_restore_rbac_assignment_sql_contributor" {
137+
for_each = contains(var.sqlserver.user_assigned_identities, "db-restore") && var.sqlserver != {} ? var.regions : {}
138+
139+
source = "../../../dtos-devops-templates/infrastructure/modules/rbac-assignment"
140+
141+
principal_id = module.user_assigned_managed_identity_sql["db-restore-${each.key}"].principal_id
142+
role_definition_name = "SQL DB Contributor"
143+
scope = module.azure_sql_server[each.key].sql_server_id
144+
}
145+
146+
module "sql_db_restore_rbac_assignment_storage_reader" {
147+
for_each = contains(var.sqlserver.user_assigned_identities, "db-restore") && var.sqlserver != {} ? var.regions : {}
148+
149+
source = "../../../dtos-devops-templates/infrastructure/modules/rbac-assignment"
150+
151+
principal_id = module.user_assigned_managed_identity_sql["db-restore-${each.key}"].principal_id
152+
role_definition_name = "Storage Blob Data Reader"
153+
scope = data.terraform_remote_state.audit.outputs.storage_account_audit["sqlbackups-${local.primary_region}"].id
154+
}

0 commit comments

Comments
 (0)