feat: DTOSS-9884 Schedule Immutable Backups in Prod

.azuredevops/pipelines/task_azure_sql_backup_prod.yaml

@@ -0,0 +1,53 @@
+---
+
+name: $(Build.SourceBranchName)-$(Date:yyyyMMdd)_$(Rev:r)
+trigger: none
+pr: none
+
+schedules:
+  - cron: "0 2 * * *" # Run daily at 2:00 AM UTC
+    displayName: 'Run Backup'
+    branches:
+      include:
+      - main
+    always: true
+
+resources:
+  repositories:
+    - repository: dtos-devops-templates
+      type: github
+      name: NHSDigital/dtos-devops-templates
+      ref: 34a7e5c3072bddaa350804905c60b9c8e7ae7191
+      endpoint: NHSDigital
+
+variables:
+  - name: hostPoolName
+    value: private-pool-prod-uks
+  - group: PROD_core_backend
+  - group: PROD_image_pipelines
+  - name: TF_VERSION
+    value: 1.11.4
+  - name: TF_PLAN_ARTIFACT
+    value: tf_plan_core_PROD
+  - name: TF_DIRECTORY
+    value: $(System.DefaultWorkingDirectory)/$(System.TeamProject)/infrastructure/tf-core
+  - name: ENVIRONMENT
+    value: production
+
+stages:
+- stage: db_backup_stage
+  displayName: Database backup stage
+  pool:
+    name: $(hostPoolName)
+  jobs:
+    - job: db_changes
+      displayName: Create Database Backup and Upload to Storage Account
+      steps:
+        - checkout: self
+        - checkout: dtos-devops-templates
+        - template: .azuredevops/templates/steps/app-container-job-start.yaml@dtos-devops-templates
+          parameters:
+            serviceConnection: $(SERVICE_CONNECTION)
+            targetSubscriptionId: $(TF_VAR_TARGET_SUBSCRIPTION_ID)
+            resourceGroupName: $(RESOURCE_GROUP_NAME_SQL)
+            jobName: ca-db-backup-uksouth

infrastructure/tf-audit/environments/production.tfvars

@@ -72,4 +72,24 @@ storage_accounts = {
       }
     }
   }
+  sqlbackups = {
+    name_suffix                             = "sqlbackups"
+    account_tier                            = "Standard"
+    replication_type                        = "GRS"
+    public_network_access_enabled           = false
+    blob_properties_delete_retention_policy = 28
+    blob_properties_versioning_enabled      = true
+    containers = {
+      sql-backups-immutable = {
+        container_name        = "sql-backups-immutable"
+        container_access_type = "private"
+        immutability_policy = {
+          is_locked                           = false
+          immutability_period_in_days         = 1
+          protected_append_writes_all_enabled = false
+          protected_append_writes_enabled     = false
+        }
+      }
+    }
+  }
 }

infrastructure/tf-audit/environments/sandbox.tfvars

@@ -1,6 +1,6 @@
 application           = "cohman"
 application_full_name = "cohort-manager"
-environment           = "SBX"
+environment           = "SBRK"
 
 tags = {
   Environment = "sandbox"
@@ -17,7 +17,7 @@ features = {
 regions = {
   uksouth = {
     is_primary_region = true
-    address_space     = "10.127.0.0/16"
+    address_space     = "10.129.0.0/16"
     connect_peering   = true
     subnets = {
       pep = {
@@ -52,7 +52,16 @@ storage_accounts = {
         container_name        = "vulnerability-assessment"
         container_access_type = "private"
       }
+      sql-backups-immutable = {
+        container_name        = "sql-backups-immutable"
+        container_access_type = "private"
+        immutability_policy = {
+          is_locked                           = false
+          immutability_period_in_days         = 1
+          protected_append_writes_all_enabled = false
+          protected_append_writes_enabled     = false
+        }
+      }
     }
-
   }
 }

infrastructure/tf-audit/variables.tf

@@ -113,6 +113,12 @@ variable "storage_accounts" {
     containers = optional(map(object({
       container_name        = string
       container_access_type = optional(string, "private")
+      immutability_policy = optional(object({
+        is_locked                           = optional(bool, false)
+        immutability_period_in_days         = optional(number, 0)
+        protected_append_writes_all_enabled = optional(bool, false)
+        protected_append_writes_enabled     = optional(bool, false)
+      }), null)
     })), {})
   }))
 }

infrastructure/tf-core/container_app_job.tf

@@ -8,7 +8,24 @@ locals {
           region            = region            # 1st iterator
           container_app_job = container_app_job # 2nd iterator
         },
-        config # the rest of the key/value pairs for a specific container_app_job
+        config, # the rest of the key/value pairs for a specific container_app_job
+        {
+          env_vars = merge(
+            # Add environment variables defined specifically for this container app job:
+            config.env_vars_static,
+
+            # Add in the database connection string if the name of the variable is provided:
+            config.add_user_assigned_identity != null && length(config.db_connection_string_name) > 0 ? {
+              (config.db_connection_string_name) = "Server=tcp:${module.regions_config[region].names.sql-server}.database.windows.net,1433;Initial Catalog=${var.sqlserver.dbs.cohman.db_name_suffix};Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;Authentication='Active Directory Managed Identity';User ID=${module.user_assigned_managed_identity_sql["${container_app_job}-${region}"].client_id};"
+            } : {},
+
+            # Add in the MANAGED_IDENTITY_CLIENT_ID environment variable if using a user assigned managed identity:
+            config.add_user_assigned_identity != null ? {
+              "MANAGED_IDENTITY_CLIENT_ID" = "${module.user_assigned_managed_identity_sql["${container_app_job}-${region}"].client_id}",
+              "TARGET_SUBSCRIPTION_ID"     = var.TARGET_SUBSCRIPTION_ID
+            } : {}
+          )
+        }
       )
     ]
   ])
@@ -29,17 +46,16 @@ module "container-app-job" {
   location            = each.value.region
 
   container_app_environment_id = module.container-app-environment["${each.value.container_app_environment_key}-${each.value.region}"].id
-  user_assigned_identity_ids   = [module.managed_identity_sql_db_management[each.value.region].id]
+  user_assigned_identity_ids   = each.value.add_user_assigned_identity ? [module.user_assigned_managed_identity_sql["${each.key}"].id] : []
 
   acr_login_server        = data.azurerm_container_registry.acr.login_server
   acr_managed_identity_id = each.value.container_registry_use_mi ? data.azurerm_user_assigned_identity.acr_mi.id : null
   docker_image            = "${data.azurerm_container_registry.acr.login_server}/${each.value.docker_image}:${each.value.docker_env_tag != "" ? each.value.docker_env_tag : var.docker_image_tag}"
+  replica_retry_limit     = each.value.replica_retry_limit != null ? each.value.replica_retry_limit : 1
 
-  environment_variables = {
-    "DtOsDatabaseConnectionString" = "Server=tcp:${module.regions_config[each.value.region].names.sql-server}.database.windows.net,1433;Initial Catalog=${var.sqlserver.dbs.cohman.db_name_suffix};Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;Authentication='Active Directory Managed Identity';User ID=${module.managed_identity_sql_db_management[each.value.region].client_id};"
-  }
+  environment_variables = each.value.env_vars != null ? each.value.env_vars : {}
 
   depends_on = [
-    module.managed_identity_sql_db_management
+    module.azure_sql_server
   ]
 }

infrastructure/tf-core/environments/production.tfvars

@@ -109,7 +109,6 @@ routes = {
 app_service_plan = {
   os_type                  = "Linux"
   vnet_integration_enabled = true
-  zone_balancing_enabled   = true
 
   autoscale = {
     scaling_rule = {
@@ -247,6 +246,38 @@ container_app_jobs = {
       container_app_environment_key = "db-management"
       docker_image                  = "cohort-manager-db-migration"
       container_registry_use_mi     = true
+      db_connection_string_name     = "DtOsDatabaseConnectionString"
+      add_user_assigned_identity    = true
+      replica_retry_limit           = 1
+    }
+    db-backup = {
+      container_app_environment_key = "db-management"
+      docker_image                  = "db-immutable-backup"
+      docker_env_tag                = "latest"
+      container_registry_use_mi     = true
+      add_user_assigned_identity    = true
+      replica_retry_limit           = 1
+      env_vars_static = {
+        SQL_SERVER_NAME        = "sqlsvr-cohman-prod-uks"
+        SQL_DATABASE_NAME      = "DToSDB"
+        STORAGE_ACCOUNT_NAME   = "stcohmanprodukssqlbackup"
+        STORAGE_CONTAINER_NAME = "sql-backups-immutable"
+      }
+    }
+    db-restore = {
+      container_app_environment_key = "db-management"
+      docker_image                  = "db-immutable-backup-restore"
+      docker_env_tag                = "latest"
+      container_registry_use_mi     = true
+      add_user_assigned_identity    = true
+      replica_retry_limit           = 1
+      env_vars_static = {
+        SQL_SERVER_NAME        = "sqlsvr-cohman-prod-uks"
+        SQL_DATABASE_NAME      = "DToSDB_RESTORE"
+        STORAGE_ACCOUNT_NAME   = "stcohmanprodukssqlbackup"
+        STORAGE_CONTAINER_NAME = "sql-backups-immutable"
+        BACKUP_FILE_NAME       = "filename.bacpac"
+      }
     }
   }
 }
@@ -1002,10 +1033,11 @@ function_apps = {
     }
 
     RetrievePDSDemographic = {
-      name_suffix            = "retrieve-pds-demographic"
-      function_endpoint_name = "RetrievePDSDemographic"
-      app_service_plan_key   = "NonScaling"
-      key_vault_url          = "KeyVaultConnectionString"
+      name_suffix             = "retrieve-pds-demographic"
+      function_endpoint_name  = "RetrievePDSDemographic"
+      app_service_plan_key    = "NonScaling"
+      service_bus_connections = ["internal"]
+      key_vault_url           = "KeyVaultConnectionString"
       app_urls = [
         {
           env_var_name     = "ExceptionFunctionURL"
@@ -1245,7 +1277,11 @@ sqlserver = {
   ad_auth_only                         = true
   auditing_policy_retention_in_days    = 30
   security_alert_policy_retention_days = 30
-  db_management_mi_name_prefix         = "mi-cohort-manager-db-management"
+  user_assigned_identities = [
+    "db-management",
+    "db-backup",
+    "db-restore"
+  ]
 
   server = {
     sqlversion                    = "12.0"
@@ -1261,7 +1297,7 @@ sqlserver = {
       licence_type         = "LicenseIncluded"
       max_gb               = 100
       read_scale           = false
-      sku                  = "S12"
+      sku                  = "S2"
       storage_account_type = "GeoZone"
       zone_redundant       = false
 

infrastructure/tf-core/environments/sandbox.tfvars

@@ -299,6 +299,8 @@ container_app_jobs = {
       container_app_environment_key = "db-management"
       docker_image                  = "cohort-manager-db-migration"
       container_registry_use_mi     = true
+      db_connection_string_name     = "DtOsDatabaseConnectionString"
+      add_user_assigned_identity    = true
     }
   }
 }
@@ -1299,7 +1301,9 @@ sqlserver = {
   ad_auth_only                         = true
   auditing_policy_retention_in_days    = 30
   security_alert_policy_retention_days = 30
-  db_management_mi_name_prefix         = "mi-cohort-manager-db-management"
+  user_assigned_identities = [
+    "db-management"
+  ]
 
   server = {
     sqlversion                    = "12.0"

infrastructure/tf-core/network_routing.tf

@@ -44,7 +44,8 @@ module "route_table_core" {
 
   subnet_ids = [
     module.subnets["${module.regions_config[each.key].names.subnet}-apps"].id,
-    module.subnets["${module.regions_config[each.key].names.subnet}-pep"].id
+    module.subnets["${module.regions_config[each.key].names.subnet}-pep"].id,
+    module.subnets["${module.regions_config[each.key].names.subnet}-container-app-db-management"].id
   ]
 
   tags = var.tags

infrastructure/tf-core/sql_server.tf

@@ -68,25 +68,87 @@ module "azure_sql_server" {
   tags = var.tags
 }
 
-module "managed_identity_sql_db_management" {
-  for_each = var.sqlserver != {} ? var.regions : {}
+# Create User Assigned Managed Identities for Azure SQL access by other resources
+
+locals {
+  managed_identities = flatten([
+    for region, _ in var.regions : [
+      for mi_name in var.sqlserver.user_assigned_identities : {
+        region  = region
+        mi_name = mi_name
+      }
+    ]
+  ])
+
+  managed_identities_map = {
+    for object in local.managed_identities : "${object.mi_name}-${object.region}" => object
+  }
+}
+
+module "user_assigned_managed_identity_sql" {
+  for_each = local.managed_identities_map
 
   source = "../../../dtos-devops-templates/infrastructure/modules/managed-identity"
 
-  uai_name            = "${var.sqlserver.db_management_mi_name_prefix}-${lower(var.environment)}-${lower(each.key)}"
-  resource_group_name = azurerm_resource_group.core[each.key].name
-  location            = each.key
+  uai_name            = "${module.regions_config[each.value.region].names.managed-identity}-${lower(each.value.mi_name)}"
+  resource_group_name = azurerm_resource_group.core[each.value.region].name
+  location            = each.value.region
 
   tags = var.tags
 }
 
+# Assign RBAC roles to the User Assigned Managed Identities for Azure SQL access by other resources
+# DB-MANAGEMENT needs Contributor on the SQL Server to be able to run migrations
 module "sql_db_management_rbac_assignment" {
-  for_each = var.sqlserver != {} ? var.regions : {}
+  for_each = contains(var.sqlserver.user_assigned_identities, "db-management") && var.sqlserver != {} ? var.regions : {}
 
   source = "../../../dtos-devops-templates/infrastructure/modules/rbac-assignment"
 
-  principal_id         = module.managed_identity_sql_db_management[each.key].principal_id
+  principal_id         = module.user_assigned_managed_identity_sql["db-management-${each.key}"].principal_id
   role_definition_name = "Contributor"
   scope                = module.azure_sql_server[each.key].sql_server_id
 
 }
+
+# DB-BACKUP needs SQL DB Contributor on the SQL Server to be able to read the database, and Storage Blob Data Contributor on the Storage Account to write the backups
+module "sql_db_backup_rbac_assignment_sql_contributor" {
+  for_each = contains(var.sqlserver.user_assigned_identities, "db-backup") && var.sqlserver != {} ? var.regions : {}
+
+  source = "../../../dtos-devops-templates/infrastructure/modules/rbac-assignment"
+
+  principal_id         = module.user_assigned_managed_identity_sql["db-backup-${each.key}"].principal_id
+  role_definition_name = "SQL DB Contributor"
+  scope                = module.azure_sql_server[each.key].sql_server_id
+}
+
+module "sql_db_backup_rbac_assignment_storage_contributor" {
+  for_each = contains(var.sqlserver.user_assigned_identities, "db-backup") && var.sqlserver != {} ? var.regions : {}
+
+  source = "../../../dtos-devops-templates/infrastructure/modules/rbac-assignment"
+
+  principal_id         = module.user_assigned_managed_identity_sql["db-backup-${each.key}"].principal_id
+  role_definition_name = "Storage Blob Data Contributor"
+  scope                = data.terraform_remote_state.audit.outputs.storage_account_audit["sqlbackups-${local.primary_region}"].id
+}
+
+
+# DB-RESTORE needs SQL DB Contributor on the SQL Server to be able to write to the database, and Storage Blob Data Reader on the Storage Account to read the backups
+module "sql_db_restore_rbac_assignment_sql_contributor" {
+  for_each = contains(var.sqlserver.user_assigned_identities, "db-restore") && var.sqlserver != {} ? var.regions : {}
+
+  source = "../../../dtos-devops-templates/infrastructure/modules/rbac-assignment"
+
+  principal_id         = module.user_assigned_managed_identity_sql["db-restore-${each.key}"].principal_id
+  role_definition_name = "SQL DB Contributor"
+  scope                = module.azure_sql_server[each.key].sql_server_id
+}
+
+module "sql_db_restore_rbac_assignment_storage_reader" {
+  for_each = contains(var.sqlserver.user_assigned_identities, "db-restore") && var.sqlserver != {} ? var.regions : {}
+
+  source = "../../../dtos-devops-templates/infrastructure/modules/rbac-assignment"
+
+  principal_id         = module.user_assigned_managed_identity_sql["db-restore-${each.key}"].principal_id
+  role_definition_name = "Storage Blob Data Reader"
+  scope                = data.terraform_remote_state.audit.outputs.storage_account_audit["sqlbackups-${local.primary_region}"].id
+}