APPT-1652 Point in time restore pipeline

azure-pipelines-pitr.yml

@@ -0,0 +1,165 @@
+trigger: none
+pr: none
+
+pool:
+  vmImage: "ubuntu-latest"
+
+parameters:
+  - name: RestoreDateTime
+    displayName: "Target Restore Date/Time (UK LOCAL TIME)"
+    type: string
+    default: "2026-04-24 10:00:00" # a template for the user
+
+stages:
+  - stage: Validation
+    displayName: "Pre-Flight Validation"
+    jobs:
+      - job: ValidateInput
+        steps:
+          - task: AzureCLI@2
+            name: ValidateStep
+            displayName: "Validate Restore Window"
+            inputs:
+              azureSubscription: "vaccs-mya-dev-uks"
+              scriptType: "pscore"
+              scriptLocation: "scriptPath"
+              scriptPath: "$(Build.SourcesDirectory)/scripts/disaster-recovery/validate-cosmos-pitr.ps1"
+              arguments: >
+                -sourceAccount "vaccs-mya-cdb-dev-uks"
+                -resourceGroup "vaccs-mya-dev-uks"
+                -restoreDateAndTimeStr "${{ parameters.RestoreDateTime }}"
+
+  - stage: StopWebApp
+    displayName: "Stop Applications"
+    dependsOn: Validation
+    jobs:
+      - job: StopApps
+        steps:
+          - task: AzureCLI@2
+            displayName: "Stop Applications"
+            inputs:
+              azureSubscription: "vaccs-mya-dev-uks"
+              scriptType: "pscore"
+              scriptLocation: scriptPath
+              scriptPath: "$(Build.SourcesDirectory)/scripts/disaster-recovery/stop-dev-app-services.ps1"
+              arguments: "-resourceGroup vaccs-mya-dev-uks -region uks"
+
+  - stage: DecommissionSourceDB
+    displayName: "Take Source DB Offline"
+    dependsOn: StopWebApp
+    jobs:
+      - job: TakeOffline
+        steps:
+          - checkout: self
+          - task: AzureCLI@2
+            inputs:
+              azureSubscription: "vaccs-mya-dev-uks"
+              scriptType: "pscore"
+              scriptLocation: "scriptPath"
+              scriptPath: "$(Build.SourcesDirectory)/scripts/disaster-recovery/take-offline-source-db.ps1"
+              arguments: >
+                -sourceAccount "vaccs-mya-cdb-dev-uks"
+                -resourceGroup "vaccs-mya-dev-uks"
+
+  - stage: CosmosPITR
+    displayName: "Perform Cosmos DB PITR"
+    dependsOn:
+      - StopWebApp
+      - Validation
+      - DecommissionSourceDB
+    variables:
+      RestoreTimeUtc: $[ stageDependencies.Validation.ValidateInput.outputs['ValidateStep.ConvertedUtcTime'] ]
+    jobs:
+      - job: RestoreCosmos
+        steps:
+          - task: AzureCLI@2
+            displayName: "Execute Restore"
+            inputs:
+              azureSubscription: "vaccs-mya-dev-uks"
+              scriptType: "pscore"
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                az cosmosdb restore `
+                  --account-name "vaccs-mya-cdb-dev-uks" `
+                  --resource-group "vaccs-mya-dev-uks" `
+                  --target-database-account-name "vaccs-mya-cdb-dev-uks-secondary" `
+                  --location "uksouth" `
+                  --restore-timestamp "$(RestoreTimeUtc)"
+
+  - stage: UpdateConfig
+    displayName: "Update Key Vault Configuration"
+    dependsOn: CosmosPITR
+    jobs:
+      - job: Update
+        steps:
+          - task: AzureCLI@2
+            displayName: "Execute KV Update Script"
+            inputs:
+              azureSubscription: "vaccs-mya-dev-uks"
+              scriptType: "pscore"
+              scriptLocation: "scriptPath"
+              scriptPath: "$(Build.SourcesDirectory)/scripts/disaster-recovery/update-active-db-config.ps1"
+              arguments: >
+                -vaultName "vaccs-mya-kv-dev-uks"
+                -resourceGroup "vaccs-mya-dev-uks"
+                -cosmosAccountName "vaccs-mya-cdb-dev-uks-secondary"
+
+  - stage: CreateAuditBlobStorageAccount
+    displayName: "Create new audit blob storage account"
+    dependsOn: UpdateConfig
+    jobs:
+      - job: Create
+        steps:
+          - task: AzureCLI@2
+            displayName: "Create secondary audit blob storage account"
+            inputs:
+              azureSubscription: "vaccs-mya-dev-uks"
+              scriptType: "pscore"
+              scriptLocation: "scriptPath"
+              scriptPath: "$(Build.SourcesDirectory)/scripts/disaster-recovery/create-audit-blob-storage.ps1"
+              arguments: >
+                -resourceGroup "vaccs-mya-dev-uks"
+                -blobAccountName "vaccsmyastrgaudit2devuks"
+                -subscriptionId "85913fdb-7f11-4963-8519-70879ffd2910"
+
+  - stage: UpdateAuditContainerApp
+    displayName: "Update audit container app to point to new storage account"
+    dependsOn: CreateAuditBlobStorageAccount
+    jobs:
+      - job: Update
+        steps:
+          - task: AzureCLI@2
+            displayName: "Update audit container app to point to new storage account"
+            inputs:
+              azureSubscription: "vaccs-mya-dev-uks"
+              scriptType: "pscore"
+              scriptLocation: "scriptPath"
+              scriptPath: "$(Build.SourcesDirectory)/scripts/disaster-recovery/update-audit-storage-account.ps1"
+              arguments: >
+                -resourceGroup "vaccs-mya-dev-uks"
+                -blobAccountName "vaccsmyastrgaudit2devuks"
+                -containerApp "vaccs-mya-auditor-dev-uks"
+
+  - stage: RestartServices
+    displayName: "Restart Apps to Pick Up New Config"
+    dependsOn: UpdateAuditContainerApp
+    # This 'environment' line triggers the manual approval check if configured in ADO
+    jobs:
+      - deployment: Restart
+        # NOTE: For this to work, you MUST set up an Approval check on that environment in the Azure DevOps UI.
+        # Go to Pipelines -> Environments -> 'mya-dev-recovery' -> Approvals and Checks.
+        environment: "mya-dev-recovery"
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+                - checkout: self
+                - task: AzureCLI@2
+                  displayName: "Run Restart Services Script"
+                  inputs:
+                    azureSubscription: "vaccs-mya-dev-uks"
+                    scriptType: "pscore"
+                    scriptLocation: "scriptPath"
+                    # Ensure this path matches your repo structure
+                    scriptPath: "$(Build.SourcesDirectory)/scripts/disaster-recovery/restart-services.ps1"
+                    arguments: "-resourceGroup vaccs-mya-dev-uks -region uks"

data/BookingsDataExtracts/Program.cs

@@ -4,6 +4,7 @@
 using Microsoft.FeatureManagement;
 using Nhs.Appointments.Core.Logger;
 using Nhs.Appointments.Persistance.Models;
+using Nhs.Appointments.Core.Configuration;
 
 var builder = Host.CreateApplicationBuilder(args);
 var azureAppConfigConnection = Environment.GetEnvironmentVariable("APP_CONFIG_CONNECTION");
@@ -12,8 +13,8 @@
 builder.Configuration
     .SetBasePath(AppContext.BaseDirectory)
     .AddJsonFile("appsettings.json", false, true)
-    .AddEnvironmentVariables()
-    .AddNbsAzureKeyVault();
+    .AddNbsAzureKeyVault()
+    .AddMyaConfiguration();
 
 // Handle Feature Flag (Local vs Cloud)
 if (azureAppConfigConnection == "local")

data/CapacityDataExtract/Program.cs

@@ -2,6 +2,7 @@
 using DataExtract;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
+using Nhs.Appointments.Core.Configuration;
 using Nhs.Appointments.Core.Logger;
 using Nhs.Appointments.Persistance.Models;
 
@@ -11,8 +12,8 @@
 builder.Configuration
             .SetBasePath(AppContext.BaseDirectory)
             .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
-            .AddEnvironmentVariables()
-            .AddNbsAzureKeyVault();
+            .AddNbsAzureKeyVault()
+            .AddMyaConfiguration();
 
 builder.UseAppointmentsSerilog();
 

infrastructure/resources/containerapp.tf

@@ -136,6 +136,10 @@ resource "azurerm_container_app_job" "nbs_mya_booking_extracts_job" {
         name  = "COSMOS_TOKEN"
         secret_name = "container-cosmos-password"
       }
+      env {
+        name  = "KEY_VAULT_URI"
+        value = azurerm_key_vault.nbs_mya_key_vault.vault_uri
+      }
       env {
         name = "FileSenderOptions__Type"
         value = var.data_extract_file_sender_options_type
@@ -259,6 +263,10 @@ resource "azurerm_container_app_job" "nbs_mya_capacity_extracts_job" {
         name  = "COSMOS_TOKEN"
         secret_name = "container-cosmos-password"
       }
+      env {
+        name  = "KEY_VAULT_URI"
+        value = azurerm_key_vault.nbs_mya_key_vault.vault_uri
+      }
       env {
         name = "FileSenderOptions__Type"
         value = var.data_extract_file_sender_options_type
@@ -385,6 +393,11 @@ resource "azurerm_container_app" "nbs_mya_auditor" {
         value = var.cosmos_endpoint != "" ? var.cosmos_endpoint : azurerm_cosmosdb_account.nbs_mya_cosmos_db[0].endpoint
       }
 
+      env {
+        name  = "KEY_VAULT_URI"
+        value = azurerm_key_vault.nbs_mya_key_vault.vault_uri
+      }
+
       env {
         name  = "SPLUNK_HOST_URL"
         value = var.splunk_host_url
@@ -532,6 +545,11 @@ resource "azurerm_container_app" "nbs_mya_aggregator" {
         secret_name = "splunk-hec-token"
       }
 
+      env {
+        name  = "KEY_VAULT_URI"
+        value = azurerm_key_vault.nbs_mya_key_vault.vault_uri
+      }
+
       env {
         name  = "COSMOS_ENDPOINT"
         value = var.cosmos_endpoint != "" ? var.cosmos_endpoint : azurerm_cosmosdb_account.nbs_mya_cosmos_db[0].endpoint

infrastructure/resources/high_load_functions.tf

@@ -83,6 +83,7 @@ resource "azurerm_windows_function_app" "nbs_mya_high_load_func_app" {
     Auth__Providers__1__RequiresStateForAuthorize                          = true
     CANCEL_A_DATE_RANGE_MAXIMUM_DAYS                                       = var.cancel_a_date_range_maximum_days
     APPLICATION_NAME                                                       = "High Load Function App"
+    KEY_VAULT_URI                                                          = azurerm_key_vault.nbs_mya_key_vault.vault_uri
     "AzureWebJobs.NotifyBookingCancelled.Disabled"                         = true
     "AzureWebJobs.NotifyBookingMade.Disabled"                              = true
     "AzureWebJobs.NotifyBookingReminder.Disabled"                          = true
@@ -238,6 +239,7 @@ resource "azurerm_windows_function_app_slot" "nbs_mya_high_load_func_app_preview
     Auth__Providers__1__RequiresStateForAuthorize                          = true
     CANCEL_A_DATE_RANGE_MAXIMUM_DAYS                                       = var.cancel_a_date_range_maximum_days
     APPLICATION_NAME                                                       = "High Load Function App"
+    KEY_VAULT_URI                                                          = azurerm_key_vault.nbs_mya_key_vault.vault_uri
     "AzureWebJobs.NotifyBookingCancelled.Disabled"                         = true
     "AzureWebJobs.NotifyBookingMade.Disabled"                              = true
     "AzureWebJobs.NotifyBookingReminder.Disabled"                          = true

infrastructure/resources/http_functions.tf

@@ -82,6 +82,7 @@ resource "azurerm_windows_function_app" "nbs_mya_http_func_app" {
     SITE_SUPPORTS_SERVICE_BATCH_MULTIPLIER                         = var.site_supports_service_batch_multiplier
     CANCEL_A_DATE_RANGE_MAXIMUM_DAYS                               = var.cancel_a_date_range_maximum_days
     APPLICATION_NAME                                               = "HTTP Function App"
+    KEY_VAULT_URI                                                  = azurerm_key_vault.nbs_mya_key_vault.vault_uri
     "AzureWebJobs.QueryAvailabilityFunction.Disabled"              = var.disable_query_availability_function
     "AzureWebJobs.NotifyBookingCancelled.Disabled"                 = true
     "AzureWebJobs.NotifyBookingMade.Disabled"                      = true
@@ -189,6 +190,7 @@ resource "azurerm_windows_function_app_slot" "nbs_mya_http_func_app_preview" {
     SITE_SUPPORTS_SERVICE_BATCH_MULTIPLIER                         = var.site_supports_service_batch_multiplier
     CANCEL_A_DATE_RANGE_MAXIMUM_DAYS                               = var.cancel_a_date_range_maximum_days
     APPLICATION_NAME                                               = "HTTP Function App"
+    KEY_VAULT_URI                                                  = azurerm_key_vault.nbs_mya_key_vault.vault_uri
     "AzureWebJobs.QueryAvailabilityFunction.Disabled"              = var.disable_query_availability_function
     "AzureWebJobs.NotifyBookingCancelled.Disabled"                 = true
     "AzureWebJobs.NotifyBookingMade.Disabled"                      = true

infrastructure/resources/keyvault.tf

@@ -0,0 +1,62 @@
+data "azurerm_client_config" "current" {}
+
+# Create the Key Vault
+resource "azurerm_key_vault" "nbs_mya_key_vault" {
+  name                = "${var.application}-kv-${var.environment}-${var.loc}"
+  location            = var.location
+  resource_group_name = local.resource_group_name
+  tenant_id           = data.azurerm_client_config.current.tenant_id
+  sku_name            = "standard"
+
+  enable_rbac_authorization = true
+  purge_protection_enabled  = true
+}
+
+resource "azurerm_key_vault_secret" "active_cosmos_account" {
+  name         = "ActiveCosmosDBAccountName"
+  value        = "${var.application}-cdb-${var.environment}-${var.loc}"
+  key_vault_id = azurerm_key_vault.nbs_mya_key_vault.id
+
+  depends_on = [
+    azurerm_role_assignment.pipeline_secret_access
+  ]
+
+  lifecycle {
+    ignore_changes = [value]
+  }
+}
+
+resource "azurerm_role_assignment" "pipeline_secret_access" {
+  scope                = azurerm_key_vault.nbs_mya_key_vault.id
+  role_definition_name = "Key Vault Secrets Officer"
+  principal_id         = data.azurerm_client_config.current.object_id
+}
+
+
+locals {
+  principals_map = merge(
+    { for i, v in azurerm_container_app_job.nbs_mya_booking_extracts_job : "booking_extracts_job_${i}" => v.identity[0].principal_id },
+    { for i, v in azurerm_container_app_job.nbs_mya_capacity_extracts_job : "capacity_extracts_job_${i}" => v.identity[0].principal_id },
+    { for i, v in azurerm_container_app.nbs_mya_auditor : "auditor_${i}" => v.identity[0].principal_id },
+    { for i, v in azurerm_container_app.nbs_mya_aggregator : "aggregator_${i}" => v.identity[0].principal_id },
+
+    { for i, v in azurerm_windows_function_app.nbs_mya_high_load_func_app : "high_load_${i}" => v.identity[0].principal_id },
+    { for i, v in azurerm_windows_function_app_slot.nbs_mya_high_load_func_app_preview : "high_load_preview_${i}" => v.identity[0].principal_id },
+
+    { for i, v in azurerm_windows_function_app.nbs_mya_http_func_app : "http_${i}" => v.identity[0].principal_id },
+    { for i, v in azurerm_windows_function_app_slot.nbs_mya_http_func_app_preview : "http_preview_${i}" => v.identity[0].principal_id },
+
+    { for i, v in azurerm_windows_function_app.nbs_mya_service_bus_func_app : "sb_func_${i}" => v.identity[0].principal_id },
+    { for i, v in azurerm_windows_function_app_slot.nbs_mya_service_bus_func_app_preview : "sb_preview_${i}" => v.identity[0].principal_id },
+
+    { for i, v in azurerm_windows_function_app.nbs_mya_timer_func_app : "timer_${i}" => v.identity[0].principal_id },
+    { for i, v in azurerm_windows_function_app_slot.nbs_mya_timer_func_app_preview : "timer_preview_${i}" => v.identity[0].principal_id }
+  )
+}
+
+resource "azurerm_role_assignment" "vault_access" {
+  for_each             = local.principals_map
+  scope                = azurerm_key_vault.nbs_mya_key_vault.id
+  role_definition_name = "Key Vault Secrets User"
+  principal_id         = each.value
+}

infrastructure/resources/servicebus_functions.tf

@@ -57,6 +57,7 @@ resource "azurerm_windows_function_app" "nbs_mya_service_bus_func_app" {
     DISABLE_SITE_CACHE                                                     = var.disable_site_cache
     CANCEL_A_DATE_RANGE_MAXIMUM_DAYS                                       = var.cancel_a_date_range_maximum_days
     APPLICATION_NAME                                                       = "Service Bus Function App"
+    KEY_VAULT_URI                                                          = azurerm_key_vault.nbs_mya_key_vault.vault_uri
     "AzureWebJobs.ApplyAvailabilityTemplateFunction.Disabled"              = true
     "AzureWebJobs.AuthenticateCallbackFunction.Disabled"                   = true
     "AzureWebJobs.AuthenticateFunction.Disabled"                           = true
@@ -192,6 +193,7 @@ resource "azurerm_windows_function_app_slot" "nbs_mya_service_bus_func_app_previ
     SITE_SUMMARY_MINIMUM_PARALLELIZATION                                   = var.site_summary_minimum_parallelization
     CANCEL_A_DATE_RANGE_MAXIMUM_DAYS                                       = var.cancel_a_date_range_maximum_days
     APPLICATION_NAME                                                       = "Service Bus Function App"
+    KEY_VAULT_URI                                                          = azurerm_key_vault.nbs_mya_key_vault.vault_uri
     "AzureWebJobs.ApplyAvailabilityTemplateFunction.Disabled"              = true
     "AzureWebJobs.AuthenticateCallbackFunction.Disabled"                   = true
     "AzureWebJobs.AuthenticateFunction.Disabled"                           = true

infrastructure/resources/timer_functions.tf

@@ -61,6 +61,7 @@ resource "azurerm_windows_function_app" "nbs_mya_timer_func_app" {
     AutoCancelledBookingsCronSchedule                                      = var.auto_cancelled_bookings_cron_schedule
     CANCEL_A_DATE_RANGE_MAXIMUM_DAYS                                       = var.cancel_a_date_range_maximum_days
     APPLICATION_NAME                                                       = "Timer Function App"
+    KEY_VAULT_URI                                                          = azurerm_key_vault.nbs_mya_key_vault.vault_uri
     "AzureWebJobs.NotifyBookingCancelled.Disabled"                         = true
     "AzureWebJobs.NotifyBookingMade.Disabled"                              = true
     "AzureWebJobs.NotifyBookingReminder.Disabled"                          = true
@@ -201,6 +202,7 @@ resource "azurerm_windows_function_app_slot" "nbs_mya_timer_func_app_preview" {
     AutoCancelledBookingsCronSchedule                                      = var.auto_cancelled_bookings_cron_schedule
     CANCEL_A_DATE_RANGE_MAXIMUM_DAYS                                       = var.cancel_a_date_range_maximum_days
     APPLICATION_NAME                                                       = "Timer Function App"
+    KEY_VAULT_URI                                                          = azurerm_key_vault.nbs_mya_key_vault.vault_uri
     "AzureWebJobs.NotifyBookingCancelled.Disabled"                         = true
     "AzureWebJobs.NotifyBookingMade.Disabled"                              = true
     "AzureWebJobs.NotifyBookingReminder.Disabled"                          = true

scripts/disaster-recovery/create-audit-blob-storage.ps1

@@ -0,0 +1,12 @@
+param (
+[string][Parameter(Mandatory)]$resourceGroup,
+[string][Parameter(Mandatory)]$blobAccountName,
+[string][Parameter(Mandatory)]$subscriptionId
+)
+
+
+az storage account create --name $blobAccountName --resource-group $resourceGroup --subscription $subscriptionId --location uksouth --sku Standard_LRS --min-tls-version TLS1_2
+
+if ($LASTEXITCODE -ne 0) {
+  throw "Failed to create new storage account '$blobAccountName' in '$resourceGroup'"
+}