feat: [DTOSS-12154] disabled shared access key for storage account and enabled use managed identity for storage account

.azuredevops/pipelines/acr-image-retag.yaml

@@ -9,7 +9,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/adhoc-post-deployment-tests-dev.yaml

@@ -8,7 +8,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 variables:

.azuredevops/pipelines/cd-infrastructure-dev-audit.yaml

@@ -13,7 +13,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 variables:

.azuredevops/pipelines/cd-infrastructure-dev-core.yaml

@@ -9,7 +9,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/cd-infrastructure-int-audit.yaml

@@ -13,7 +13,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 variables:

.azuredevops/pipelines/cd-infrastructure-int-core.yaml

@@ -9,7 +9,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/cd-infrastructure-nft-audit.yaml

@@ -13,7 +13,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 variables:

.azuredevops/pipelines/cd-infrastructure-nft-core.yaml

@@ -9,7 +9,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/cd-infrastructure-preprod-audit.yaml

@@ -13,7 +13,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 variables:

.azuredevops/pipelines/cd-infrastructure-preprod-core.yaml

@@ -9,7 +9,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/cd-infrastructure-prod-audit.yaml

@@ -13,7 +13,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 variables:

.azuredevops/pipelines/cd-infrastructure-prod-core.yaml

@@ -9,7 +9,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/cd-infrastructure-sandbox-audit.yaml

@@ -13,7 +13,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/cd-infrastructure-sandbox-core.yaml

@@ -9,15 +9,15 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:
   - name: dockerImageTag
     # Used to set the Docker image tag via Terraform in the Continuous Deployment settings of function and container apps
     displayName: 'Docker Image Tag'
     type: string
-    default: sbrk
+    default: sb
 
   - name: retagImages
     # Only required until commit hashes are passed in by calling pipeline, except for the
@@ -36,7 +36,7 @@ parameters:
     # Only required until commit hashes are passed in by calling pipeline
     displayName: 'Docker Image Retag to value'
     type: string
-    default: sbrk
+    default: sb
 
   - name: terraformActions
     # This parameter allows the user to run only the plan for testing purposes

.azuredevops/pipelines/owasp-zap-report-dev-core.yaml

@@ -16,7 +16,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 variables:

.azuredevops/pipelines/owasp-zap-report-sandbox-core.yaml

@@ -8,7 +8,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 variables:
@@ -24,5 +24,5 @@ stages:
       hostPoolName: $(hostPoolName)
       environment: $(ENVIRONMENT)
       slackWebHook: $(SLACK_WEBHOOK_URL_WORKFLOWS)
-      apiUrl: 'https://sbrk-uks-retrieve-cohort-distribution-data.azurewebsites.net/api/RetrieveCohortDistributionData'
+      apiUrl: 'https://sb-uks-retrieve-cohort-distribution-data.azurewebsites.net/api/RetrieveCohortDistributionData'
       webUrl: 'https://cohort-dev.non-live.screening.nhs.uk'

.azuredevops/pipelines/post-deployment-tests-dev.yaml

@@ -16,7 +16,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 variables:

.azuredevops/pipelines/task_azure_sql_backup_prod.yaml

@@ -17,7 +17,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 variables:

.azuredevops/pipelines/task_azure_sql_scaling_dev.yaml

@@ -27,7 +27,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/task_azure_sql_scaling_int.yaml

@@ -27,7 +27,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/task_azure_sql_scaling_nft.yaml

@@ -27,7 +27,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/task_azure_sql_scaling_preprod.yaml

@@ -27,7 +27,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

.azuredevops/pipelines/task_azure_sql_scaling_prod.yaml

@@ -27,7 +27,7 @@ resources:
     - repository: dtos-devops-templates
       type: github
       name: NHSDigital/dtos-devops-templates
-      ref: 0abead1e42da2bb60cbc85054ac2452746679d29
+      ref: d4f16d361f70210e56cb698f5d4bbba14a42c892
       endpoint: NHSDigital
 
 parameters:

application/CohortManager/src/Functions/AuditServices/AuditWriter/AuditConfig.cs

@@ -1,13 +1,13 @@
 namespace NHS.CohortManager.AuditServices;
 
+using Common;
 using System.ComponentModel.DataAnnotations;
 
 public class AuditConfig
 {
     [Required]
     public required string ServiceBusConnectionString { get; set; }
-    [Required]
-    public required string AzureWebJobsStorage { get; set; }
+    public BlobStorageConfig? AzureWebJobsStorage { get; set; }
     [Required]
     public required string AuditTopicName { get; set; }
 }

application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/Program.cs

@@ -68,7 +68,7 @@
         services.AddSingleton<IBlobStorageHelper, BlobStorageHelper>();
         services.AddTransient<IMeshToBlobTransferHandler, MeshToBlobTransferHandler>();
         // Register health checks
-        services.AddBlobStorageHealthCheck("RetrieveMeshFile");
+        services.AddBlobStorageHealthCheck("RetrieveMeshFile", config.AzureWebJobsStorage!);
     })
     .AddTelemetry()
     .AddExceptionHandler();

application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/RetrieveMeshFile.cs

@@ -1,17 +1,15 @@
 namespace NHS.Screening.RetrieveMeshFile;
 
-using System;
-using System.Diagnostics.CodeAnalysis;
-using System.Globalization;
-using System.Text;
-using System.Text.Json;
-using System.Threading.Tasks;
 using Common;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Model;
 using NHS.MESH.Client.Models;
+using System;
+using System.Globalization;
+using System.Text.Json;
+using System.Threading.Tasks;
 
 
 public class RetrieveMeshFile
@@ -20,11 +18,13 @@ public class RetrieveMeshFile
 
     private readonly IMeshToBlobTransferHandler _meshToBlobTransferHandler;
     private readonly string _mailboxId;
-    private readonly string _blobConnectionString;
+    private readonly string? _blobConnectionString;
+    private readonly Uri? _blobServiceUri;
     private readonly IBlobStorageHelper _blobStorageHelper;
     private readonly RetrieveMeshFileConfig _config;
     private const string NextHandShakeTimeConfigKey = "NextHandShakeTime";
     private const string ConfigFileName = "MeshState.json";
+    private const string ConfigContainerName = "config";
 
     public RetrieveMeshFile(ILogger<RetrieveMeshFile> logger, IMeshToBlobTransferHandler meshToBlobTransferHandler, IBlobStorageHelper blobStorageHelper, IOptions<RetrieveMeshFileConfig> options)
     {
@@ -33,7 +33,14 @@ public RetrieveMeshFile(ILogger<RetrieveMeshFile> logger, IMeshToBlobTransferHan
         _blobStorageHelper = blobStorageHelper;
         _mailboxId = options.Value.BSSMailBox;
         _config = options.Value;
-        _blobConnectionString = _config.caasfolder_STORAGE;
+        if (_config.nemsmeshfolder_STORAGE != null)
+        {
+            _blobServiceUri = new Uri(_config.nemsmeshfolder_STORAGE.BlobServiceUri);
+        }
+        else
+        {
+            _blobConnectionString = Environment.GetEnvironmentVariable("nemsmeshfolder_STORAGE");
+        }
     }
     /// <summary>
     /// This function polls the MESH Mailbox every 5 minutes, if there is a file posted to the mailbox.
@@ -51,8 +58,7 @@ public async Task RunAsync([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer)
         try
         {
             var shouldExecuteHandShake = await ShouldExecuteHandShake();
-            var result = await _meshToBlobTransferHandler.MoveFilesFromMeshToBlob(messageFilter, fileNameFunction, _mailboxId, _blobConnectionString, "inbound", shouldExecuteHandShake);
-
+            var result = await _meshToBlobTransferHandler.MoveFilesFromMeshToBlob(messageFilter, fileNameFunction, _mailboxId, _blobServiceUri, _blobConnectionString, "inbound", shouldExecuteHandShake);
             if (!result)
             {
                 _logger.LogError("An error was encountered while moving files from Mesh to Blob");
@@ -74,10 +80,18 @@ private async Task<bool> ShouldExecuteHandShake()
 
         Dictionary<string, string> configValues;
         TimeSpan handShakeInterval = new TimeSpan(0, 23, 54, 0);
-        var meshState = await _blobStorageHelper.GetFileFromBlobStorage(_blobConnectionString, "config", ConfigFileName);
-        if (meshState == null)
+        BlobFile? meshState = null;
+        if (_blobServiceUri != null)
         {
+            meshState = await _blobStorageHelper.GetFileFromBlobStorage(_blobServiceUri, ConfigContainerName, ConfigFileName);
+        }
+        else if (_blobConnectionString != null)
+        {
+            meshState = await _blobStorageHelper.GetFileFromBlobStorage(_blobConnectionString, ConfigContainerName, ConfigFileName);
+        }
 
+        if (meshState == null)
+        {
             _logger.LogInformation("MeshState File did not exist, Creating new MeshState File in blob Storage");
             configValues = new Dictionary<string, string>
             {
@@ -140,7 +154,15 @@ private async Task<bool> SetConfigState(Dictionary<string, string> state)
             using (var stream = GenerateStreamFromString(jsonString))
             {
                 var blobFile = new BlobFile(stream, ConfigFileName);
-                var result = await _blobStorageHelper.UploadFileToBlobStorage(_blobConnectionString, "config", blobFile, true);
+                var result = false;
+                if (_blobServiceUri != null)
+                {
+                    result = await _blobStorageHelper.UploadFileToBlobStorage(_blobServiceUri, ConfigContainerName, blobFile, true);
+                }
+                else if (_blobConnectionString != null)
+                {
+                    result = await _blobStorageHelper.UploadFileToBlobStorage(_blobConnectionString, ConfigContainerName, blobFile, true);
+                }
                 return result;
             }
         }

application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/RetrieveMeshFileConfig.cs

@@ -1,5 +1,6 @@
 namespace NHS.Screening.RetrieveMeshFile;
 
+using Common;
 using System.ComponentModel.DataAnnotations;
 
 public class RetrieveMeshFileConfig
@@ -14,8 +15,8 @@ public class RetrieveMeshFileConfig
     public string? MeshKeyPassphrase { get; set; }
     public string? MeshKeyName { get; set; }
     public string KeyVaultConnectionString { get; set; }
-    [Required]
-    public required string caasfolder_STORAGE { get; set; }
+    public BlobStorageConfig? AzureWebJobsStorage { get; set; }
+    public BlobStorageConfig? nemsmeshfolder_STORAGE { get; set; }
     public string? ServerSideCerts { get; set; }
     public string? MeshCertName { get; set; }
     public bool? BypassServerCertificateValidation { get; set; }

application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/ProcessFileClasses/CopyFailedBatchToBlob.cs

@@ -29,7 +29,19 @@ public async Task<bool> writeBatchToBlob(string jsonFromBatch, InvalidOperationE
         {
             // we do this so that we do not have files with the same names either failing to be added or over writing another failed batch
             var blobFile = new BlobFile(stream, $"failedBatch-{Guid.NewGuid()}.json");
-            var copied = await _blobStorageHelper.UploadFileToBlobStorage(_config.caasfolder_STORAGE, "failed-batch", blobFile);
+            var copied = false;
+            if (_config.caasfolder_STORAGE != null)
+            {
+                copied = await _blobStorageHelper.UploadFileToBlobStorage(new Uri(_config.caasfolder_STORAGE.BlobServiceUri), "failed-batch", blobFile);
+            }
+            else
+            {
+                var connectionString = Environment.GetEnvironmentVariable("caasfolder_STORAGE");
+                if (connectionString != null)
+                {
+                    copied = await _blobStorageHelper.UploadFileToBlobStorage(connectionString, "failed-batch", blobFile);
+                }
+            }
 
             if (copied)
             {