diff --git a/.azuredevops/pipelines/acr-image-retag.yaml b/.azuredevops/pipelines/acr-image-retag.yaml index 71996d0055..121bdf1293 100644 --- a/.azuredevops/pipelines/acr-image-retag.yaml +++ b/.azuredevops/pipelines/acr-image-retag.yaml @@ -9,7 +9,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/adhoc-post-deployment-tests-dev.yaml b/.azuredevops/pipelines/adhoc-post-deployment-tests-dev.yaml index 60bb4a2ea8..d5a5341de1 100644 --- a/.azuredevops/pipelines/adhoc-post-deployment-tests-dev.yaml +++ b/.azuredevops/pipelines/adhoc-post-deployment-tests-dev.yaml @@ -8,7 +8,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital variables: diff --git a/.azuredevops/pipelines/cd-infrastructure-dev-audit.yaml b/.azuredevops/pipelines/cd-infrastructure-dev-audit.yaml index 143699b07e..2ddde13b08 100644 --- a/.azuredevops/pipelines/cd-infrastructure-dev-audit.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-dev-audit.yaml @@ -13,7 +13,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital variables: diff --git a/.azuredevops/pipelines/cd-infrastructure-dev-core.yaml b/.azuredevops/pipelines/cd-infrastructure-dev-core.yaml index 2c6e721821..6772271587 100644 --- a/.azuredevops/pipelines/cd-infrastructure-dev-core.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-dev-core.yaml @@ -9,7 +9,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/cd-infrastructure-int-audit.yaml b/.azuredevops/pipelines/cd-infrastructure-int-audit.yaml index 2ac72fd918..5436abab07 100644 --- a/.azuredevops/pipelines/cd-infrastructure-int-audit.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-int-audit.yaml @@ -13,7 +13,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital variables: diff --git a/.azuredevops/pipelines/cd-infrastructure-int-core.yaml b/.azuredevops/pipelines/cd-infrastructure-int-core.yaml index 50ced23bcd..4ca7aa4109 100644 --- a/.azuredevops/pipelines/cd-infrastructure-int-core.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-int-core.yaml @@ -9,7 +9,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/cd-infrastructure-nft-audit.yaml b/.azuredevops/pipelines/cd-infrastructure-nft-audit.yaml index f96fe3d829..42d1223f70 100644 --- a/.azuredevops/pipelines/cd-infrastructure-nft-audit.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-nft-audit.yaml @@ -13,7 +13,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital variables: diff --git a/.azuredevops/pipelines/cd-infrastructure-nft-core.yaml b/.azuredevops/pipelines/cd-infrastructure-nft-core.yaml index ed8d9bed8f..f49f559991 100644 --- a/.azuredevops/pipelines/cd-infrastructure-nft-core.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-nft-core.yaml @@ -9,7 +9,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/cd-infrastructure-preprod-audit.yaml b/.azuredevops/pipelines/cd-infrastructure-preprod-audit.yaml index d842aeded9..9ab606def6 100644 --- a/.azuredevops/pipelines/cd-infrastructure-preprod-audit.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-preprod-audit.yaml @@ -13,7 +13,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital variables: diff --git a/.azuredevops/pipelines/cd-infrastructure-preprod-core.yaml b/.azuredevops/pipelines/cd-infrastructure-preprod-core.yaml index 8fb9488d93..86d360dcdc 100644 --- a/.azuredevops/pipelines/cd-infrastructure-preprod-core.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-preprod-core.yaml @@ -9,7 +9,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/cd-infrastructure-prod-audit.yaml b/.azuredevops/pipelines/cd-infrastructure-prod-audit.yaml index 243f125c2e..ae051fc91d 100644 --- a/.azuredevops/pipelines/cd-infrastructure-prod-audit.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-prod-audit.yaml @@ -13,7 +13,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital variables: diff --git a/.azuredevops/pipelines/cd-infrastructure-prod-core.yaml b/.azuredevops/pipelines/cd-infrastructure-prod-core.yaml index 77f34bfce4..e9b16c96c4 100644 --- a/.azuredevops/pipelines/cd-infrastructure-prod-core.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-prod-core.yaml @@ -9,7 +9,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/cd-infrastructure-sandbox-audit.yaml b/.azuredevops/pipelines/cd-infrastructure-sandbox-audit.yaml index b7777f0c9f..4c6fe2ebf5 100644 --- a/.azuredevops/pipelines/cd-infrastructure-sandbox-audit.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-sandbox-audit.yaml @@ -13,7 +13,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/cd-infrastructure-sandbox-core.yaml b/.azuredevops/pipelines/cd-infrastructure-sandbox-core.yaml index 51f57bcd50..98fee3360b 100644 --- a/.azuredevops/pipelines/cd-infrastructure-sandbox-core.yaml +++ b/.azuredevops/pipelines/cd-infrastructure-sandbox-core.yaml @@ -9,7 +9,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: @@ -17,7 +17,7 @@ parameters: # Used to set the Docker image tag via Terraform in the Continuous Deployment settings of function and container apps displayName: 'Docker Image Tag' type: string - default: sbrk + default: sb - name: retagImages # Only required until commit hashes are passed in by calling pipeline, except for the @@ -36,7 +36,7 @@ parameters: # Only required until commit hashes are passed in by calling pipeline displayName: 'Docker Image Retag to value' type: string - default: sbrk + default: sb - name: terraformActions # This parameter allows the user to run only the plan for testing purposes diff --git a/.azuredevops/pipelines/owasp-zap-report-dev-core.yaml b/.azuredevops/pipelines/owasp-zap-report-dev-core.yaml index 3fea9407da..aaf9a5923a 100644 --- a/.azuredevops/pipelines/owasp-zap-report-dev-core.yaml +++ b/.azuredevops/pipelines/owasp-zap-report-dev-core.yaml @@ -16,7 +16,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital variables: diff --git a/.azuredevops/pipelines/owasp-zap-report-sandbox-core.yaml b/.azuredevops/pipelines/owasp-zap-report-sandbox-core.yaml index b6d760764b..cc86a0fa4d 100644 --- a/.azuredevops/pipelines/owasp-zap-report-sandbox-core.yaml +++ b/.azuredevops/pipelines/owasp-zap-report-sandbox-core.yaml @@ -8,7 +8,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital variables: @@ -24,5 +24,5 @@ stages: hostPoolName: $(hostPoolName) environment: $(ENVIRONMENT) slackWebHook: $(SLACK_WEBHOOK_URL_WORKFLOWS) - apiUrl: 'https://sbrk-uks-retrieve-cohort-distribution-data.azurewebsites.net/api/RetrieveCohortDistributionData' + apiUrl: 'https://sb-uks-retrieve-cohort-distribution-data.azurewebsites.net/api/RetrieveCohortDistributionData' webUrl: 'https://cohort-dev.non-live.screening.nhs.uk' diff --git a/.azuredevops/pipelines/post-deployment-tests-dev.yaml b/.azuredevops/pipelines/post-deployment-tests-dev.yaml index 1297a50b6d..bcccaf4086 100644 --- a/.azuredevops/pipelines/post-deployment-tests-dev.yaml +++ b/.azuredevops/pipelines/post-deployment-tests-dev.yaml @@ -16,7 +16,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital variables: diff --git a/.azuredevops/pipelines/task_azure_sql_backup_prod.yaml b/.azuredevops/pipelines/task_azure_sql_backup_prod.yaml index e188f308fe..14d1ef2714 100644 --- a/.azuredevops/pipelines/task_azure_sql_backup_prod.yaml +++ b/.azuredevops/pipelines/task_azure_sql_backup_prod.yaml @@ -17,7 +17,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital variables: diff --git a/.azuredevops/pipelines/task_azure_sql_scaling_dev.yaml b/.azuredevops/pipelines/task_azure_sql_scaling_dev.yaml index ee27ffa29e..e29c5874a9 100644 --- a/.azuredevops/pipelines/task_azure_sql_scaling_dev.yaml +++ b/.azuredevops/pipelines/task_azure_sql_scaling_dev.yaml @@ -27,7 +27,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/task_azure_sql_scaling_int.yaml b/.azuredevops/pipelines/task_azure_sql_scaling_int.yaml index d58149b5ee..64d8b75686 100644 --- a/.azuredevops/pipelines/task_azure_sql_scaling_int.yaml +++ b/.azuredevops/pipelines/task_azure_sql_scaling_int.yaml @@ -27,7 +27,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/task_azure_sql_scaling_nft.yaml b/.azuredevops/pipelines/task_azure_sql_scaling_nft.yaml index 705372db9f..4c2fb78ccb 100644 --- a/.azuredevops/pipelines/task_azure_sql_scaling_nft.yaml +++ b/.azuredevops/pipelines/task_azure_sql_scaling_nft.yaml @@ -27,7 +27,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/task_azure_sql_scaling_preprod.yaml b/.azuredevops/pipelines/task_azure_sql_scaling_preprod.yaml index 75f8c4a5c6..3b64d1ea34 100644 --- a/.azuredevops/pipelines/task_azure_sql_scaling_preprod.yaml +++ b/.azuredevops/pipelines/task_azure_sql_scaling_preprod.yaml @@ -27,7 +27,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/.azuredevops/pipelines/task_azure_sql_scaling_prod.yaml b/.azuredevops/pipelines/task_azure_sql_scaling_prod.yaml index 27f0d3cca9..d07d8f44c8 100644 --- a/.azuredevops/pipelines/task_azure_sql_scaling_prod.yaml +++ b/.azuredevops/pipelines/task_azure_sql_scaling_prod.yaml @@ -27,7 +27,7 @@ resources: - repository: dtos-devops-templates type: github name: NHSDigital/dtos-devops-templates - ref: 0abead1e42da2bb60cbc85054ac2452746679d29 + ref: d4f16d361f70210e56cb698f5d4bbba14a42c892 endpoint: NHSDigital parameters: diff --git a/application/CohortManager/src/Functions/AuditServices/AuditWriter/AuditConfig.cs b/application/CohortManager/src/Functions/AuditServices/AuditWriter/AuditConfig.cs index 0bff129799..cd26fbb096 100644 --- a/application/CohortManager/src/Functions/AuditServices/AuditWriter/AuditConfig.cs +++ b/application/CohortManager/src/Functions/AuditServices/AuditWriter/AuditConfig.cs @@ -1,13 +1,13 @@ namespace NHS.CohortManager.AuditServices; +using Common; using System.ComponentModel.DataAnnotations; public class AuditConfig { [Required] public required string ServiceBusConnectionString { get; set; } - [Required] - public required string AzureWebJobsStorage { get; set; } + public BlobStorageConfig? AzureWebJobsStorage { get; set; } [Required] public required string AuditTopicName { get; set; } } diff --git a/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/Program.cs b/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/Program.cs index 7e768799f4..ee2dac6914 100644 --- a/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/Program.cs +++ b/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/Program.cs @@ -68,7 +68,7 @@ services.AddSingleton(); services.AddTransient(); // Register health checks - services.AddBlobStorageHealthCheck("RetrieveMeshFile"); + services.AddBlobStorageHealthCheck("RetrieveMeshFile", config.AzureWebJobsStorage!); }) .AddTelemetry() .AddExceptionHandler(); diff --git a/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/RetrieveMeshFile.cs b/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/RetrieveMeshFile.cs index 3307c94078..c8672f47fc 100644 --- a/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/RetrieveMeshFile.cs +++ b/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/RetrieveMeshFile.cs @@ -1,17 +1,15 @@ namespace NHS.Screening.RetrieveMeshFile; -using System; -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; using Common; using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Model; using NHS.MESH.Client.Models; +using System; +using System.Globalization; +using System.Text.Json; +using System.Threading.Tasks; public class RetrieveMeshFile @@ -20,11 +18,13 @@ public class RetrieveMeshFile private readonly IMeshToBlobTransferHandler _meshToBlobTransferHandler; private readonly string _mailboxId; - private readonly string _blobConnectionString; + private readonly string? _blobConnectionString; + private readonly Uri? _blobServiceUri; private readonly IBlobStorageHelper _blobStorageHelper; private readonly RetrieveMeshFileConfig _config; private const string NextHandShakeTimeConfigKey = "NextHandShakeTime"; private const string ConfigFileName = "MeshState.json"; + private const string ConfigContainerName = "config"; public RetrieveMeshFile(ILogger logger, IMeshToBlobTransferHandler meshToBlobTransferHandler, IBlobStorageHelper blobStorageHelper, IOptions options) { @@ -33,7 +33,14 @@ public RetrieveMeshFile(ILogger logger, IMeshToBlobTransferHan _blobStorageHelper = blobStorageHelper; _mailboxId = options.Value.BSSMailBox; _config = options.Value; - _blobConnectionString = _config.caasfolder_STORAGE; + if (_config.nemsmeshfolder_STORAGE != null) + { + _blobServiceUri = new Uri(_config.nemsmeshfolder_STORAGE.BlobServiceUri); + } + else + { + _blobConnectionString = Environment.GetEnvironmentVariable("nemsmeshfolder_STORAGE"); + } } /// /// This function polls the MESH Mailbox every 5 minutes, if there is a file posted to the mailbox. @@ -51,8 +58,7 @@ public async Task RunAsync([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer) try { var shouldExecuteHandShake = await ShouldExecuteHandShake(); - var result = await _meshToBlobTransferHandler.MoveFilesFromMeshToBlob(messageFilter, fileNameFunction, _mailboxId, _blobConnectionString, "inbound", shouldExecuteHandShake); - + var result = await _meshToBlobTransferHandler.MoveFilesFromMeshToBlob(messageFilter, fileNameFunction, _mailboxId, _blobServiceUri, _blobConnectionString, "inbound", shouldExecuteHandShake); if (!result) { _logger.LogError("An error was encountered while moving files from Mesh to Blob"); @@ -74,10 +80,18 @@ private async Task ShouldExecuteHandShake() Dictionary configValues; TimeSpan handShakeInterval = new TimeSpan(0, 23, 54, 0); - var meshState = await _blobStorageHelper.GetFileFromBlobStorage(_blobConnectionString, "config", ConfigFileName); - if (meshState == null) + BlobFile? meshState = null; + if (_blobServiceUri != null) { + meshState = await _blobStorageHelper.GetFileFromBlobStorage(_blobServiceUri, ConfigContainerName, ConfigFileName); + } + else if (_blobConnectionString != null) + { + meshState = await _blobStorageHelper.GetFileFromBlobStorage(_blobConnectionString, ConfigContainerName, ConfigFileName); + } + if (meshState == null) + { _logger.LogInformation("MeshState File did not exist, Creating new MeshState File in blob Storage"); configValues = new Dictionary { @@ -140,7 +154,15 @@ private async Task SetConfigState(Dictionary state) using (var stream = GenerateStreamFromString(jsonString)) { var blobFile = new BlobFile(stream, ConfigFileName); - var result = await _blobStorageHelper.UploadFileToBlobStorage(_blobConnectionString, "config", blobFile, true); + var result = false; + if (_blobServiceUri != null) + { + result = await _blobStorageHelper.UploadFileToBlobStorage(_blobServiceUri, ConfigContainerName, blobFile, true); + } + else if (_blobConnectionString != null) + { + result = await _blobStorageHelper.UploadFileToBlobStorage(_blobConnectionString, ConfigContainerName, blobFile, true); + } return result; } } diff --git a/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/RetrieveMeshFileConfig.cs b/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/RetrieveMeshFileConfig.cs index 0502e101f1..7d36ea6b4a 100644 --- a/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/RetrieveMeshFileConfig.cs +++ b/application/CohortManager/src/Functions/CaasIntegration/RetrieveMeshFile/RetrieveMeshFileConfig.cs @@ -1,5 +1,6 @@ namespace NHS.Screening.RetrieveMeshFile; +using Common; using System.ComponentModel.DataAnnotations; public class RetrieveMeshFileConfig @@ -14,8 +15,8 @@ public class RetrieveMeshFileConfig public string? MeshKeyPassphrase { get; set; } public string? MeshKeyName { get; set; } public string KeyVaultConnectionString { get; set; } - [Required] - public required string caasfolder_STORAGE { get; set; } + public BlobStorageConfig? AzureWebJobsStorage { get; set; } + public BlobStorageConfig? nemsmeshfolder_STORAGE { get; set; } public string? ServerSideCerts { get; set; } public string? MeshCertName { get; set; } public bool? BypassServerCertificateValidation { get; set; } diff --git a/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/ProcessFileClasses/CopyFailedBatchToBlob.cs b/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/ProcessFileClasses/CopyFailedBatchToBlob.cs index 61858f07ca..0bbd33ae50 100644 --- a/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/ProcessFileClasses/CopyFailedBatchToBlob.cs +++ b/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/ProcessFileClasses/CopyFailedBatchToBlob.cs @@ -29,7 +29,19 @@ public async Task writeBatchToBlob(string jsonFromBatch, InvalidOperationE { // we do this so that we do not have files with the same names either failing to be added or over writing another failed batch var blobFile = new BlobFile(stream, $"failedBatch-{Guid.NewGuid()}.json"); - var copied = await _blobStorageHelper.UploadFileToBlobStorage(_config.caasfolder_STORAGE, "failed-batch", blobFile); + var copied = false; + if (_config.caasfolder_STORAGE != null) + { + copied = await _blobStorageHelper.UploadFileToBlobStorage(new Uri(_config.caasfolder_STORAGE.BlobServiceUri), "failed-batch", blobFile); + } + else + { + var connectionString = Environment.GetEnvironmentVariable("caasfolder_STORAGE"); + if (connectionString != null) + { + copied = await _blobStorageHelper.UploadFileToBlobStorage(connectionString, "failed-batch", blobFile); + } + } if (copied) { diff --git a/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/Program.cs b/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/Program.cs index 1eac0759d8..c4220a8b54 100644 --- a/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/Program.cs +++ b/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/Program.cs @@ -39,7 +39,7 @@ services.AddScoped(); services.AddTransient(); // Register health checks - services.AddBlobStorageHealthCheck("receiveCaasFile"); + services.AddBlobStorageHealthCheck("receiveCaasFile", config.AzureWebJobsStorage!); }) .AddTelemetry() .AddHttpClient() diff --git a/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/receiveCaasFile.cs b/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/receiveCaasFile.cs index 61532c4f36..9b31061c9e 100644 --- a/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/receiveCaasFile.cs +++ b/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/receiveCaasFile.cs @@ -106,7 +106,18 @@ public async Task Run([BlobTrigger("inbound/{name}", Connection = "caasfolder_ST { _logger.LogError(ex, "There was a system exception in receive-caas-file"); await _exceptionHandler.CreateSystemExceptionLogFromNhsNumber(ex, "", name, screeningName, ""); - await _blobStorageHelper.CopyFileToPoisonAsync(_config.caasfolder_STORAGE, name, _config.inboundBlobName); + if (_config.caasfolder_STORAGE != null) + { + await _blobStorageHelper.CopyFileToPoisonAsync(new Uri(_config.caasfolder_STORAGE.BlobServiceUri), name, _config.inboundBlobName); + } + else + { + var connectionString = Environment.GetEnvironmentVariable("caasfolder_STORAGE"); + if (connectionString != null) + { + await _blobStorageHelper.CopyFileToPoisonAsync(connectionString, name, _config.inboundBlobName); + } + } } finally { diff --git a/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/receiveCaasFileConfig.cs b/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/receiveCaasFileConfig.cs index e72e446c98..6fc61dd9dc 100644 --- a/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/receiveCaasFileConfig.cs +++ b/application/CohortManager/src/Functions/CaasIntegration/receiveCaasFile/receiveCaasFileConfig.cs @@ -1,5 +1,6 @@ namespace NHS.Screening.ReceiveCaasFile; +using Common; using System.ComponentModel.DataAnnotations; public class ReceiveCaasFileConfig @@ -15,8 +16,8 @@ public class ReceiveCaasFileConfig public bool AllowDeleteRecords { get; set; } [Required] public required int maxNumberOfChecks { get; set; } - [Required] - public required string caasfolder_STORAGE { get; set; } + public BlobStorageConfig? AzureWebJobsStorage { get; set; } + public BlobStorageConfig? caasfolder_STORAGE { get; set; } [Required] public required string inboundBlobName { get; set; } [Required] diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/NemsMeshRetrieval.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/NemsMeshRetrieval.cs index 93916a624a..c7c9de9e30 100644 --- a/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/NemsMeshRetrieval.cs +++ b/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/NemsMeshRetrieval.cs @@ -1,15 +1,15 @@ namespace NHS.Screening.NemsMeshRetrieval; -using System; -using System.Globalization; -using System.Text.Json; -using System.Threading.Tasks; using Common; using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Model; using NHS.MESH.Client.Models; +using System; +using System.Globalization; +using System.Text.Json; +using System.Threading.Tasks; public class NemsMeshRetrieval @@ -18,7 +18,8 @@ public class NemsMeshRetrieval private readonly IMeshToBlobTransferHandler _meshToBlobTransferHandler; private readonly string _mailboxId; - private readonly string _blobConnectionString; + private readonly string? _blobConnectionString; + private readonly Uri? _blobServiceUri; private readonly IBlobStorageHelper _blobStorageHelper; private readonly NemsMeshRetrievalConfig _config; private const string NextHandShakeTimeConfigKey = "NextHandShakeTime"; @@ -31,7 +32,14 @@ public NemsMeshRetrieval(ILogger logger, IMeshToBlobTransferH _blobStorageHelper = blobStorageHelper; _mailboxId = options.Value.NemsMeshMailBox; _config = options.Value; - _blobConnectionString = _config.nemsmeshfolder_STORAGE; + if (_config.nemsmeshfolder_STORAGE != null) + { + _blobServiceUri = new Uri(_config.nemsmeshfolder_STORAGE.BlobServiceUri); + } + else + { + _blobConnectionString = Environment.GetEnvironmentVariable("nemsmeshfolder_STORAGE"); + } } /// /// This function polls the MESH Mailbox every 5 minutes, if there is a file posted to the mailbox. @@ -49,7 +57,7 @@ public async Task RunAsync([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer) try { var shouldExecuteHandShake = await ShouldExecuteHandShake(); - var result = await _meshToBlobTransferHandler.MoveFilesFromMeshToBlob(messageFilter, fileNameFunction, _mailboxId, _blobConnectionString, _config.NemsMeshInboundContainer, shouldExecuteHandShake); + var result = await _meshToBlobTransferHandler.MoveFilesFromMeshToBlob(messageFilter, fileNameFunction, _mailboxId, _blobServiceUri, _blobConnectionString, _config.NemsMeshInboundContainer, shouldExecuteHandShake); if (!result) { @@ -69,13 +77,19 @@ public async Task RunAsync([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer) private async Task ShouldExecuteHandShake() { - Dictionary configValues; TimeSpan handShakeInterval = new TimeSpan(0, 23, 54, 0); - var meshState = await _blobStorageHelper.GetFileFromBlobStorage(_blobConnectionString, _config.NemsMeshConfigContainer, ConfigFileName); + BlobFile? meshState = null; + if (_blobServiceUri != null) + { + meshState = await _blobStorageHelper.GetFileFromBlobStorage(_blobServiceUri, _config.NemsMeshConfigContainer, ConfigFileName); + } + else if (_blobConnectionString != null) + { + meshState = await _blobStorageHelper.GetFileFromBlobStorage(_blobConnectionString, _config.NemsMeshConfigContainer, ConfigFileName); + } if (meshState == null) { - _logger.LogInformation("MeshState File did not exist, Creating new MeshState File in blob Storage"); configValues = new Dictionary { @@ -84,8 +98,8 @@ private async Task ShouldExecuteHandShake() await SetConfigState(configValues); return true; - } + using (StreamReader reader = new StreamReader(meshState.Data)) { meshState.Data.Seek(0, SeekOrigin.Begin); @@ -101,8 +115,6 @@ private async Task ShouldExecuteHandShake() configValues.Add(NextHandShakeTimeConfigKey, DateTime.UtcNow.Add(handShakeInterval).ToString()); await SetConfigState(configValues); return true; - - } DateTime nextHandShakeDateTime; //date cannot be parsed @@ -138,7 +150,15 @@ private async Task SetConfigState(Dictionary state) using (var stream = GenerateStreamFromString(jsonString)) { var blobFile = new BlobFile(stream, ConfigFileName); - var result = await _blobStorageHelper.UploadFileToBlobStorage(_blobConnectionString, _config.NemsMeshConfigContainer, blobFile, true); + var result = false; + if (_blobServiceUri != null) + { + result = await _blobStorageHelper.UploadFileToBlobStorage(_blobServiceUri, _config.NemsMeshConfigContainer, blobFile, true); + } + else if (_blobConnectionString != null) + { + result = await _blobStorageHelper.UploadFileToBlobStorage(_blobConnectionString, _config.NemsMeshConfigContainer, blobFile, true); + } return result; } } diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/NemsMeshRetrievalConfig.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/NemsMeshRetrievalConfig.cs index 6f3ce48d12..4a3f68af46 100644 --- a/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/NemsMeshRetrievalConfig.cs +++ b/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/NemsMeshRetrievalConfig.cs @@ -1,5 +1,6 @@ namespace NHS.Screening.NemsMeshRetrieval; +using Common; using System.ComponentModel.DataAnnotations; public class NemsMeshRetrievalConfig @@ -14,8 +15,8 @@ public class NemsMeshRetrievalConfig public string NemsMeshKeyPassphrase {get; set;} public string NemsMeshKeyName {get; set;} public string KeyVaultConnectionString {get; set;} - [Required] - public required string nemsmeshfolder_STORAGE {get; set;} + public BlobStorageConfig? AzureWebJobsStorage { get; set; } + public BlobStorageConfig? nemsmeshfolder_STORAGE { get; set;} public string NemsMeshInboundContainer { get; set; } = "nems-updates"; public string NemsMeshConfigContainer { get; set; } = "nems-config"; public string NemsMeshServerSideCerts { get; set; } diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/Program.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/Program.cs index 30ba26dc2d..f093729d9d 100644 --- a/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/Program.cs +++ b/application/CohortManager/src/Functions/NemsSubscriptionService/NemsMeshRetrieval/Program.cs @@ -68,7 +68,7 @@ services.AddSingleton(); services.AddTransient(); // Register health checks - services.AddBlobStorageHealthCheck("NemsMeshRetrieval"); + services.AddBlobStorageHealthCheck("NemsMeshRetrieval", config.AzureWebJobsStorage!); }) .AddTelemetry() .AddExceptionHandler(); diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdate.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdate.cs index 85069656d9..7b0a07d41f 100644 --- a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdate.cs +++ b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdate.cs @@ -1,4 +1,4 @@ -namespace NHS.Screening.ProcessNemsUpdate; +namespace NHS.Screening.ProcessNemsUpdate; using System.Collections.Concurrent; using System.Collections.Specialized; @@ -119,7 +119,18 @@ public async Task Run([BlobTrigger("nems-updates/{name}", Connection = "nemsmesh private async Task CopyToPoisonContainer(string fileName) { - await _blobStorageHelper.CopyFileToPoisonAsync(_config.nemsmeshfolder_STORAGE, fileName, _config.NemsMessages, _config.NemsPoisonContainer, addTimestamp: true); + if (_config.nemsmeshfolder_STORAGE != null) + { + await _blobStorageHelper.CopyFileToPoisonAsync(new Uri(_config.nemsmeshfolder_STORAGE.BlobServiceUri), fileName, _config.NemsMessages, _config.NemsPoisonContainer, addTimestamp: true); + } + else + { + var connectionString = Environment.GetEnvironmentVariable("nemsmeshfolder_STORAGE"); + if (connectionString != null) + { + await _blobStorageHelper.CopyFileToPoisonAsync(connectionString, fileName, _config.NemsMessages, _config.NemsPoisonContainer, addTimestamp: true); + } + } _logger.LogInformation("Copied failed NEMS file {FileName} to poison container with timestamp.", fileName); } diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdateConfig.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdateConfig.cs index 4f22b7e388..ae16512182 100644 --- a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdateConfig.cs +++ b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdateConfig.cs @@ -1,5 +1,6 @@ namespace NHS.Screening.ProcessNemsUpdate; +using Common; using System.ComponentModel.DataAnnotations; public class ProcessNemsUpdateConfig @@ -25,7 +26,8 @@ public class ProcessNemsUpdateConfig [Required] public required string DemographicDataServiceURL { get; set; } - [Required] - public required string nemsmeshfolder_STORAGE { get; set; } + public BlobStorageConfig? AzureWebJobsStorage { get; set; } + public BlobStorageConfig? nemsmeshfolder_STORAGE { get; set; } + public string NemsPoisonContainer { get; set; } = "nems-poison"; } diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/Program.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/Program.cs index a274b58371..d31a8bee28 100644 --- a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/Program.cs +++ b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/Program.cs @@ -19,7 +19,7 @@ services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddBlobStorageHealthCheck("ProcessNemsUpdate"); + services.AddBlobStorageHealthCheck("ProcessNemsUpdate", config.AzureWebJobsStorage!); }) .AddTelemetry() .AddExceptionHandler() diff --git a/application/CohortManager/src/Functions/ScreeningValidationService/LookupValidation/LookupValidationConfig.cs b/application/CohortManager/src/Functions/ScreeningValidationService/LookupValidation/LookupValidationConfig.cs index 650b90880a..e3780f3984 100644 --- a/application/CohortManager/src/Functions/ScreeningValidationService/LookupValidation/LookupValidationConfig.cs +++ b/application/CohortManager/src/Functions/ScreeningValidationService/LookupValidation/LookupValidationConfig.cs @@ -1,4 +1,7 @@ namespace NHS.CohortManager.ScreeningValidationService; + +using Common; + public class LookupValidationConfig { public string ExceptionFunctionUrl {get;set;} @@ -7,5 +10,5 @@ public class LookupValidationConfig public string CurrentPostingUrl {get;set;} public string ExcludedSMULookupUrl {get;set;} public string BlobContainerName {get;set;} - public string AzureWebJobsStorage {get;set;} + public BlobStorageConfig? AzureWebJobsStorage {get;set;} } diff --git a/application/CohortManager/src/Functions/ServiceNowIntegration/ServiceNowCohortLookup/Program.cs b/application/CohortManager/src/Functions/ServiceNowIntegration/ServiceNowCohortLookup/Program.cs index e82d12a9b4..f60c4059a7 100644 --- a/application/CohortManager/src/Functions/ServiceNowIntegration/ServiceNowCohortLookup/Program.cs +++ b/application/CohortManager/src/Functions/ServiceNowIntegration/ServiceNowCohortLookup/Program.cs @@ -17,7 +17,7 @@ { services.AddSingleton(); // Register health checks - services.AddBlobStorageHealthCheck("ServiceNowCohortLookup"); + services.AddBlobStorageHealthCheck("ServiceNowCohortLookup", config.AzureWebJobsStorage!); }) .AddTelemetry() .AddExceptionHandler() diff --git a/application/CohortManager/src/Functions/ServiceNowIntegration/ServiceNowCohortLookup/ServiceNowCohortLookupConfig.cs b/application/CohortManager/src/Functions/ServiceNowIntegration/ServiceNowCohortLookup/ServiceNowCohortLookupConfig.cs index 33e88cd1c9..56fcff1a43 100644 --- a/application/CohortManager/src/Functions/ServiceNowIntegration/ServiceNowCohortLookup/ServiceNowCohortLookupConfig.cs +++ b/application/CohortManager/src/Functions/ServiceNowIntegration/ServiceNowCohortLookup/ServiceNowCohortLookupConfig.cs @@ -1,5 +1,6 @@ namespace NHS.CohortManager.ServiceNowIntegrationService; +using Common; using System.ComponentModel.DataAnnotations; public class ServiceNowCohortLookupConfig @@ -9,4 +10,6 @@ public class ServiceNowCohortLookupConfig [Required] public required string CohortDistributionDataServiceURL { get; set; } + + public BlobStorageConfig? AzureWebJobsStorage { get; set; } } diff --git a/application/CohortManager/src/Functions/Shared/Common/AuditClientConfig.cs b/application/CohortManager/src/Functions/Shared/Common/AuditClientConfig.cs index e99ce98926..65e56a1919 100644 --- a/application/CohortManager/src/Functions/Shared/Common/AuditClientConfig.cs +++ b/application/CohortManager/src/Functions/Shared/Common/AuditClientConfig.cs @@ -6,6 +6,5 @@ public class AuditClientConfig { [Required] public required string AuditTopicName { get; set; } - [Required] - public required string AzureWebJobsStorage { get; set; } + public BlobStorageConfig? AzureWebJobsStorage { get; set; } } diff --git a/application/CohortManager/src/Functions/Shared/Common/AuditLogClient.cs b/application/CohortManager/src/Functions/Shared/Common/AuditLogClient.cs index 04367c2984..8387ee46db 100644 --- a/application/CohortManager/src/Functions/Shared/Common/AuditLogClient.cs +++ b/application/CohortManager/src/Functions/Shared/Common/AuditLogClient.cs @@ -77,8 +77,23 @@ public async Task AddBatchAsync(IEnumerable messag var payload = JsonSerializer.SerializeToUtf8Bytes(message.RequestSnapshot, JsonOptions); var blobFile = new BlobFile(payload, blobPath); - var uri = await _blobStorageHelper.UploadFileToBlobStorageAndGetUri( - _config.AzureWebJobsStorage, AuditBlobContainer, blobFile, overwrite: true); + + string? uri = null; + + if (_config.AzureWebJobsStorage != null) + { + uri = await _blobStorageHelper.UploadFileToBlobStorageAndGetUri( + _config.AzureWebJobsStorage.BlobServiceUri, AuditBlobContainer, blobFile, overwrite: true); + } + else + { + var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); + if (connectionString != null) + { + uri = await _blobStorageHelper.UploadFileToBlobStorageAndGetUri( + connectionString, AuditBlobContainer, blobFile, overwrite: true); + } + } if (uri is null) { diff --git a/application/CohortManager/src/Functions/Shared/Common/BlobStorageConfig.cs b/application/CohortManager/src/Functions/Shared/Common/BlobStorageConfig.cs new file mode 100644 index 0000000000..5f5a0db7e5 --- /dev/null +++ b/application/CohortManager/src/Functions/Shared/Common/BlobStorageConfig.cs @@ -0,0 +1,8 @@ +namespace Common; + +public class BlobStorageConfig +{ + public required string AccountName { get; set; } + public required string BlobServiceUri { get; set; } + public required string QueueServiceUri { get; set; } +} diff --git a/application/CohortManager/src/Functions/Shared/Common/BlobstorageHelper.cs b/application/CohortManager/src/Functions/Shared/Common/BlobstorageHelper.cs index b776e8b82c..38a2ea6e4b 100644 --- a/application/CohortManager/src/Functions/Shared/Common/BlobstorageHelper.cs +++ b/application/CohortManager/src/Functions/Shared/Common/BlobstorageHelper.cs @@ -1,6 +1,7 @@ namespace Common; using Azure; +using Azure.Identity; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; @@ -16,20 +17,38 @@ public BlobStorageHelper(ILogger logger) } public async Task CopyFileToPoisonAsync(string connectionString, string fileName, string containerName) { - // Delegate to the extended overload to avoid duplication; preserve env var behaviour var poisonContainerName = Environment.GetEnvironmentVariable("fileExceptions"); - await CopyFileToPoisonAsync(connectionString, fileName, containerName, poisonContainerName, addTimestamp: false); + await CopyFileToPoisonAsync(connectionString, fileName, containerName, poisonContainerName!, addTimestamp: false); + } + + public async Task CopyFileToPoisonAsync(Uri serviceUri, string fileName, string containerName) + { + var poisonContainerName = Environment.GetEnvironmentVariable("fileExceptions"); + await CopyFileToPoisonAsync(serviceUri, fileName, containerName, poisonContainerName!, addTimestamp: false); } public async Task CopyFileToPoisonAsync(string connectionString, string fileName, string containerName, string poisonContainerName, bool addTimestamp = false) { var sourceBlobServiceClient = new BlobServiceClient(connectionString); + var destinationBlobServiceClient = new BlobServiceClient(connectionString); + await CopyFileToPoisonAsync(sourceBlobServiceClient, destinationBlobServiceClient, fileName, containerName, poisonContainerName, addTimestamp: false); + } + + public async Task CopyFileToPoisonAsync(Uri serviceUri, string fileName, string containerName, string poisonContainerName, bool addTimestamp = false) + { + var defaultAzureCredential = new DefaultAzureCredential(); + var sourceBlobServiceClient = new BlobServiceClient(serviceUri, defaultAzureCredential); + var destinationBlobServiceClient = new BlobServiceClient(serviceUri, defaultAzureCredential); + await CopyFileToPoisonAsync(sourceBlobServiceClient, destinationBlobServiceClient, fileName, containerName, poisonContainerName, addTimestamp: false); + } + + private async Task CopyFileToPoisonAsync(BlobServiceClient sourceBlobServiceClient, BlobServiceClient destinationBlobServiceClient, string fileName, string containerName, string poisonContainerName, bool addTimestamp = false) + { var sourceContainerClient = sourceBlobServiceClient.GetBlobContainerClient(containerName); var sourceBlobClient = sourceContainerClient.GetBlobClient(fileName); BlobLeaseClient sourceBlobLease = new(sourceBlobClient); - var destinationBlobServiceClient = new BlobServiceClient(connectionString); var destinationContainerClient = destinationBlobServiceClient.GetBlobContainerClient(poisonContainerName); // Conditionally add timestamp to prevent collisions and maintain audit trail @@ -65,6 +84,17 @@ public async Task CopyFileToPoisonAsync(string connectionString, string fileName public async Task UploadFileToBlobStorage(string connectionString, string containerName, BlobFile blobFile, bool overwrite = false) { var sourceBlobServiceClient = new BlobServiceClient(connectionString); + return await UploadFileToBlobStorage(sourceBlobServiceClient, "failed-batch", blobFile); + } + + public async Task UploadFileToBlobStorage(Uri serviceUri, string containerName, BlobFile blobFile, bool overwrite = false) + { + var sourceBlobServiceClient = new BlobServiceClient(serviceUri, new DefaultAzureCredential()); + return await UploadFileToBlobStorage(sourceBlobServiceClient, "failed-batch", blobFile); + } + + private async Task UploadFileToBlobStorage(BlobServiceClient sourceBlobServiceClient, string containerName, BlobFile blobFile, bool overwrite = false) + { var sourceContainerClient = sourceBlobServiceClient.GetBlobContainerClient(containerName); var sourceBlobClient = sourceContainerClient.GetBlobClient(blobFile.FileName); @@ -83,9 +113,20 @@ public async Task UploadFileToBlobStorage(string connectionString, string return true; } + public async Task UploadFileToBlobStorageAndGetUri(Uri serviceUri, string containerName, BlobFile blobFile, bool overwrite = false) + { + var blobServiceClient = new BlobServiceClient(serviceUri, new DefaultAzureCredential()); + return await UploadFileToBlobStorageAndGetUri(blobServiceClient, containerName, blobFile, overwrite); + } + public async Task UploadFileToBlobStorageAndGetUri(string connectionString, string containerName, BlobFile blobFile, bool overwrite = false) { var blobServiceClient = new BlobServiceClient(connectionString); + return await UploadFileToBlobStorageAndGetUri(blobServiceClient, containerName, blobFile, overwrite); + } + + private async Task UploadFileToBlobStorageAndGetUri(BlobServiceClient blobServiceClient, string containerName, BlobFile blobFile, bool overwrite = false) + { var containerClient = blobServiceClient.GetBlobContainerClient(containerName); var blobClient = containerClient.GetBlobClient(blobFile.FileName); @@ -103,11 +144,22 @@ public async Task UploadFileToBlobStorage(string connectionString, string } public async Task GetFileFromBlobStorage(string connectionString, string containerName, string fileName) + { + var blobServiceClient = new BlobServiceClient(connectionString); + return await GetFileFromBlobStorage(blobServiceClient, containerName, fileName); + } + + public async Task GetFileFromBlobStorage(Uri serviceUri, string containerName, string fileName) + { + var blobServiceClient = new BlobServiceClient(serviceUri, new DefaultAzureCredential()); + return await GetFileFromBlobStorage(blobServiceClient, containerName, fileName); + } + + private async Task GetFileFromBlobStorage(BlobServiceClient blobServiceClient, string containerName, string fileName) { _logger.LogInformation("Downloading File: {FileName} From blobStorage Container: {ContainerName}", fileName, containerName); - var blobServiceClient = new BlobServiceClient(connectionString); var containerClient = blobServiceClient.GetBlobContainerClient(containerName); var blobClient = containerClient.GetBlobClient(fileName); @@ -123,5 +175,4 @@ public async Task GetFileFromBlobStorage(string connectionString, stri return null; } - } diff --git a/application/CohortManager/src/Functions/Shared/Common/Interfaces/IBlobstorageHelper.cs b/application/CohortManager/src/Functions/Shared/Common/Interfaces/IBlobstorageHelper.cs index 0efc63d0bc..2c9393b813 100644 --- a/application/CohortManager/src/Functions/Shared/Common/Interfaces/IBlobstorageHelper.cs +++ b/application/CohortManager/src/Functions/Shared/Common/Interfaces/IBlobstorageHelper.cs @@ -5,8 +5,15 @@ namespace Common; public interface IBlobStorageHelper { Task CopyFileToPoisonAsync(string connectionString, string fileName, string containerName); + Task CopyFileToPoisonAsync(Uri serviceUri, string fileName, string containerName); Task CopyFileToPoisonAsync(string connectionString, string fileName, string containerName, string poisonContainerName, bool addTimestamp = false); - Task UploadFileToBlobStorage(string connectionString, string containerName, BlobFile blobFile, bool overwrite = false); + Task CopyFileToPoisonAsync(Uri serviceUri, string fileName, string containerName, string poisonContainerName, bool addTimestamp = false); + Task UploadFileToBlobStorageAndGetUri(string connectionString, string containerName, BlobFile blobFile, bool overwrite = false); + Task UploadFileToBlobStorageAndGetUri(Uri serviceUri, string containerName, BlobFile blobFile, bool overwrite = false); + Task UploadFileToBlobStorage(string connectionString, string containerName, BlobFile blobFile, bool overwrite = false); + Task UploadFileToBlobStorage(Uri serviceUri, string containerName, BlobFile blobFile, bool overwrite = false); + Task GetFileFromBlobStorage(string connectionString, string containerName, string fileName); + Task GetFileFromBlobStorage(Uri serviceUri, string containerName, string fileName); } diff --git a/application/CohortManager/src/Functions/Shared/Common/Interfaces/IMeshtoBlobTransferHandler.cs b/application/CohortManager/src/Functions/Shared/Common/Interfaces/IMeshtoBlobTransferHandler.cs index 18907db85f..b5945f5fe4 100644 --- a/application/CohortManager/src/Functions/Shared/Common/Interfaces/IMeshtoBlobTransferHandler.cs +++ b/application/CohortManager/src/Functions/Shared/Common/Interfaces/IMeshtoBlobTransferHandler.cs @@ -4,5 +4,5 @@ namespace Common; public interface IMeshToBlobTransferHandler { - Task MoveFilesFromMeshToBlob(Func predicate, Func fileNameFunction, string mailboxId, string blobConnectionString, string destinationContainer, bool executeHandshake = false); + Task MoveFilesFromMeshToBlob(Func predicate, Func fileNameFunction, string mailboxId, Uri? blobServiceUri, string? blobConnectionString, string destinationContainer, bool executeHandshake = false); } diff --git a/application/CohortManager/src/Functions/Shared/Common/MeshToBlobTransferHandler.cs b/application/CohortManager/src/Functions/Shared/Common/MeshToBlobTransferHandler.cs index 2255a80a23..4a495a9131 100644 --- a/application/CohortManager/src/Functions/Shared/Common/MeshToBlobTransferHandler.cs +++ b/application/CohortManager/src/Functions/Shared/Common/MeshToBlobTransferHandler.cs @@ -1,6 +1,5 @@ namespace Common; -using System.Reflection.Metadata; using Microsoft.Extensions.Logging; using Model; using NHS.MESH.Client.Contracts.Services; @@ -16,7 +15,8 @@ public class MeshToBlobTransferHandler : IMeshToBlobTransferHandler private readonly IBlobStorageHelper _blobStorageHelper; private readonly ILogger _logger; - private string _blobConnectionString; + private Uri? _blobServiceUri; + private string? _blobConnectionString; private string _mailboxId; private string _destinationContainer; @@ -28,11 +28,11 @@ public MeshToBlobTransferHandler(ILogger logger, IBlo _meshInboxService = meshInboxService; _blobStorageHelper = blobStorageHelper; _meshOperationService = meshOperationService; - } - public async Task MoveFilesFromMeshToBlob(Func predicate, Func fileNameFunction, string mailboxId, string blobConnectionString, string destinationContainer, bool executeHandshake = false) + public async Task MoveFilesFromMeshToBlob(Func predicate, Func fileNameFunction, string mailboxId, Uri? blobServiceUri, string? blobConnectionString, string destinationContainer, bool executeHandshake = false) { + _blobServiceUri = blobServiceUri; _blobConnectionString = blobConnectionString; _mailboxId = mailboxId; _destinationContainer = destinationContainer; @@ -51,7 +51,6 @@ public async Task MoveFilesFromMeshToBlob(Func predi do { - var checkForMessages = await _meshInboxService.GetMessagesAsync(mailboxId); if(!checkForMessages.IsSuccessful) { @@ -60,7 +59,6 @@ public async Task MoveFilesFromMeshToBlob(Func predi return false; } - messageCount = checkForMessages.Response.Messages.Count(); _logger.LogInformation("{MessageCount} Messages were found within mailbox {MailboxId}",messageCount,mailboxId); @@ -86,7 +84,7 @@ public async Task MoveFilesFromMeshToBlob(Func predi } - private async Task MoveAllMessagesToBlobStorage(IEnumerable messages,Func predicate) + private async Task MoveAllMessagesToBlobStorage(IEnumerable messages, Func predicate) { var messagesMovedToBlobStorage = 0; foreach(var message in messages) @@ -139,8 +137,15 @@ private async Task TransferMessageToBlobStorage(MessageMetaData messageHea { return false; } - - var uploadedToBlob = await _blobStorageHelper.UploadFileToBlobStorage(_blobConnectionString,_destinationContainer,blobFile); + var uploadedToBlob = false; + if (_blobServiceUri != null) + { + uploadedToBlob = await _blobStorageHelper.UploadFileToBlobStorage(_blobServiceUri, _destinationContainer, blobFile); + } + else if (_blobConnectionString != null) + { + uploadedToBlob = await _blobStorageHelper.UploadFileToBlobStorage(_blobConnectionString, _destinationContainer, blobFile); + } return uploadedToBlob; } diff --git a/application/CohortManager/src/Functions/Shared/HealthChecks/Extensions/BlobStorageHealthCheckExtension.cs b/application/CohortManager/src/Functions/Shared/HealthChecks/Extensions/BlobStorageHealthCheckExtension.cs index df7bb76cf1..774e2bf4a3 100644 --- a/application/CohortManager/src/Functions/Shared/HealthChecks/Extensions/BlobStorageHealthCheckExtension.cs +++ b/application/CohortManager/src/Functions/Shared/HealthChecks/Extensions/BlobStorageHealthCheckExtension.cs @@ -1,11 +1,13 @@ namespace HealthChecks.Extensions; +using Azure.Identity; using Azure.Storage.Blobs; +using Common; using Microsoft.Extensions.DependencyInjection; public static class BlobStorageHealthCheckExtension { - public static IServiceCollection AddBlobStorageHealthCheck(this IServiceCollection services, string name) + public static IServiceCollection AddBlobStorageHealthCheck(this IServiceCollection services, string name, BlobStorageConfig azureWebJobsStorage) { // Register blob storage health checks services.AddHealthChecks() @@ -15,8 +17,12 @@ public static IServiceCollection AddBlobStorageHealthCheck(this IServiceCollecti // Register BlobServiceClient service for health check services.AddSingleton(provider => { - var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); - return new BlobServiceClient(connectionString); + if (azureWebJobsStorage == null) + { + var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); + return new BlobServiceClient(connectionString); + } + return new BlobServiceClient(new Uri(azureWebJobsStorage.BlobServiceUri), new DefaultAzureCredential()); }); return services; diff --git a/infrastructure/tf-audit/environments/sandbox.tfvars b/infrastructure/tf-audit/environments/sandbox.tfvars index ed665d2e78..52f4e436ec 100644 --- a/infrastructure/tf-audit/environments/sandbox.tfvars +++ b/infrastructure/tf-audit/environments/sandbox.tfvars @@ -1,6 +1,6 @@ application = "cohman" application_full_name = "cohort-manager" -environment = "SBRK" +environment = "SB" tags = { Environment = "sandbox" diff --git a/infrastructure/tf-core/environments/development.tfvars b/infrastructure/tf-core/environments/development.tfvars index 3ae6c6fcd1..8d252cb0cf 100644 --- a/infrastructure/tf-core/environments/development.tfvars +++ b/infrastructure/tf-core/environments/development.tfvars @@ -298,7 +298,7 @@ function_apps = { https_only = true http2_enabled = true remote_debugging_enabled = false - storage_uses_managed_identity = null + storage_uses_managed_identity = true worker_32bit = false health_check_path = "/api/health" @@ -1390,6 +1390,7 @@ storage_accounts = { file_exceptions = { name_suffix = "filexptns" account_tier = "Standard" + shared_access_key_enabled = true replication_type = "LRS" public_network_access_enabled = false blob_properties_delete_retention_policy = 7 diff --git a/infrastructure/tf-core/environments/integration.tfvars b/infrastructure/tf-core/environments/integration.tfvars index bb051878f6..66f680fdd1 100644 --- a/infrastructure/tf-core/environments/integration.tfvars +++ b/infrastructure/tf-core/environments/integration.tfvars @@ -288,25 +288,25 @@ function_apps = { docker_CI_enable = "true" docker_img_prefix = "cohort-manager" - enable_appsrv_storage = "false" - ftps_state = "Disabled" - https_only = true - http2_enabled = true - remote_debugging_enabled = false - storage_uses_managed_identity = null - worker_32bit = false - health_check_path = "/api/health" + enable_appsrv_storage = "false" + ftps_state = "Disabled" + https_only = true + http2_enabled = true + remote_debugging_enabled = false + worker_32bit = false + health_check_path = "/api/health" fa_config = { ReceiveCaasFile = { - name_suffix = "receive-caas-file" - function_endpoint_name = "ReceiveCaasFile" - app_service_plan_key = "NonScaling" - producer_to_service_bus = ["dtoss-nsp"] - db_connection_string = "DtOsDatabaseConnectionString" - service_bus_connections = ["internal"] - storage_account_env_var_name = "caasfolder_STORAGE" + name_suffix = "receive-caas-file" + function_endpoint_name = "ReceiveCaasFile" + app_service_plan_key = "NonScaling" + producer_to_service_bus = ["dtoss-nsp"] + db_connection_string = "DtOsDatabaseConnectionString" + service_bus_connections = ["internal"] + storage_account_env_var_name = "caasfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -360,11 +360,12 @@ function_apps = { } RetrieveMeshFile = { - name_suffix = "retrieve-mesh-file" - function_endpoint_name = "RetrieveMeshFile" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "caasfolder_STORAGE" + name_suffix = "retrieve-mesh-file" + function_endpoint_name = "RetrieveMeshFile" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "caasfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -378,12 +379,13 @@ function_apps = { } ProcessNemsUpdate = { - name_suffix = "process-nems-update" - function_endpoint_name = "ProcessNemsUpdate" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "nemsmeshfolder_STORAGE" - service_bus_connections = ["internal"] + name_suffix = "process-nems-update" + function_endpoint_name = "ProcessNemsUpdate" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "nemsmeshfolder_STORAGE" + storage_uses_managed_identity = true + service_bus_connections = ["internal"] app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -672,7 +674,7 @@ function_apps = { } ] env_vars_static = { - AcceptableLatencyThresholdMs = "500" + AcceptableLatencyThresholdMs = "500" RetrieveSupersededRecordsLast = "true" } } @@ -869,7 +871,7 @@ function_apps = { ] env_vars_static = { AcceptableLatencyThresholdMs = "500" - MaxRetryCount=3 + MaxRetryCount = 3 } } @@ -1155,11 +1157,12 @@ function_apps = { } NemsMeshRetrieval = { - name_suffix = "nems-mesh-retrieval" - function_endpoint_name = "NemsMeshRetrieval" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "nemsmeshfolder_STORAGE" + name_suffix = "nems-mesh-retrieval" + function_endpoint_name = "NemsMeshRetrieval" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "nemsmeshfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" diff --git a/infrastructure/tf-core/environments/nft.tfvars b/infrastructure/tf-core/environments/nft.tfvars index b34887810e..7f61ee1aa7 100644 --- a/infrastructure/tf-core/environments/nft.tfvars +++ b/infrastructure/tf-core/environments/nft.tfvars @@ -288,24 +288,24 @@ function_apps = { docker_CI_enable = "true" docker_img_prefix = "cohort-manager" - enable_appsrv_storage = "false" - ftps_state = "Disabled" - https_only = true - http2_enabled = true - remote_debugging_enabled = false - storage_uses_managed_identity = null - worker_32bit = false - health_check_path = "/api/health" + enable_appsrv_storage = "false" + ftps_state = "Disabled" + https_only = true + http2_enabled = true + remote_debugging_enabled = false + worker_32bit = false + health_check_path = "/api/health" fa_config = { ReceiveCaasFile = { - name_suffix = "receive-caas-file" - function_endpoint_name = "ReceiveCaasFile" - app_service_plan_key = "NonScaling" - db_connection_string = "DtOsDatabaseConnectionString" - service_bus_connections = ["internal"] - storage_account_env_var_name = "caasfolder_STORAGE" + name_suffix = "receive-caas-file" + function_endpoint_name = "ReceiveCaasFile" + app_service_plan_key = "NonScaling" + db_connection_string = "DtOsDatabaseConnectionString" + service_bus_connections = ["internal"] + storage_account_env_var_name = "caasfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -358,11 +358,12 @@ function_apps = { } RetrieveMeshFile = { - name_suffix = "retrieve-mesh-file" - function_endpoint_name = "RetrieveMeshFile" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "caasfolder_STORAGE" + name_suffix = "retrieve-mesh-file" + function_endpoint_name = "RetrieveMeshFile" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "caasfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -376,12 +377,13 @@ function_apps = { } ProcessNemsUpdate = { - name_suffix = "process-nems-update" - function_endpoint_name = "ProcessNemsUpdate" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "nemsmeshfolder_STORAGE" - service_bus_connections = ["internal"] + name_suffix = "process-nems-update" + function_endpoint_name = "ProcessNemsUpdate" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "nemsmeshfolder_STORAGE" + storage_uses_managed_identity = true + service_bus_connections = ["internal"] app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -391,7 +393,7 @@ function_apps = { env_var_name = "RetrievePdsDemographicURL" function_app_key = "RetrievePDSDemographic" }, - { + { env_var_name = "ManageNemsSubscriptionUnsubscribeURL" function_app_key = "ManageNemsSubscription" endpoint_name = "Unsubscribe" @@ -669,7 +671,7 @@ function_apps = { } ] env_vars_static = { - AcceptableLatencyThresholdMs = "500" + AcceptableLatencyThresholdMs = "500" RetrieveSupersededRecordsLast = "true" } } @@ -866,7 +868,7 @@ function_apps = { ] env_vars_static = { AcceptableLatencyThresholdMs = "500" - MaxRetryCount=3 + MaxRetryCount = 3 } } @@ -1152,11 +1154,12 @@ function_apps = { } NemsMeshRetrieval = { - name_suffix = "nems-mesh-retrieval" - function_endpoint_name = "NemsMeshRetrieval" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "nemsmeshfolder_STORAGE" + name_suffix = "nems-mesh-retrieval" + function_endpoint_name = "NemsMeshRetrieval" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "nemsmeshfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" diff --git a/infrastructure/tf-core/environments/preprod.tfvars b/infrastructure/tf-core/environments/preprod.tfvars index 8e8509d600..0b2ce6e68f 100644 --- a/infrastructure/tf-core/environments/preprod.tfvars +++ b/infrastructure/tf-core/environments/preprod.tfvars @@ -59,7 +59,7 @@ regions = { service_delegation_name = "Microsoft.App/environments" service_delegation_actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] } - power-bi= { + power-bi = { cidr_newbits = 8 cidr_offset = 6 delegation_name = "Microsoft.PowerPlatform/vnetaccesslinks" @@ -298,25 +298,25 @@ function_apps = { docker_CI_enable = "true" docker_img_prefix = "cohort-manager" - enable_appsrv_storage = "false" - ftps_state = "Disabled" - https_only = true - http2_enabled = true - remote_debugging_enabled = false - storage_uses_managed_identity = null - worker_32bit = false - health_check_path = "/api/health" + enable_appsrv_storage = "false" + ftps_state = "Disabled" + https_only = true + http2_enabled = true + remote_debugging_enabled = false + worker_32bit = false + health_check_path = "/api/health" fa_config = { ReceiveCaasFile = { - name_suffix = "receive-caas-file" - function_endpoint_name = "ReceiveCaasFile" - app_service_plan_key = "NonScaling" - producer_to_service_bus = ["dtoss-nsp"] - db_connection_string = "DtOsDatabaseConnectionString" - service_bus_connections = ["internal"] - storage_account_env_var_name = "caasfolder_STORAGE" + name_suffix = "receive-caas-file" + function_endpoint_name = "ReceiveCaasFile" + app_service_plan_key = "NonScaling" + producer_to_service_bus = ["dtoss-nsp"] + db_connection_string = "DtOsDatabaseConnectionString" + service_bus_connections = ["internal"] + storage_account_env_var_name = "caasfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -370,11 +370,12 @@ function_apps = { } RetrieveMeshFile = { - name_suffix = "retrieve-mesh-file" - function_endpoint_name = "RetrieveMeshFile" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "caasfolder_STORAGE" + name_suffix = "retrieve-mesh-file" + function_endpoint_name = "RetrieveMeshFile" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "caasfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -387,13 +388,14 @@ function_apps = { } } - ProcessNemsUpdate = { - name_suffix = "process-nems-update" - function_endpoint_name = "ProcessNemsUpdate" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "nemsmeshfolder_STORAGE" - service_bus_connections = ["internal"] + ProcessNemsUpdate = { + name_suffix = "process-nems-update" + function_endpoint_name = "ProcessNemsUpdate" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "nemsmeshfolder_STORAGE" + storage_uses_managed_identity = true + service_bus_connections = ["internal"] app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -403,7 +405,7 @@ function_apps = { env_var_name = "RetrievePdsDemographicURL" function_app_key = "RetrievePDSDemographic" }, - { + { env_var_name = "ManageNemsSubscriptionUnsubscribeURL" function_app_key = "ManageNemsSubscription" endpoint_name = "Unsubscribe" @@ -682,7 +684,7 @@ function_apps = { } ] env_vars_static = { - AcceptableLatencyThresholdMs = "500" + AcceptableLatencyThresholdMs = "500" RetrieveSupersededRecordsLast = "true" } } @@ -879,7 +881,7 @@ function_apps = { ] env_vars_static = { AcceptableLatencyThresholdMs = "500" - MaxRetryCount=3 + MaxRetryCount = 3 } } @@ -1165,11 +1167,12 @@ function_apps = { } NemsMeshRetrieval = { - name_suffix = "nems-mesh-retrieval" - function_endpoint_name = "NemsMeshRetrieval" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "nemsmeshfolder_STORAGE" + name_suffix = "nems-mesh-retrieval" + function_endpoint_name = "NemsMeshRetrieval" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "nemsmeshfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" diff --git a/infrastructure/tf-core/environments/production.tfvars b/infrastructure/tf-core/environments/production.tfvars index 366a1f8b4b..2a98c7ef93 100644 --- a/infrastructure/tf-core/environments/production.tfvars +++ b/infrastructure/tf-core/environments/production.tfvars @@ -324,7 +324,6 @@ function_apps = { https_only = true http2_enabled = true remote_debugging_enabled = false - storage_uses_managed_identity = null worker_32bit = false health_check_path = "/api/health" @@ -338,6 +337,7 @@ function_apps = { db_connection_string = "DtOsDatabaseConnectionString" service_bus_connections = ["internal"] storage_account_env_var_name = "caasfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -396,6 +396,7 @@ function_apps = { app_service_plan_key = "NonScaling" key_vault_url = "KeyVaultConnectionString" storage_account_env_var_name = "caasfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -414,6 +415,7 @@ function_apps = { app_service_plan_key = "NonScaling" key_vault_url = "KeyVaultConnectionString" storage_account_env_var_name = "nemsmeshfolder_STORAGE" + storage_uses_managed_identity = true service_bus_connections = ["internal"] app_urls = [ { @@ -1191,6 +1193,7 @@ function_apps = { app_service_plan_key = "NonScaling" key_vault_url = "KeyVaultConnectionString" storage_account_env_var_name = "nemsmeshfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" diff --git a/infrastructure/tf-core/environments/sandbox.tfvars b/infrastructure/tf-core/environments/sandbox.tfvars index 764500d932..dd4cdeb65c 100644 --- a/infrastructure/tf-core/environments/sandbox.tfvars +++ b/infrastructure/tf-core/environments/sandbox.tfvars @@ -1,6 +1,6 @@ application = "cohman" application_full_name = "cohort-manager" -environment = "SBRK" +environment = "SB" features = { acr_enabled = false @@ -10,8 +10,8 @@ features = { private_service_connection_is_manual = false public_network_access_enabled = false frontdoor_endpoint_enabled = false - alerts_enabled = true - alerts_function_errors_enabled = true + alerts_enabled = false + alerts_function_errors_enabled = false } # these will be merged with compliance tags in locals.tf @@ -292,25 +292,25 @@ function_apps = { docker_CI_enable = "true" docker_img_prefix = "cohort-manager" - enable_appsrv_storage = "false" - ftps_state = "Disabled" - https_only = true - http2_enabled = true - remote_debugging_enabled = false - storage_uses_managed_identity = null - worker_32bit = false - health_check_path = "/api/health" + enable_appsrv_storage = "false" + ftps_state = "Disabled" + https_only = true + http2_enabled = true + remote_debugging_enabled = false + worker_32bit = false + health_check_path = "/api/health" fa_config = { ReceiveCaasFile = { - name_suffix = "receive-caas-file" - function_endpoint_name = "ReceiveCaasFile" - app_service_plan_key = "NonScaling" - producer_to_service_bus = ["dtoss-nsp"] - db_connection_string = "DtOsDatabaseConnectionString" - service_bus_connections = ["internal"] - storage_account_env_var_name = "caasfolder_STORAGE" + name_suffix = "receive-caas-file" + function_endpoint_name = "ReceiveCaasFile" + app_service_plan_key = "NonScaling" + producer_to_service_bus = ["dtoss-nsp"] + db_connection_string = "DtOsDatabaseConnectionString" + service_bus_connections = ["internal"] + storage_account_env_var_name = "caasfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -363,11 +363,12 @@ function_apps = { } RetrieveMeshFile = { - name_suffix = "retrieve-mesh-file" - function_endpoint_name = "RetrieveMeshFile" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "caasfolder_STORAGE" + name_suffix = "retrieve-mesh-file" + function_endpoint_name = "RetrieveMeshFile" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "caasfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -381,12 +382,13 @@ function_apps = { } ProcessNemsUpdate = { - name_suffix = "process-nems-update" - function_endpoint_name = "ProcessNemsUpdate" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "nemsmeshfolder_STORAGE" - service_bus_connections = ["internal"] + name_suffix = "process-nems-update" + function_endpoint_name = "ProcessNemsUpdate" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "nemsmeshfolder_STORAGE" + storage_uses_managed_identity = true + service_bus_connections = ["internal"] app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -1156,11 +1158,12 @@ function_apps = { } NemsMeshRetrieval = { - name_suffix = "nems-mesh-retrieval" - function_endpoint_name = "NemsMeshRetrieval" - app_service_plan_key = "NonScaling" - key_vault_url = "KeyVaultConnectionString" - storage_account_env_var_name = "nemsmeshfolder_STORAGE" + name_suffix = "nems-mesh-retrieval" + function_endpoint_name = "NemsMeshRetrieval" + app_service_plan_key = "NonScaling" + key_vault_url = "KeyVaultConnectionString" + storage_account_env_var_name = "nemsmeshfolder_STORAGE" + storage_uses_managed_identity = true app_urls = [ { env_var_name = "ExceptionFunctionURL" @@ -1255,14 +1258,14 @@ frontdoor_endpoint = { } # custom_domains = { # cohort-dev = { - # host_name = "cohort-sbrk.non-live.screening.nhs.uk" + # host_name = "cohort-sb.non-live.screening.nhs.uk" # dns_zone_name = "non-live.screening.nhs.uk" # dns_zone_rg_name = "rg-hub-dev-uks-public-dns-zones" # } # } # security_policies = { # AllowedIPs = { - # cdn_frontdoor_firewall_policy_name = "wafhubnonlivecohmansbrk" + # cdn_frontdoor_firewall_policy_name = "wafhubnonlivecohmansb" # cdn_frontdoor_firewall_policy_rg_name = "rg-hub-dev-uks-hub-networking" # associated_domain_keys = ["cohort-dev"] # From custom_domains above. Use "endpoint" for the default domain (if linked in Front Door route). # } @@ -1304,7 +1307,7 @@ service_bus = { } sqlserver = { - sql_admin_group_name = "sqlsvr_cohman_sbrk_uks_admin" + sql_admin_group_name = "sqlsvr_cohman_sb_uks_admin" ad_auth_only = true auditing_policy_retention_in_days = 30 security_alert_policy_retention_days = 30 diff --git a/infrastructure/tf-core/function_app.tf b/infrastructure/tf-core/function_app.tf index 38c9c5b8b7..a5fb81914f 100644 --- a/infrastructure/tf-core/function_app.tf +++ b/infrastructure/tf-core/function_app.tf @@ -28,7 +28,6 @@ module "functionapp" { # Use the storage account assigned identity for the Function Apps: storage_account_name = module.storage["fnapp-${each.value.region}"].storage_account_name - storage_account_access_key = var.function_apps.storage_uses_managed_identity == true ? null : module.storage["fnapp-${each.value.region}"].storage_account_primary_access_key storage_uses_managed_identity = var.function_apps.storage_uses_managed_identity # Connection string for Application Insights: @@ -154,15 +153,16 @@ locals { } : {}, # Storage - The C# code should be updated to use System Managed Identity, rather than connection string - length(config.storage_account_env_var_name) > 0 ? merge( - { - (config.storage_account_env_var_name) = module.storage["file_exceptions-${region}"].storage_account_primary_connection_string - }, - var.features.private_endpoints_enabled ? { - "${config.storage_account_env_var_name}__blobServiceUri" = "https://${module.storage["file_exceptions-${region}"].storage_account_name}.blob.core.windows.net" - "${config.storage_account_env_var_name}__queueServiceUri" = "https://${module.storage["file_exceptions-${region}"].storage_account_name}.queue.core.windows.net" - } : {} - ) : {}, + { + "AzureWebJobsStorage__accountName" = module.storage["fnapp-${region}"].storage_account_name + "AzureWebJobsStorage__blobServiceUri" = "https://${module.storage["fnapp-${region}"].storage_account_name}.blob.core.windows.net" + "AzureWebJobsStorage__queueServiceUri" = "https://${module.storage["fnapp-${region}"].storage_account_name}.queue.core.windows.net" + }, + length(config.storage_account_env_var_name) > 0 && var.features.private_endpoints_enabled ? { + "${config.storage_account_env_var_name}__accountName" = module.storage["file_exceptions-${region}"].storage_account_name + "${config.storage_account_env_var_name}__blobServiceUri" = "https://${module.storage["file_exceptions-${region}"].storage_account_name}.blob.core.windows.net" + "${config.storage_account_env_var_name}__queueServiceUri" = "https://${module.storage["file_exceptions-${region}"].storage_account_name}.queue.core.windows.net" + } : {}, length(config.storage_containers) > 0 ? { for k, v in config.storage_containers : diff --git a/infrastructure/tf-core/providers.tf b/infrastructure/tf-core/providers.tf index 0728c69e6b..affc75f3a0 100644 --- a/infrastructure/tf-core/providers.tf +++ b/infrastructure/tf-core/providers.tf @@ -15,7 +15,8 @@ terraform { } provider "azurerm" { - subscription_id = var.TARGET_SUBSCRIPTION_ID + subscription_id = var.TARGET_SUBSCRIPTION_ID + storage_use_azuread = true features {} } diff --git a/infrastructure/tf-core/storage.tf b/infrastructure/tf-core/storage.tf index 9df3060980..7ca8c80d37 100644 --- a/infrastructure/tf-core/storage.tf +++ b/infrastructure/tf-core/storage.tf @@ -14,9 +14,10 @@ module "storage" { monitor_diagnostic_setting_storage_account_metrics = local.monitor_diagnostic_setting_storage_account_metrics monitor_diagnostic_setting_storage_account_resource_metrics = local.monitor_diagnostic_setting_storage_account_resource_metrics - account_replication_type = each.value.replication_type - account_tier = each.value.account_tier - access_tier = title(lower(each.value.access_tier)) + account_replication_type = each.value.replication_type + account_tier = each.value.account_tier + access_tier = title(lower(each.value.access_tier)) + shared_access_key_enabled = each.value.shared_access_key_enabled blob_properties_delete_retention_policy = each.value.blob_properties_delete_retention_policy blob_properties_versioning_enabled = each.value.blob_properties_versioning_enabled @@ -53,6 +54,7 @@ locals { name_suffix = storage_val.name_suffix replication_type = storage_val.replication_type account_tier = storage_val.account_tier + shared_access_key_enabled = storage_val.shared_access_key_enabled public_network_access_enabled = storage_val.public_network_access_enabled access_tier = storage_val.access_tier blob_properties_delete_retention_policy = storage_val.blob_properties_delete_retention_policy diff --git a/infrastructure/tf-core/variables.tf b/infrastructure/tf-core/variables.tf index b476fc2637..a9ca55cb41 100644 --- a/infrastructure/tf-core/variables.tf +++ b/infrastructure/tf-core/variables.tf @@ -650,6 +650,7 @@ variable "storage_accounts" { replication_type = optional(string, "LRS") public_network_access_enabled = optional(bool, false) access_tier = optional(string, "Hot") + shared_access_key_enabled = optional(bool, false) containers = optional(map(object({ container_name = string container_access_type = optional(string, "private") diff --git a/tests/UnitTests/CaasIntegrationTests/processCaasFileTest/processCaasFileTests.cs b/tests/UnitTests/CaasIntegrationTests/processCaasFileTest/processCaasFileTests.cs index 0bbebfd41b..2d9d1a177a 100644 --- a/tests/UnitTests/CaasIntegrationTests/processCaasFileTest/processCaasFileTests.cs +++ b/tests/UnitTests/CaasIntegrationTests/processCaasFileTest/processCaasFileTests.cs @@ -1,8 +1,5 @@ namespace NHS.CohortManager.Tests.CaasIntegrationTests; -using System.Collections.Concurrent; -using System.Linq.Expressions; -using System.Reflection; using Common; using Common.Interfaces; using Microsoft.Extensions.Logging; @@ -10,6 +7,9 @@ namespace NHS.CohortManager.Tests.CaasIntegrationTests; using Model; using Moq; using NHS.Screening.ReceiveCaasFile; +using System.Collections.Concurrent; +using System.Linq.Expressions; +using System.Reflection; [TestClass] public class ProcessCaasFileTests @@ -59,11 +59,22 @@ private ReceiveCaasFileConfig GetDefaultConfig(bool allowDeleteRecords) DemographicDataServiceURL = "DemographicDataService", ScreeningLkpDataServiceURL = "ScreeningLkpDataServiceURL", GetOrchestrationStatusURL = "GetOrchestrationStatusURL", - caasfolder_STORAGE = "caasfolder_STORAGE", inboundBlobName = "inbound", ServiceBusConnectionString_client_internal = "ServiceBusConnectionString_client_internal", maxNumberOfChecks = 50, - BatchSize = 50 + BatchSize = 50, + AzureWebJobsStorage = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, + caasfolder_STORAGE = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + } }; } diff --git a/tests/UnitTests/CaasIntegrationTests/receiveCaasFileTest/ReceiveCaasFileTests.cs b/tests/UnitTests/CaasIntegrationTests/receiveCaasFileTest/ReceiveCaasFileTests.cs index 1b3da30e0d..7147be1b63 100644 --- a/tests/UnitTests/CaasIntegrationTests/receiveCaasFileTest/ReceiveCaasFileTests.cs +++ b/tests/UnitTests/CaasIntegrationTests/receiveCaasFileTest/ReceiveCaasFileTests.cs @@ -1,20 +1,20 @@ namespace NHS.CohortManager.Tests.CaasIntegrationTests; -using System.IO; -using System.Net; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Moq; -using Microsoft.VisualStudio.TestTools.UnitTesting; using Common; using Common.Interfaces; -using NHS.Screening.ReceiveCaasFile; -using Model; using DataServices.Client; -using NHS.CohortManager.Tests.TestUtils; -using System.Linq.Expressions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Model; using Model.Enums; +using Moq; +using NHS.CohortManager.Tests.TestUtils; +using NHS.Screening.ReceiveCaasFile; +using System.IO; +using System.Linq.Expressions; +using System.Net; +using System.Threading.Tasks; [TestClass] public class ReceiveCaasFileTests @@ -39,17 +39,28 @@ public ReceiveCaasFileTests() var testConfig = new ReceiveCaasFileConfig { - DemographicDataServiceURL = "DemographicDataServiceURL", + DemographicDataServiceURL = "DemographicDataServiceURL1", ScreeningLkpDataServiceURL = "ScreeningLkpDataServiceURL", DemographicURI = "DemographicURI", BatchSize = 2000, AllowDeleteRecords = true, - caasfolder_STORAGE = "caasfolder_STORAGE", ServiceBusConnectionString_client_internal = "ServiceBusConnectionString_client_internal", GetOrchestrationStatusURL = "GetOrchestrationStatusURL", inboundBlobName = "inbound", ParticipantManagementTopic = "ParticipantManagementTopic", maxNumberOfChecks = 50, + AzureWebJobsStorage = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, + caasfolder_STORAGE = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, }; _config.Setup(c => c.Value).Returns(testConfig); @@ -223,7 +234,7 @@ public async Task Run_FileNameInvalid_CreateExceptionAndCopyFileToPoison(string _blobStorageHelperMock .Verify(x => x.CopyFileToPoisonAsync( - It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny() )); @@ -267,7 +278,7 @@ public async Task Run_CannotGetScreeningId_CreateExceptionAndCopyFileToPoison(In _blobStorageHelperMock .Verify(x => x.CopyFileToPoisonAsync( - It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny() )); diff --git a/tests/UnitTests/CaasIntegrationTests/retrieveMeshFileTest/RetrieveMeshFileTest.cs b/tests/UnitTests/CaasIntegrationTests/retrieveMeshFileTest/RetrieveMeshFileTest.cs index 2e9ced1f7e..1fba905c5b 100644 --- a/tests/UnitTests/CaasIntegrationTests/retrieveMeshFileTest/RetrieveMeshFileTest.cs +++ b/tests/UnitTests/CaasIntegrationTests/retrieveMeshFileTest/RetrieveMeshFileTest.cs @@ -1,17 +1,18 @@ namespace NHS.CohortManager.Tests.CaasIntegrationTests; -using System.Threading.Tasks; + +using Common; using Microsoft.Extensions.Logging; -using Moq; +using Microsoft.Extensions.Options; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Common; -using NHS.Screening.RetrieveMeshFile; +using Model; +using Moq; using NHS.MESH.Client.Contracts.Services; using NHS.MESH.Client.Models; -using Model; -using Microsoft.Extensions.Options; -using System.Text.Json; -using System.Text; +using NHS.Screening.RetrieveMeshFile; using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; [TestClass] public class RetrieveMeshFileTest @@ -26,7 +27,7 @@ public class RetrieveMeshFileTest private const string mailboxId = "TestMailBox"; private const string NextHandShakeTimeConfigKey = "NextHandShakeTime"; private const string ConfigFileName = "MeshState.json"; - private const string BlobStorageConnectionString = "BlobStorage_ConnectionString"; + private const string BlobStorageServiceUri = "https://localhost:8888"; private const string MessageId = "MessageId"; private const string FileName = "testFile.csv"; private const string ContentType = "application/octet-stream"; @@ -38,7 +39,18 @@ public RetrieveMeshFileTest() var testConfig = new RetrieveMeshFileConfig { BSSMailBox = mailboxId, - caasfolder_STORAGE = BlobStorageConnectionString, + AzureWebJobsStorage = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, + nemsmeshfolder_STORAGE = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, MeshPassword = "MeshPassword", MeshSharedKey = "MeshSharedKey", MeshKeyPassphrase = "MeshKeyPassphrase", @@ -66,7 +78,7 @@ public async Task Run_DownloadSingleFileFromMesh_Success() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(BlobStorageConnectionString, ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri(BlobStorageServiceUri), ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -77,7 +89,7 @@ public async Task Run_DownloadSingleFileFromMesh_Success() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Once); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); } @@ -97,7 +109,7 @@ public async Task Run_DownloadSingleChunkedFileFromMesh_Success() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetChunkedMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(BlobStorageConnectionString, ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri(BlobStorageServiceUri), ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -108,7 +120,7 @@ public async Task Run_DownloadSingleChunkedFileFromMesh_Success() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetChunkedMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Once); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); } @@ -136,7 +148,7 @@ public async Task Run_DownloadTwoSingleFilesFromMesh_Success() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(headResponses.Dequeue()); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(messageResponses.Dequeue()); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(BlobStorageConnectionString, ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri(BlobStorageServiceUri), ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -147,7 +159,7 @@ public async Task Run_DownloadTwoSingleFilesFromMesh_Success() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Exactly(2)); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Exactly(2)); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); } @@ -170,7 +182,7 @@ public async Task Run_NoFilesAvailableInMesh_NoAttemptsToDownload() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.GetChunkedMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Never); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -188,7 +200,7 @@ public async Task Run_SingleFileAvailableCannotUploadToBlob_DoesntMarkAsAcknowle _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(BlobStorageConnectionString, ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(false); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri(BlobStorageServiceUri), ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(false); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -199,7 +211,7 @@ public async Task Run_SingleFileAvailableCannotUploadToBlob_DoesntMarkAsAcknowle _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Once); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -225,7 +237,7 @@ public async Task Run_SingleFileFailsToGetMessageHead_DoesNotAttemptToDownload() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(BlobStorageConnectionString, ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(false); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri(BlobStorageServiceUri), ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(false); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -236,7 +248,7 @@ public async Task Run_SingleFileFailsToGetMessageHead_DoesNotAttemptToDownload() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Never); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -261,7 +273,7 @@ public async Task Run_DownloadSingleFile_FailToGetMessage() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(BlobStorageConnectionString, ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri(BlobStorageServiceUri), ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -272,7 +284,7 @@ public async Task Run_DownloadSingleFile_FailToGetMessage() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Never); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -294,7 +306,7 @@ public async Task Run_DownloadMessages_FailedToGetMessages() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(It.IsAny>); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(It.IsAny>); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(BlobStorageConnectionString, ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri(BlobStorageServiceUri), ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(It.IsAny>); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -305,7 +317,7 @@ public async Task Run_DownloadMessages_FailedToGetMessages() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Never); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -331,7 +343,7 @@ public async Task Run_DownloadSingleChunkFile_FailToGetMessage() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetChunkedMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(BlobStorageConnectionString, ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri(BlobStorageServiceUri), ContainerName, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, MessageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -342,7 +354,7 @@ public async Task Run_DownloadSingleChunkFile_FailToGetMessage() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetChunkedMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Never); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), ContainerName, It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -405,7 +417,7 @@ public async Task ShouldExecuteHandshake_NextHandshakeTime_InPast() }; string json = JsonSerializer.Serialize(configValues); byte[] bytes = Encoding.UTF8.GetBytes(json); - _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), "config", ConfigFileName)).ReturnsAsync(new BlobFile(bytes, ConfigFileName)); + _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), "config", ConfigFileName)).ReturnsAsync(new BlobFile(bytes, ConfigFileName)); // act await _retrieveMeshFile.RunAsync(new Microsoft.Azure.Functions.Worker.TimerInfo()); @@ -418,7 +430,7 @@ public async Task ShouldExecuteHandshake_NextHandshakeTime_InPast() public async Task ShouldExecuteHandshake_NoMeshStateFile_ReadError() { // arrange - _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), "config", ConfigFileName)).ReturnsAsync((BlobFile)null); + _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), "config", ConfigFileName)).ReturnsAsync((BlobFile)null); // act await _retrieveMeshFile.RunAsync(new Microsoft.Azure.Functions.Worker.TimerInfo()); @@ -437,7 +449,7 @@ public async Task ShouldExecuteHandshake_NextHandshakeTime_ParsingError() }; string json = JsonSerializer.Serialize(configValues); byte[] bytes = Encoding.UTF8.GetBytes(json); - _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), "config", ConfigFileName)).ReturnsAsync(new BlobFile(bytes, ConfigFileName)); + _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), "config", ConfigFileName)).ReturnsAsync(new BlobFile(bytes, ConfigFileName)); // act await _retrieveMeshFile.RunAsync(new Microsoft.Azure.Functions.Worker.TimerInfo()); diff --git a/tests/UnitTests/CallDurableDemographicFuncTests/CallDurableDemograpghicFunc.cs b/tests/UnitTests/CallDurableDemographicFuncTests/CallDurableDemograpghicFunc.cs index de2e629f82..10fe22fc83 100644 --- a/tests/UnitTests/CallDurableDemographicFuncTests/CallDurableDemograpghicFunc.cs +++ b/tests/UnitTests/CallDurableDemographicFuncTests/CallDurableDemograpghicFunc.cs @@ -1,17 +1,15 @@ namespace NHS.CohortManager.Tests.UnitTests.CheckDemographicTests; using Common; -using NHS.CohortManager.Tests.TestUtils; -using System.Net; using Microsoft.Extensions.Logging; -using Moq; +using Microsoft.Extensions.Options; using Model; -using System.Text.Json; -using System.Threading.Tasks; +using Moq; using Moq.Protected; using NHS.Screening.ReceiveCaasFile; -using Microsoft.Extensions.Options; -using System.Net.Http.Headers; +using System.Net; +using System.Text.Json; +using System.Threading.Tasks; [TestClass] public class CallDurableDemographicFuncTests @@ -34,12 +32,23 @@ public CallDurableDemographicFuncTests() DemographicURI = "DemographicURI", BatchSize = 2000, AllowDeleteRecords = true, - caasfolder_STORAGE = "caasfolder_STORAGE", ServiceBusConnectionString_client_internal = "ServiceBusConnectionString_client_internal", GetOrchestrationStatusURL = "GetOrchestrationStatusURL", inboundBlobName = "inbound", ParticipantManagementTopic = "ParticipantManagementTopic", maxNumberOfChecks = 50, + AzureWebJobsStorage = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, + caasfolder_STORAGE = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + } }; _config.Setup(c => c.Value).Returns(testConfig); diff --git a/tests/UnitTests/NemsSubscriptionServiceTests/NemsMeshRetrievalTests/NemsMeshRetrievalTests.cs b/tests/UnitTests/NemsSubscriptionServiceTests/NemsMeshRetrievalTests/NemsMeshRetrievalTests.cs index bceeebab81..151df60eb1 100644 --- a/tests/UnitTests/NemsSubscriptionServiceTests/NemsMeshRetrievalTests/NemsMeshRetrievalTests.cs +++ b/tests/UnitTests/NemsSubscriptionServiceTests/NemsMeshRetrievalTests/NemsMeshRetrievalTests.cs @@ -1,14 +1,14 @@ namespace NHS.CohortManager.Tests.NemsIntegrationServiceTests; -using System.Threading.Tasks; +using Common; using Microsoft.Extensions.Logging; -using Moq; +using Microsoft.Extensions.Options; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Common; -using NHS.Screening.NemsMeshRetrieval; +using Model; +using Moq; using NHS.MESH.Client.Contracts.Services; using NHS.MESH.Client.Models; -using Model; -using Microsoft.Extensions.Options; +using NHS.Screening.NemsMeshRetrieval; +using System.Threading.Tasks; [TestClass] public class NemsMeshRetrievalTests @@ -34,7 +34,18 @@ public NemsMeshRetrievalTests() var testConfig = new NemsMeshRetrievalConfig { NemsMeshMailBox = mailboxId, - nemsmeshfolder_STORAGE = "BlobStorage_ConnectionString", + AzureWebJobsStorage = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, + nemsmeshfolder_STORAGE = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, NemsMeshPassword = "MeshPassword", NemsMeshSharedKey = "MeshSharedKey", NemsMeshKeyPassphrase = "MeshKeyPassphrase", @@ -67,7 +78,7 @@ public async Task Run_DownloadSingleFileFromMesh_Success() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage("BlobStorage_ConnectionString", TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri("https://localhost:8888"), TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -80,7 +91,7 @@ public async Task Run_DownloadSingleFileFromMesh_Success() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Once); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); } @@ -105,7 +116,7 @@ public async Task Run_DownloadSingleChunkedFileFromMesh_Success() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetChunkedMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage("BlobStorage_ConnectionString", TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri("https://localhost:8888"), TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -118,7 +129,7 @@ public async Task Run_DownloadSingleChunkedFileFromMesh_Success() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetChunkedMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Once); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); } @@ -146,7 +157,7 @@ public async Task Run_DownloadTwoSingleFilesFromMesh_Success() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(headResponses.Dequeue()); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(messageResponses.Dequeue()); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage("BlobStorage_ConnectionString", TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri("https://localhost:8888"), TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -158,7 +169,7 @@ public async Task Run_DownloadTwoSingleFilesFromMesh_Success() _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Exactly(2)); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Exactly(2)); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); } @@ -184,7 +195,7 @@ public async Task Run_NoFilesAvailableInMesh_NoAttemptsToDownload() _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.GetChunkedMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Never); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -204,7 +215,7 @@ public async Task Run_SingleFileAvailableCannotUploadToBlob_DoesntMarkAsAcknowle _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage("BlobStorage_ConnectionString", TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(false); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri("https://localhost:8888"), TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(false); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -215,7 +226,7 @@ public async Task Run_SingleFileAvailableCannotUploadToBlob_DoesntMarkAsAcknowle _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Once); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -243,7 +254,7 @@ public async Task Run_SingleFileFailsToGetMessageHead_DoesNotAttemptToDownload() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage("BlobStorage_ConnectionString", TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(false); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri("https://localhost:8888"), TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(false); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -254,7 +265,7 @@ public async Task Run_SingleFileFailsToGetMessageHead_DoesNotAttemptToDownload() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Never); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -284,7 +295,7 @@ public async Task Run_DownloadSingleFile_FailToGetMessage() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage("BlobStorage_ConnectionString", TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri("https://localhost:8888"), TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -297,7 +308,7 @@ public async Task Run_DownloadSingleFile_FailToGetMessage() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Never); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -320,7 +331,7 @@ public async Task Run_DownloadMessages_FailedToGetMessages() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(It.IsAny>); _mockMeshInboxService.Setup(i => i.GetMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(It.IsAny>); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage("BlobStorage_ConnectionString", TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri("https://localhost:8888"), TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, It.IsAny())).ReturnsAsync(It.IsAny>); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -332,7 +343,7 @@ public async Task Run_DownloadMessages_FailedToGetMessages() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.GetMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Never); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -361,7 +372,7 @@ public async Task Run_DownloadSingleChunkFile_FailToGetMessage() _mockMeshInboxService.Setup(i => i.GetMessagesAsync(mailboxId)).ReturnsAsync(inboxResponse); _mockMeshInboxService.Setup(i => i.GetHeadMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(headResponse); _mockMeshInboxService.Setup(i => i.GetChunkedMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(messageResponse); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage("BlobStorage_ConnectionString", TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(new Uri("https://localhost:8888"), TestInboundContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); _mockMeshInboxService.Setup(i => i.AcknowledgeMessageByIdAsync(mailboxId, messageId)).ReturnsAsync(acknowledgeMessageResponse); _mockMeshOperationService.Setup(i => i.MeshHandshakeAsync(mailboxId)).ReturnsAsync(handshakeResponse); @@ -374,7 +385,7 @@ public async Task Run_DownloadSingleChunkFile_FailToGetMessage() _mockMeshInboxService.Verify(i => i.GetMessagesAsync(It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetHeadMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); _mockMeshInboxService.Verify(i => i.GetChunkedMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Once); - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Never); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(It.IsAny(), TestInboundContainer, It.IsAny(), It.IsAny()), Times.Never); _mockMeshInboxService.Verify(i => i.AcknowledgeMessageByIdAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -389,7 +400,18 @@ public async Task Run_WithCustomContainerNames_UsesCustomConfigContainer() var customConfig = new NemsMeshRetrievalConfig { NemsMeshMailBox = mailboxId, - nemsmeshfolder_STORAGE = "BlobStorage_ConnectionString", + AzureWebJobsStorage = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, + nemsmeshfolder_STORAGE = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, NemsMeshPassword = "MeshPassword", NemsMeshSharedKey = "MeshSharedKey", NemsMeshKeyPassphrase = "MeshKeyPassphrase", @@ -413,13 +435,13 @@ public async Task Run_WithCustomContainerNames_UsesCustomConfigContainer() List messages = new List(); MeshResponse inboxResponse = NemsMeshResponseTestHelper.CreateSuccessfulCheckInboxResponse(messages); _mockMeshInboxService.Setup(i => i.GetMessagesAsync(It.IsAny())).ReturnsAsync(inboxResponse); - _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), customConfigContainer, It.IsAny())).ReturnsAsync((BlobFile)null); + _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), customConfigContainer, It.IsAny())).ReturnsAsync((BlobFile)null); // Act await customNemsMeshRetrieval.RunAsync(new Microsoft.Azure.Functions.Worker.TimerInfo()); // Assert - Verify that the custom config container is used for GetFileFromBlobStorage - _mockBlobStorageHelper.Verify(i => i.GetFileFromBlobStorage("BlobStorage_ConnectionString", customConfigContainer, "MeshState.json"), Times.Once); + _mockBlobStorageHelper.Verify(i => i.GetFileFromBlobStorage(new Uri("https://localhost:8888"), customConfigContainer, "MeshState.json"), Times.Once); } [TestMethod] @@ -429,13 +451,13 @@ public async Task Run_WithDefaultContainerNames_UsesDefaultConfigContainer() List messages = new List(); MeshResponse inboxResponse = NemsMeshResponseTestHelper.CreateSuccessfulCheckInboxResponse(messages); _mockMeshInboxService.Setup(i => i.GetMessagesAsync(It.IsAny())).ReturnsAsync(inboxResponse); - _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), TestConfigContainer, It.IsAny())).ReturnsAsync((BlobFile)null); + _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), TestConfigContainer, It.IsAny())).ReturnsAsync((BlobFile)null); // Act await _nemsMeshRetrieval.RunAsync(new Microsoft.Azure.Functions.Worker.TimerInfo()); // Assert - Verify that the default config container is used - _mockBlobStorageHelper.Verify(i => i.GetFileFromBlobStorage("BlobStorage_ConnectionString", TestConfigContainer, "MeshState.json"), Times.Once); + _mockBlobStorageHelper.Verify(i => i.GetFileFromBlobStorage(new Uri("https://localhost:8888"), TestConfigContainer, "MeshState.json"), Times.Once); } [TestMethod] @@ -447,7 +469,18 @@ public async Task SetConfigState_WithCustomConfigContainer_UsesCustomContainer() var customConfig = new NemsMeshRetrievalConfig { NemsMeshMailBox = mailboxId, - nemsmeshfolder_STORAGE = "BlobStorage_ConnectionString", + AzureWebJobsStorage = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, + nemsmeshfolder_STORAGE = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, NemsMeshPassword = "MeshPassword", NemsMeshSharedKey = "MeshSharedKey", NemsMeshKeyPassphrase = "MeshKeyPassphrase", @@ -471,15 +504,15 @@ public async Task SetConfigState_WithCustomConfigContainer_UsesCustomContainer() List messages = new List(); MeshResponse inboxResponse = NemsMeshResponseTestHelper.CreateSuccessfulCheckInboxResponse(messages); _mockMeshInboxService.Setup(i => i.GetMessagesAsync(It.IsAny())).ReturnsAsync(inboxResponse); - _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), customConfigContainer, It.IsAny())).ReturnsAsync((BlobFile)null); - _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(It.IsAny(), customConfigContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); + _mockBlobStorageHelper.Setup(i => i.GetFileFromBlobStorage(It.IsAny(), customConfigContainer, It.IsAny())).ReturnsAsync((BlobFile)null); + _mockBlobStorageHelper.Setup(i => i.UploadFileToBlobStorage(It.IsAny(), customConfigContainer, It.IsAny(), It.IsAny())).ReturnsAsync(true); // Act await customNemsMeshRetrieval.RunAsync(new Microsoft.Azure.Functions.Worker.TimerInfo()); // Assert - Verify that the custom config container is used for UploadFileToBlobStorage - _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage("BlobStorage_ConnectionString", customConfigContainer, It.IsAny(), true), Times.Once); + _mockBlobStorageHelper.Verify(i => i.UploadFileToBlobStorage(new Uri("https://localhost:8888"), customConfigContainer, It.IsAny(), true), Times.Once); } -} \ No newline at end of file +} diff --git a/tests/UnitTests/NemsSubscriptionServiceTests/ProcessNemsUpdateTests/ProcessNemsUpdateTests.cs b/tests/UnitTests/NemsSubscriptionServiceTests/ProcessNemsUpdateTests/ProcessNemsUpdateTests.cs index 38dde3a8dd..b9bfa91021 100644 --- a/tests/UnitTests/NemsSubscriptionServiceTests/ProcessNemsUpdateTests/ProcessNemsUpdateTests.cs +++ b/tests/UnitTests/NemsSubscriptionServiceTests/ProcessNemsUpdateTests/ProcessNemsUpdateTests.cs @@ -41,7 +41,18 @@ public ProcessNemsUpdateTests() DemographicDataServiceURL = "ParticipantDemographicDataServiceURL", ServiceBusConnectionString_client_internal = "ServiceBusConnectionString_client_internal", ParticipantManagementTopic = "update-participant-queue", - nemsmeshfolder_STORAGE = "BlobStorage_ConnectionString" + AzureWebJobsStorage = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + }, + nemsmeshfolder_STORAGE = new BlobStorageConfig + { + AccountName = "accountname", + BlobServiceUri = "https://localhost:8888", + QueueServiceUri = "https://localhost:8888" + } }; _config.Setup(c => c.Value).Returns(testConfig); @@ -68,7 +79,7 @@ public ProcessNemsUpdateTests() // Default: simulate successful poison copy _blobStorageHelperMock .Setup(x => x.CopyFileToPoisonAsync( - It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), @@ -114,7 +125,7 @@ public async Task Run_FailsToRetrieveNhsNumberFromNemsUpdateFile_LogsError() // Verify poison copy occurs due to null NHS number handling _blobStorageHelperMock.Verify(x => x.CopyFileToPoisonAsync( - "BlobStorage_ConnectionString", + new Uri("https://localhost:8888"), _fileName, "nems-updates", "nems-poison", @@ -142,7 +153,7 @@ public async Task Run_PdsReturns404_CopiesFileToPoison_AndStopsProcessing() // Assert: poison copy invoked _blobStorageHelperMock.Verify(x => x.CopyFileToPoisonAsync( - "BlobStorage_ConnectionString", + new Uri("https://localhost:8888"), _fileName, "nems-updates", "nems-poison", @@ -568,7 +579,7 @@ public async Task Run_AddBatchToQueueFails_CopiesFileToPoisonContainer() await _sut.Run(fileStream, _fileName); // Assert _blobStorageHelperMock.Verify(x => x.CopyFileToPoisonAsync( - "BlobStorage_ConnectionString", + new Uri("https://localhost:8888"), _fileName, "nems-updates", "nems-poison", @@ -614,7 +625,7 @@ public async Task Run_InvalidNhsNumberValidation_CopiesFileToPoisonContainer() await _sut.Run(fileStream, _fileName); // Assert _blobStorageHelperMock.Verify(x => x.CopyFileToPoisonAsync( - "BlobStorage_ConnectionString", + new Uri("https://localhost:8888"), _fileName, "nems-updates", "nems-poison", @@ -644,7 +655,7 @@ public async Task Run_PoisonContainerUploadFails_LogsError() .ReturnsAsync("invalid-json"); _blobStorageHelperMock .Setup(x => x.CopyFileToPoisonAsync( - It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), @@ -654,7 +665,7 @@ public async Task Run_PoisonContainerUploadFails_LogsError() await _sut.Run(fileStream, _fileName); // Assert _blobStorageHelperMock.Verify(x => x.CopyFileToPoisonAsync( - "BlobStorage_ConnectionString", + new Uri("https://localhost:8888"), _fileName, "nems-updates", "nems-poison", @@ -698,7 +709,7 @@ public async Task Run_DataServiceClientThrowsException_CopiesFileToPoisonContain await _sut.Run(fileStream, _fileName); // Assert _blobStorageHelperMock.Verify(x => x.CopyFileToPoisonAsync( - "BlobStorage_ConnectionString", + new Uri("https://localhost:8888"), _fileName, "nems-updates", "nems-poison", @@ -732,7 +743,7 @@ public async Task Run_SuccessfulProcessing_DoesNotCallPoisonContainer() // Assert - Verify poison container is never called on successful processing _blobStorageHelperMock.Verify(x => x.CopyFileToPoisonAsync( - It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), diff --git a/tests/UnitTests/SharedTests/BlobStorageHelperTests/BlobStorageHelperTests.cs b/tests/UnitTests/SharedTests/BlobStorageHelperTests/BlobStorageHelperTests.cs index d4f85bd576..e4dddee77d 100644 --- a/tests/UnitTests/SharedTests/BlobStorageHelperTests/BlobStorageHelperTests.cs +++ b/tests/UnitTests/SharedTests/BlobStorageHelperTests/BlobStorageHelperTests.cs @@ -1,12 +1,13 @@ namespace NHS.Screening.BlobStorageHelperTests; -using Microsoft.Extensions.Logging; -using Moq; -using Common; -using Model; +using Azure; +using Azure.Identity; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; -using Azure; +using Common; +using Microsoft.Extensions.Logging; +using Model; +using Moq; using System.Text; [TestClass] @@ -14,7 +15,7 @@ public class BlobStorageHelperTests { private readonly Mock> _mockLogger; private readonly BlobStorageHelper _blobStorageHelper; - private const string TestConnectionString = "UseDevelopmentStorage=true"; + private const string TestBlobStorageServiceUri = "https://localhost:8888"; private const string TestFileName = "test-file.json"; private const string TestFileNameNoExtension = "test-file"; private const string TestSourceContainer = "source-container"; @@ -157,7 +158,15 @@ public void GenerateTimestampedFileName_TimestampFormat_IsCorrect() public async Task CopyFileToPoisonAsync_WithNullConnectionString_ThrowsArgumentNullException() { // Act & Assert - await _blobStorageHelper.CopyFileToPoisonAsync(null!, TestFileName, TestSourceContainer, TestPoisonContainer, false); + await _blobStorageHelper.CopyFileToPoisonAsync(connectionString: null!, TestFileName, TestSourceContainer, TestPoisonContainer, false); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentNullException))] + public async Task CopyFileToPoisonAsync_WithNullServiceUri_ThrowsArgumentNullException() + { + // Act & Assert + await _blobStorageHelper.CopyFileToPoisonAsync(serviceUri: null!, TestFileName, TestSourceContainer, TestPoisonContainer, false); } [TestMethod] @@ -166,7 +175,7 @@ public async Task CopyFileToPoisonAsync_WithEmptyFileName_ThrowsArgumentExceptio // Act & Assert try { - await _blobStorageHelper.CopyFileToPoisonAsync(TestConnectionString, "", TestSourceContainer, TestPoisonContainer, false); + await _blobStorageHelper.CopyFileToPoisonAsync(new Uri(TestBlobStorageServiceUri), "", TestSourceContainer, TestPoisonContainer, false); Assert.Fail("Expected exception was not thrown"); } catch (Exception ex) @@ -182,7 +191,7 @@ public async Task CopyFileToPoisonAsync_WithEmptyFileName_ThrowsArgumentExceptio public async Task CopyFileToPoisonAsync_WithNullFileName_ThrowsArgumentNullException() { // Act & Assert - await _blobStorageHelper.CopyFileToPoisonAsync(TestConnectionString, null!, TestSourceContainer, TestPoisonContainer, false); + await _blobStorageHelper.CopyFileToPoisonAsync(new Uri(TestBlobStorageServiceUri), null!, TestSourceContainer, TestPoisonContainer, false); } [TestMethod] @@ -190,7 +199,7 @@ public void CopyFileToPoisonAsync_MethodSignature_HasCorrectDefaults() { // Arrange & Act var method = typeof(IBlobStorageHelper).GetMethod("CopyFileToPoisonAsync", - new[] { typeof(string), typeof(string), typeof(string), typeof(string), typeof(bool) }); + new[] { typeof(Uri), typeof(string), typeof(string), typeof(string), typeof(bool) }); // Assert Assert.IsNotNull(method, "Method with 5 parameters should exist"); @@ -206,7 +215,7 @@ public void CopyFileToPoisonAsync_BackwardCompatibilityOverload_Exists() { // Arrange & Act var method = typeof(IBlobStorageHelper).GetMethod("CopyFileToPoisonAsync", - new[] { typeof(string), typeof(string), typeof(string) }); + new[] { typeof(Uri), typeof(string), typeof(string) }); // Assert Assert.IsNotNull(method, "3-parameter overload should exist for backward compatibility"); @@ -247,7 +256,18 @@ public async Task UploadFileToBlobStorage_WithNullConnectionString_ThrowsArgumen var mockBlobFile = CreateMockBlobFile(); // Act & Assert - await _blobStorageHelper.UploadFileToBlobStorage(null!, TestSourceContainer, mockBlobFile); + await _blobStorageHelper.UploadFileToBlobStorage(connectionString: null!, TestSourceContainer, mockBlobFile); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentNullException))] + public async Task UploadFileToBlobStorage_WithServiceUri_ThrowsArgumentNullException() + { + // Arrange + var mockBlobFile = CreateMockBlobFile(); + + // Act & Assert + await _blobStorageHelper.UploadFileToBlobStorage(serviceUri: null!, TestSourceContainer, mockBlobFile); } [TestMethod] @@ -255,7 +275,7 @@ public async Task UploadFileToBlobStorage_WithNullConnectionString_ThrowsArgumen public async Task UploadFileToBlobStorage_WithNullBlobFile_ThrowsNullReferenceException() { // Act & Assert - await _blobStorageHelper.UploadFileToBlobStorage(TestConnectionString, TestSourceContainer, null!); + await _blobStorageHelper.UploadFileToBlobStorage(new Uri(TestBlobStorageServiceUri), TestSourceContainer, null!); } [TestMethod] @@ -271,21 +291,21 @@ public async Task UploadFileToBlobStorage_WithValidParameters_ReturnsTrue() // We expect this to fail due to invalid connection string, but not throw null reference try { - await _blobStorageHelper.UploadFileToBlobStorage(TestConnectionString, TestSourceContainer, mockBlobFile); + await _blobStorageHelper.UploadFileToBlobStorage(new Uri(TestBlobStorageServiceUri), TestSourceContainer, mockBlobFile); } catch (Exception ex) { // Should fail due to invalid connection string, not due to null reference - Assert.IsTrue(ex is RequestFailedException || ex is FormatException || ex is ArgumentException || ex is AggregateException, + Assert.IsTrue(ex is RequestFailedException || ex is FormatException || ex is ArgumentException || ex is AggregateException || ex is CredentialUnavailableException, $"Expected storage-related exception, but got {ex.GetType().Name}: {ex.Message}"); } } [TestMethod] - public void UploadFileToBlobStorage_OverwriteParameter_HasCorrectDefault() + public void UploadFileToBlobStorage_ByServiceUri_OverwriteParameter_HasCorrectDefault() { // Arrange & Act - var method = typeof(IBlobStorageHelper).GetMethod("UploadFileToBlobStorage"); + var method = typeof(IBlobStorageHelper).GetMethod("UploadFileToBlobStorage", new Type[] { typeof(Uri), typeof(string), typeof(BlobFile), typeof(bool) }); // Assert Assert.IsNotNull(method, "UploadFileToBlobStorage method should exist"); @@ -297,6 +317,22 @@ public void UploadFileToBlobStorage_OverwriteParameter_HasCorrectDefault() Assert.AreEqual(false, overwriteParam.DefaultValue, "overwrite should default to false"); } + [TestMethod] + public void UploadFileToBlobStorage_ByConnectionString_OverwriteParameter_HasCorrectDefault() + { + // Arrange & Act + var method = typeof(IBlobStorageHelper).GetMethod("UploadFileToBlobStorage", new Type[] { typeof(string), typeof(string), typeof(BlobFile), typeof(bool) }); + + // Assert + Assert.IsNotNull(method, "UploadFileToBlobStorage method should exist"); + var parameters = method.GetParameters(); + var overwriteParam = parameters.FirstOrDefault(p => p.Name == "overwrite"); + + Assert.IsNotNull(overwriteParam, "overwrite parameter should exist"); + Assert.IsTrue(overwriteParam.HasDefaultValue, "overwrite should have default value"); + Assert.AreEqual(false, overwriteParam.DefaultValue, "overwrite should default to false"); + } + #endregion #region GetFileFromBlobStorage Tests @@ -306,7 +342,15 @@ public void UploadFileToBlobStorage_OverwriteParameter_HasCorrectDefault() public async Task GetFileFromBlobStorage_WithNullConnectionString_ThrowsArgumentNullException() { // Act & Assert - await _blobStorageHelper.GetFileFromBlobStorage(null!, TestSourceContainer, TestFileName); + await _blobStorageHelper.GetFileFromBlobStorage(connectionString: null!, TestSourceContainer, TestFileName); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentNullException))] + public async Task GetFileFromBlobStorage_WithNullServiceUri_ThrowsArgumentNullException() + { + // Act & Assert + await _blobStorageHelper.GetFileFromBlobStorage(serviceUri: null!, TestSourceContainer, TestFileName); } [TestMethod] @@ -314,7 +358,7 @@ public async Task GetFileFromBlobStorage_WithNullConnectionString_ThrowsArgument public async Task GetFileFromBlobStorage_WithNullFileName_ThrowsArgumentNullException() { // Act & Assert - await _blobStorageHelper.GetFileFromBlobStorage(TestConnectionString, TestSourceContainer, null!); + await _blobStorageHelper.GetFileFromBlobStorage(new Uri(TestBlobStorageServiceUri), TestSourceContainer, null!); } [TestMethod] @@ -323,7 +367,7 @@ public async Task GetFileFromBlobStorage_WithEmptyFileName_ThrowsException() // Act & Assert try { - await _blobStorageHelper.GetFileFromBlobStorage(TestConnectionString, TestSourceContainer, ""); + await _blobStorageHelper.GetFileFromBlobStorage(new Uri(TestBlobStorageServiceUri), TestSourceContainer, ""); Assert.Fail("Expected exception was not thrown"); } catch (Exception ex) @@ -343,23 +387,34 @@ public async Task GetFileFromBlobStorage_WithValidParameters_ReturnsNullForNonEx // Act & Assert try { - var result = await _blobStorageHelper.GetFileFromBlobStorage(TestConnectionString, TestSourceContainer, TestFileName); + var result = await _blobStorageHelper.GetFileFromBlobStorage(new Uri(TestBlobStorageServiceUri), TestSourceContainer, TestFileName); // If we get here, the method handled the invalid connection gracefully Assert.IsNull(result, "Should return null for non-existent file"); } catch (Exception ex) { // Should fail due to invalid connection string - Assert.IsTrue(ex is RequestFailedException || ex is FormatException || ex is ArgumentException || ex is AggregateException, + Assert.IsTrue(ex is RequestFailedException || ex is FormatException || ex is ArgumentException || ex is AggregateException || ex is CredentialUnavailableException, $"Expected storage-related exception, but got {ex.GetType().Name}: {ex.Message}"); } } [TestMethod] - public void GetFileFromBlobStorage_ReturnType_IsCorrect() + public void GetFileFromBlobStorage_ByConnectionString_ReturnType_IsCorrect() + { + // Arrange & Act + var method = typeof(IBlobStorageHelper).GetMethod("GetFileFromBlobStorage", new Type[] { typeof(string), typeof(string), typeof(string) }); + + // Assert + Assert.IsNotNull(method, "GetFileFromBlobStorage method should exist"); + Assert.AreEqual(typeof(Task), method.ReturnType, "Should return Task"); + } + + [TestMethod] + public void GetFileFromBlobStorage_ByServiceUri_ReturnType_IsCorrect() { // Arrange & Act - var method = typeof(IBlobStorageHelper).GetMethod("GetFileFromBlobStorage"); + var method = typeof(IBlobStorageHelper).GetMethod("GetFileFromBlobStorage", new Type[] { typeof(Uri), typeof(string), typeof(string) }); // Assert Assert.IsNotNull(method, "GetFileFromBlobStorage method should exist"); @@ -434,4 +489,4 @@ private static string GenerateTimestampedFileName(string fileName) } #endregion -} \ No newline at end of file +}