diff --git a/application/CohortManager/compose.data-services.yaml b/application/CohortManager/compose.data-services.yaml index fcb7d533d..9c2baacf4 100644 --- a/application/CohortManager/compose.data-services.yaml +++ b/application/CohortManager/compose.data-services.yaml @@ -155,6 +155,23 @@ services: - DtOsDatabaseConnectionString=Server=db,1433;Database=${DB_NAME};User Id=SA;Password=${PASSWORD};TrustServerCertificate=True - AcceptableLatencyThresholdMs=500 + reference-data-updater: + container_name: reference-data-updater + image: cohort-manager-reference-data-updater + networks: [cohman-network] + build: + context: ./src/Functions/ + dockerfile: screeningDataServices/ReferenceDataUpdater/Dockerfile + args: + BASE_IMAGE: ${FUNCTION_BASE_IMAGE} + environment: + - DtOsDatabaseConnectionString=Server=db,1433;Database=${DB_NAME};User Id=SA;Password=${PASSWORD};TrustServerCertificate=True + - ServiceBusConnectionString=${SERVICE_BUS_CONNECTION_STRING} + - ReferenceDataTopicName=reference-data-updates + - ReferenceDataSubscription=reference-data-updater-sub + - AzureWebJobsStorage=${AZURE_WEB_JOBS_STORAGE} + - SeedDataBlobContainer=seed-data + servicenow-cases-data-service: container_name: servicenow-cases-data-service image: cohort-manager-servicenow-cases-data-service diff --git a/application/CohortManager/src/Functions/Shared/DataServices.Migrations/DataServices.Migrations.csproj b/application/CohortManager/src/Functions/Shared/DataServices.Migrations/DataServices.Migrations.csproj index 0fb55d3e4..f7c1029d3 100644 --- a/application/CohortManager/src/Functions/Shared/DataServices.Migrations/DataServices.Migrations.csproj +++ b/application/CohortManager/src/Functions/Shared/DataServices.Migrations/DataServices.Migrations.csproj @@ -1,4 +1,4 @@ - + net8.0 diff --git a/application/CohortManager/src/Functions/Shared/Model/ReferenceDataUpdateMessage.cs b/application/CohortManager/src/Functions/Shared/Model/ReferenceDataUpdateMessage.cs new file mode 100644 index 000000000..404ea3950 --- /dev/null +++ b/application/CohortManager/src/Functions/Shared/Model/ReferenceDataUpdateMessage.cs @@ -0,0 +1,11 @@ +namespace Model; + +using System.Text.Json; + +public class ReferenceDataUpdateMessage +{ + public required string DataType { get; set; } + public required JsonElement Data { get; set; } + public string? CorrelationId { get; set; } + public DateTime Timestamp { get; set; } = DateTime.UtcNow; +} diff --git a/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/Dockerfile b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/Dockerfile new file mode 100644 index 000000000..ec25afc8a --- /dev/null +++ b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/Dockerfile @@ -0,0 +1,17 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} AS function + +COPY ./screeningDataServices/ReferenceDataUpdater /app/src/dotnet-function-app +WORKDIR /app/src/dotnet-function-app + +RUN --mount=type=cache,target=/root/.nuget/packages \ + dotnet publish ./*.csproj --output /home/site/wwwroot + +# To enable ssh & remote debugging on app service change the base image to the one below +# FROM mcr.microsoft.com/azure-functions/dotnet-isolated:4-dotnet-isolated8.0-appservice +FROM mcr.microsoft.com/azure-functions/dotnet-isolated:4-dotnet-isolated8.0 +ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ + AzureFunctionsJobHost__Logging__Console__IsEnabled=true + +COPY --from=function ["/home/site/wwwroot", "/home/site/wwwroot"] + diff --git a/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/IReferenceDataInsertHandler.cs b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/IReferenceDataInsertHandler.cs new file mode 100644 index 000000000..87e278584 --- /dev/null +++ b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/IReferenceDataInsertHandler.cs @@ -0,0 +1,8 @@ +namespace ReferenceDataUpdater; + +using System.Text.Json; + +public interface IReferenceDataInsertHandler +{ + Task ProcessRecord(string dataType, JsonElement data); +} diff --git a/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/Program.cs b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/Program.cs new file mode 100644 index 000000000..8dcd727d3 --- /dev/null +++ b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/Program.cs @@ -0,0 +1,22 @@ +using Common; +using DataServices.Core; +using DataServices.Database; +using HealthChecks.Extensions; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using ReferenceDataUpdater; + +var host = new HostBuilder() + .ConfigureFunctionsWorkerDefaults() + .AddDataServicesHandler() + .ConfigureServices(services => + { + services.AddSingleton(); + services.AddScoped(); + services.AddDatabaseHealthCheck("ReferenceDataUpdater"); + }) + .AddTelemetry() + .Build(); + +await host.RunAsync(); diff --git a/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/ReferenceDataInsertHandler.cs b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/ReferenceDataInsertHandler.cs new file mode 100644 index 000000000..7956288ad --- /dev/null +++ b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/ReferenceDataInsertHandler.cs @@ -0,0 +1,176 @@ +namespace ReferenceDataUpdater; + +using System.Collections.Concurrent; +using System.Text; +using System.Text.Json; +using Common; +using DataServices.Core; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Model; + +public class ReferenceDataInsertHandler : IReferenceDataInsertHandler +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + WriteIndented = true + }; + + private static readonly Dictionary TypeRegistry = new(StringComparer.OrdinalIgnoreCase) + { + ["BsSelectGpPractice"] = (typeof(BsSelectGpPractice), "BsSelectGpPractice.json"), + ["BsSelectOutCode"] = (typeof(BsSelectOutCode), "BsSelectOutCode.json"), + ["CurrentPosting"] = (typeof(CurrentPosting), "CurrentPosting.json"), + ["ExcludedSMULookup"] = (typeof(ExcludedSMULookup), "ExcludedSMULookup.json"), + ["LanguageCode"] = (typeof(LanguageCode), "LanguageCode.json"), + ["ScreeningLkp"] = (typeof(ScreeningLkp), "ScreeningLkp.json"), + ["GeneCodeLkp"] = (typeof(GeneCodeLkp), "GeneCodeLkp.json"), + ["HigherRiskReferralReasonLkp"] = (typeof(HigherRiskReferralReasonLkp), "HigherRiskReferralReasonLkp.json"), + ["BsoOrganisation"] = (typeof(BsoOrganisation), "BsoOrganisation.json"), + ["GenderMaster"] = (typeof(GenderMaster), "GenderMaster.json"), + }; + + private readonly IServiceProvider _serviceProvider; + private readonly IBlobStorageHelper _blobStorageHelper; + private readonly ILogger _logger; + private readonly string _storageConnectionString; + private readonly string _seedDataContainerName; + private static readonly ConcurrentDictionary>> _insertDelegates = new(); + + public ReferenceDataInsertHandler( + IServiceProvider serviceProvider, + IBlobStorageHelper blobStorageHelper, + ILogger logger) + { + _serviceProvider = serviceProvider; + _blobStorageHelper = blobStorageHelper; + _logger = logger; + _storageConnectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage") + ?? throw new InvalidOperationException("AzureWebJobsStorage environment variable is not set."); + _seedDataContainerName = Environment.GetEnvironmentVariable("SeedDataBlobContainer") ?? "seed-data"; + } + + public async Task ProcessRecord(string dataType, JsonElement data) + { + if (!TypeRegistry.TryGetValue(dataType, out var registration)) + { + _logger.LogError("Unknown reference data type: {DataType}", dataType); + return false; + } + + var entity = JsonSerializer.Deserialize(data.GetRawText(), registration.EntityType, JsonOptions); + if (entity is null) + { + _logger.LogError("Failed to deserialise payload for type {DataType}.", dataType); + return false; + } + + var dbInserted = await InsertIntoDatabase(dataType, registration.EntityType, entity); + await AppendToBlob(dataType, registration.BlobFileName, data); + + return dbInserted; + } + + private async Task InsertIntoDatabase(string dataType, Type entityType, object entity) + { + try + { + var accessorType = typeof(IDataServiceAccessor<>).MakeGenericType(entityType); + var accessor = _serviceProvider.GetRequiredService(accessorType); + + var invoker = _insertDelegates.GetOrAdd(entityType, t => + { + var aType = typeof(IDataServiceAccessor<>).MakeGenericType(t); + var method = aType.GetMethod("InsertSingle")!; + + var accessorParam = System.Linq.Expressions.Expression.Parameter(typeof(object), "a"); + var entityParam = System.Linq.Expressions.Expression.Parameter(typeof(object), "e"); + + var call = System.Linq.Expressions.Expression.Call( + System.Linq.Expressions.Expression.Convert(accessorParam, aType), + method, + System.Linq.Expressions.Expression.Convert(entityParam, t)); + + return System.Linq.Expressions.Expression.Lambda>>( + call, accessorParam, entityParam).Compile(); + }); + + var result = await invoker(accessor, entity); + + if (!result) + { + _logger.LogWarning("InsertSingle returned false for type {DataType}. Record may not have been inserted, possibly because it already exists.", dataType); + } + + return result; + } + catch (DbUpdateException ex) when (IsPrimaryKeyViolation(ex)) + { + _logger.LogWarning(ex, "Duplicate record detected for type {DataType}. Skipping insert.", dataType); + return false; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to insert record into database for type {DataType}.", dataType); + return false; + } + } + + private async Task AppendToBlob(string dataType, string blobFileName, JsonElement newRecord) + { + try + { + var existingRecords = new List(); + + var existingBlob = await _blobStorageHelper.GetFileFromBlobStorage(_storageConnectionString, _seedDataContainerName, blobFileName); + if (existingBlob?.Data != null) + { + existingBlob.Data.Position = 0; + using var reader = new StreamReader(existingBlob.Data); + var existingJson = await reader.ReadToEndAsync(); + existingRecords = JsonSerializer.Deserialize>(existingJson, JsonOptions) ?? new List(); + } + + existingRecords.Add(newRecord); + + var updatedJson = JsonSerializer.Serialize(existingRecords, JsonOptions); + var bytes = Encoding.UTF8.GetBytes(updatedJson); + var blobFile = new BlobFile(bytes, blobFileName); + + await _blobStorageHelper.UploadFileToBlobStorage(_storageConnectionString, _seedDataContainerName, blobFile, overwrite: true); + + _logger.LogInformation( + "Appended record to blob {BlobFileName} for type {DataType}. Total records: {Count}", + blobFileName, dataType, existingRecords.Count); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Failed to append record to blob for type {DataType}. DB insert was successful; blob is out of sync.", + dataType); + } + } + + private static bool IsPrimaryKeyViolation(DbUpdateException ex) + { + const int SqlServerPrimaryKeyViolation = 2627; + const int SqlServerUniqueConstraintViolation = 2601; + + for (Exception? current = ex.InnerException; current is not null; current = current.InnerException) + { + var numberProperty = current.GetType().GetProperty("Number"); + if (numberProperty?.PropertyType == typeof(int)) + { + var errorNumber = (int)numberProperty.GetValue(current)!; + if (errorNumber == SqlServerPrimaryKeyViolation || errorNumber == SqlServerUniqueConstraintViolation) + { + return true; + } + } + } + + return false; + } +} diff --git a/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/ReferenceDataUpdater.csproj b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/ReferenceDataUpdater.csproj new file mode 100644 index 000000000..20e943376 --- /dev/null +++ b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/ReferenceDataUpdater.csproj @@ -0,0 +1,31 @@ + + + net8.0 + v4 + Exe + enable + enable + + + + + + + + + + PreserveNewest + + + PreserveNewest + Never + + + + + + + + + + diff --git a/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/ReferenceDataUpdaterFunction.cs b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/ReferenceDataUpdaterFunction.cs new file mode 100644 index 000000000..d006f4a2d --- /dev/null +++ b/application/CohortManager/src/Functions/screeningDataServices/ReferenceDataUpdater/ReferenceDataUpdaterFunction.cs @@ -0,0 +1,86 @@ +namespace ReferenceDataUpdater; + +using System.Text.Json; +using Azure.Messaging.ServiceBus; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; +using Model; + +public class ReferenceDataUpdaterFunction +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly IReferenceDataInsertHandler _insertHandler; + private readonly ILogger _logger; + + public ReferenceDataUpdaterFunction( + IReferenceDataInsertHandler insertHandler, + ILogger logger) + { + _insertHandler = insertHandler; + _logger = logger; + } + + [Function(nameof(ReferenceDataUpdaterFunction))] + public async Task Run( + [ServiceBusTrigger( + topicName: "%ReferenceDataTopicName%", + subscriptionName: "%ReferenceDataSubscription%", + Connection = "ServiceBusConnectionString", + AutoCompleteMessages = false)] + ServiceBusReceivedMessage message, + ServiceBusMessageActions messageActions) + { + ReferenceDataUpdateMessage? updateMessage; + try + { + updateMessage = JsonSerializer.Deserialize(message.Body, JsonOptions); + } + catch (JsonException ex) + { + _logger.LogError(ex, "Failed to deserialise reference data update message."); + await messageActions.DeadLetterMessageAsync(message); + return; + } + + if (updateMessage is null || string.IsNullOrWhiteSpace(updateMessage.DataType)) + { + _logger.LogError("Reference data update message was null or missing DataType."); + await messageActions.DeadLetterMessageAsync(message); + return; + } + + _logger.LogInformation( + "Processing reference data update | Type: {DataType} | CorrelationId: {CorrelationId}", + updateMessage.DataType, updateMessage.CorrelationId); + + try + { + var success = await _insertHandler.ProcessRecord(updateMessage.DataType, updateMessage.Data); + + if (!success) + { + _logger.LogError( + "Failed to process reference data update for type {DataType}.", + updateMessage.DataType); + await messageActions.DeadLetterMessageAsync(message); + return; + } + + await messageActions.CompleteMessageAsync(message); + _logger.LogInformation( + "Reference data update completed | Type: {DataType} | CorrelationId: {CorrelationId}", + updateMessage.DataType, updateMessage.CorrelationId); + } + catch (Exception ex) + { + _logger.LogError(ex, + "Unexpected error processing reference data update for type {DataType}. CorrelationId: {CorrelationId}", + updateMessage.DataType, updateMessage.CorrelationId); + await messageActions.DeadLetterMessageAsync(message); + } + } +} diff --git a/infrastructure/tf-core/environments/development.tfvars b/infrastructure/tf-core/environments/development.tfvars index 3ae6c6fcd..8a7db32e0 100644 --- a/infrastructure/tf-core/environments/development.tfvars +++ b/infrastructure/tf-core/environments/development.tfvars @@ -1158,6 +1158,19 @@ function_apps = { } } + ReferenceDataUpdater = { + name_suffix = "reference-data-updater" + function_endpoint_name = "ReferenceDataUpdater" + app_service_plan_key = "NonScaling" + db_connection_string = "DtOsDatabaseConnectionString" + service_bus_connections = ["external"] + env_vars_static = { + ReferenceDataTopicName = "reference-data-updates" + ReferenceDataSubscription = "ReferenceDataUpdater" + SeedDataBlobContainer = "seed-data" + } + } + NemsSubscribe = { name_suffix = "nems-subscribe" function_endpoint_name = "NemsSubscribe" @@ -1343,6 +1356,19 @@ service_bus = { } } } + + external = { + capacity = 1 + sku_tier = "Premium" + max_payload_size = "100mb" + topics = { + reference-data-updates = { + batched_operations_enabled = true + max_delivery_count = 10 + subscribers = ["ReferenceDataUpdater"] + } + } + } } sqlserver = { diff --git a/infrastructure/tf-core/environments/integration.tfvars b/infrastructure/tf-core/environments/integration.tfvars index bb051878f..78cf3bf0d 100644 --- a/infrastructure/tf-core/environments/integration.tfvars +++ b/infrastructure/tf-core/environments/integration.tfvars @@ -1131,6 +1131,19 @@ function_apps = { } } + ReferenceDataUpdater = { + name_suffix = "reference-data-updater" + function_endpoint_name = "ReferenceDataUpdater" + app_service_plan_key = "NonScaling" + db_connection_string = "DtOsDatabaseConnectionString" + service_bus_connections = ["external"] + env_vars_static = { + ReferenceDataTopicName = "reference-data-updates" + ReferenceDataSubscription = "ReferenceDataUpdater" + SeedDataBlobContainer = "seed-data" + } + } + NemsSubscribe = { name_suffix = "nems-subscribe" function_endpoint_name = "NemsSubscribe" @@ -1315,6 +1328,19 @@ service_bus = { } } } + + external = { + capacity = 1 + sku_tier = "Premium" + max_payload_size = "100mb" + topics = { + reference-data-updates = { + batched_operations_enabled = true + max_delivery_count = 10 + subscribers = ["ReferenceDataUpdater"] + } + } + } } sqlserver = { diff --git a/infrastructure/tf-core/environments/preprod.tfvars b/infrastructure/tf-core/environments/preprod.tfvars index 8e8509d60..97f5e0c8e 100644 --- a/infrastructure/tf-core/environments/preprod.tfvars +++ b/infrastructure/tf-core/environments/preprod.tfvars @@ -1141,6 +1141,19 @@ function_apps = { } } + ReferenceDataUpdater = { + name_suffix = "reference-data-updater" + function_endpoint_name = "ReferenceDataUpdater" + app_service_plan_key = "NonScaling" + db_connection_string = "DtOsDatabaseConnectionString" + service_bus_connections = ["external"] + env_vars_static = { + ReferenceDataTopicName = "reference-data-updates" + ReferenceDataSubscription = "ReferenceDataUpdater" + SeedDataBlobContainer = "seed-data" + } + } + NemsSubscribe = { name_suffix = "nems-subscribe" function_endpoint_name = "NemsSubscribe" @@ -1326,6 +1339,19 @@ service_bus = { } } } + + external = { + capacity = 1 + sku_tier = "Premium" + max_payload_size = "100mb" + topics = { + reference-data-updates = { + batched_operations_enabled = true + max_delivery_count = 10 + subscribers = ["ReferenceDataUpdater"] + } + } + } } sqlserver = { diff --git a/infrastructure/tf-core/environments/production.tfvars b/infrastructure/tf-core/environments/production.tfvars index 366a1f8b4..fe501538c 100644 --- a/infrastructure/tf-core/environments/production.tfvars +++ b/infrastructure/tf-core/environments/production.tfvars @@ -1162,6 +1162,19 @@ function_apps = { } } + ReferenceDataUpdater = { + name_suffix = "reference-data-updater" + function_endpoint_name = "ReferenceDataUpdater" + app_service_plan_key = "NonScaling" + db_connection_string = "DtOsDatabaseConnectionString" + service_bus_connections = ["external"] + env_vars_static = { + ReferenceDataTopicName = "reference-data-updates" + ReferenceDataSubscription = "ReferenceDataUpdater" + SeedDataBlobContainer = "seed-data" + } + } + NemsSubscribe = { name_suffix = "nems-subscribe" function_endpoint_name = "NemsSubscribe" @@ -1347,6 +1360,19 @@ service_bus = { } } } + + external = { + capacity = 1 + sku_tier = "Premium" + max_payload_size = "100mb" + topics = { + reference-data-updates = { + batched_operations_enabled = true + max_delivery_count = 10 + subscribers = ["ReferenceDataUpdater"] + } + } + } } sqlserver = { diff --git a/tests/UnitTests/ScreeningDataServicesTests/ReferenceDataUpdaterTests/ReferenceDataInsertHandlerTests.cs b/tests/UnitTests/ScreeningDataServicesTests/ReferenceDataUpdaterTests/ReferenceDataInsertHandlerTests.cs new file mode 100644 index 000000000..6e2919596 --- /dev/null +++ b/tests/UnitTests/ScreeningDataServicesTests/ReferenceDataUpdaterTests/ReferenceDataInsertHandlerTests.cs @@ -0,0 +1,282 @@ +namespace NHS.CohortManager.Tests.UnitTests.ScreeningDataServicesTests; + +using System.Text; +using System.Text.Json; +using Common; +using DataServices.Core; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Model; +using Moq; +using ReferenceDataUpdater; + +[TestClass] +public class ReferenceDataInsertHandlerTests +{ + private readonly Mock _serviceProviderMock = new(); + private readonly Mock _blobStorageHelperMock = new(); + private readonly Mock> _loggerMock = new(); + private readonly ReferenceDataInsertHandler _handler; + private readonly string? _originalAzureWebJobsStorage; + private readonly string? _originalSeedDataBlobContainer; + + public ReferenceDataInsertHandlerTests() + { + _originalAzureWebJobsStorage = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); + _originalSeedDataBlobContainer = Environment.GetEnvironmentVariable("SeedDataBlobContainer"); + + Environment.SetEnvironmentVariable("AzureWebJobsStorage", "UseDevelopmentStorage=true"); + Environment.SetEnvironmentVariable("SeedDataBlobContainer", "seed-data"); + + _handler = new ReferenceDataInsertHandler( + _serviceProviderMock.Object, + _blobStorageHelperMock.Object, + _loggerMock.Object); + } + + [TestCleanup] + public void Cleanup() + { + Environment.SetEnvironmentVariable("AzureWebJobsStorage", _originalAzureWebJobsStorage); + Environment.SetEnvironmentVariable("SeedDataBlobContainer", _originalSeedDataBlobContainer); + } + + private Mock> SetupAccessor(bool insertResult = true) where T : class + { + var accessorMock = new Mock>(); + accessorMock.Setup(a => a.InsertSingle(It.IsAny())).ReturnsAsync(insertResult); + _serviceProviderMock.Setup(sp => sp.GetService(typeof(IDataServiceAccessor))) + .Returns(accessorMock.Object); + return accessorMock; + } + + private void SetupBlobStorageDefaults(string blobFileName) + { + _blobStorageHelperMock.Setup(b => b.GetFileFromBlobStorage(It.IsAny(), It.IsAny(), blobFileName)) + .ReturnsAsync((BlobFile)null!); + _blobStorageHelperMock.Setup(b => b.UploadFileToBlobStorage(It.IsAny(), It.IsAny(), It.IsAny(), true)) + .ReturnsAsync(true); + } + + private static JsonElement CreatePayload(object data) + { + return JsonSerializer.SerializeToElement(data); + } + + [TestMethod] + public async Task ProcessRecord_UnknownDataType_ReturnsFalse() + { + // Arrange + var data = CreatePayload(new { Id = 1 }); + + // Act + var result = await _handler.ProcessRecord("NonExistentType", data); + + // Assert + Assert.IsFalse(result); + } + + [TestMethod] + public async Task ProcessRecord_ValidRecord_InsertsAndAppendsToBlob_ReturnsTrue() + { + // Arrange + var data = CreatePayload(new { GpPracticeCode = "Y12345" }); + var accessorMock = SetupAccessor(); + SetupBlobStorageDefaults("BsSelectGpPractice.json"); + + // Act + var result = await _handler.ProcessRecord("BsSelectGpPractice", data); + + // Assert + Assert.IsTrue(result); + accessorMock.Verify(a => a.InsertSingle(It.IsAny()), Times.Once); + _blobStorageHelperMock.Verify(b => b.UploadFileToBlobStorage(It.IsAny(), "seed-data", It.IsAny(), true), Times.Once); + } + + [TestMethod] + public async Task ProcessRecord_DatabaseInsertThrows_ReturnsFalse() + { + // Arrange + var data = CreatePayload(new { ScreeningLkpId = 1 }); + var accessorMock = new Mock>(); + accessorMock.Setup(a => a.InsertSingle(It.IsAny())) + .ThrowsAsync(new Exception("Database connection failed")); + _serviceProviderMock.Setup(sp => sp.GetService(typeof(IDataServiceAccessor))) + .Returns(accessorMock.Object); + + // Act + var result = await _handler.ProcessRecord("ScreeningLkp", data); + + // Assert + Assert.IsFalse(result); + } + + [TestMethod] + public async Task ProcessRecord_PrimaryKeyViolation_ReturnsFalse() + { + // Arrange + var data = CreatePayload(new { LanguageCodeId = "EN", LanguageDescription = "English" }); + var sqlException = new SqlExceptionWithNumber(2627); + var dbUpdateException = new DbUpdateException("An error occurred", sqlException); + + var accessorMock = new Mock>(); + accessorMock.Setup(a => a.InsertSingle(It.IsAny())).ThrowsAsync(dbUpdateException); + _serviceProviderMock.Setup(sp => sp.GetService(typeof(IDataServiceAccessor))) + .Returns(accessorMock.Object); + SetupBlobStorageDefaults("LanguageCode.json"); + + // Act + var result = await _handler.ProcessRecord("LanguageCode", data); + + // Assert + Assert.IsFalse(result); + } + + [TestMethod] + public async Task ProcessRecord_UniqueConstraintViolation_ReturnsFalse() + { + // Arrange + var data = CreatePayload(new { GenderCode = "M" }); + var sqlException = new SqlExceptionWithNumber(2601); + var dbUpdateException = new DbUpdateException("An error occurred", sqlException); + + var accessorMock = new Mock>(); + accessorMock.Setup(a => a.InsertSingle(It.IsAny())).ThrowsAsync(dbUpdateException); + _serviceProviderMock.Setup(sp => sp.GetService(typeof(IDataServiceAccessor))) + .Returns(accessorMock.Object); + SetupBlobStorageDefaults("GenderMaster.json"); + + // Act + var result = await _handler.ProcessRecord("GenderMaster", data); + + // Assert + Assert.IsFalse(result); + } + + [TestMethod] + public async Task ProcessRecord_BlobAppendFails_ReturnsTrue() + { + // Arrange + var data = CreatePayload(new { OutCode = "AB1" }); + SetupAccessor(); + _blobStorageHelperMock.Setup(b => b.GetFileFromBlobStorage(It.IsAny(), It.IsAny(), "BsSelectOutCode.json")) + .ThrowsAsync(new Exception("Blob storage unavailable")); + + // Act + var result = await _handler.ProcessRecord("BsSelectOutCode", data); + + // Assert + Assert.IsTrue(result); + } + + [TestMethod] + public async Task ProcessRecord_ExistingBlobRecords_AppendsNewRecord() + { + // Arrange + var existingJson = JsonSerializer.Serialize(new[] { new { PostingId = 1 } }); + var existingBlob = new BlobFile(Encoding.UTF8.GetBytes(existingJson), "CurrentPosting.json"); + var data = CreatePayload(new { PostingId = 2 }); + + SetupAccessor(); + _blobStorageHelperMock.Setup(b => b.GetFileFromBlobStorage(It.IsAny(), It.IsAny(), "CurrentPosting.json")) + .ReturnsAsync(existingBlob); + + BlobFile? uploadedBlob = null; + _blobStorageHelperMock.Setup(b => b.UploadFileToBlobStorage(It.IsAny(), It.IsAny(), It.IsAny(), true)) + .Callback((_, _, blob, _) => uploadedBlob = blob) + .ReturnsAsync(true); + + // Act + var result = await _handler.ProcessRecord("CurrentPosting", data); + + // Assert + Assert.IsTrue(result); + Assert.IsNotNull(uploadedBlob); + + uploadedBlob.Data.Position = 0; + using var reader = new StreamReader(uploadedBlob.Data); + var uploadedJson = await reader.ReadToEndAsync(); + var records = JsonSerializer.Deserialize>(uploadedJson); + Assert.AreEqual(2, records!.Count); + } + + [TestMethod] + public async Task ProcessRecord_NullJsonPayload_ReturnsFalse() + { + // Arrange + var data = JsonSerializer.SerializeToElement(null!); + + // Act + var result = await _handler.ProcessRecord("BsSelectGpPractice", data); + + // Assert + Assert.IsFalse(result); + } + + [DataRow("BsSelectGpPractice", DisplayName = "BsSelectGpPractice is a registered type")] + [DataRow("BsSelectOutCode", DisplayName = "BsSelectOutCode is a registered type")] + [DataRow("CurrentPosting", DisplayName = "CurrentPosting is a registered type")] + [DataRow("ExcludedSMULookup", DisplayName = "ExcludedSMULookup is a registered type")] + [DataRow("LanguageCode", DisplayName = "LanguageCode is a registered type")] + [DataRow("ScreeningLkp", DisplayName = "ScreeningLkp is a registered type")] + [DataRow("GeneCodeLkp", DisplayName = "GeneCodeLkp is a registered type")] + [DataRow("HigherRiskReferralReasonLkp", DisplayName = "HigherRiskReferralReasonLkp is a registered type")] + [DataRow("BsoOrganisation", DisplayName = "BsoOrganisation is a registered type")] + [DataRow("GenderMaster", DisplayName = "GenderMaster is a registered type")] + [TestMethod] + public async Task ProcessRecord_RegisteredDataType_DoesNotLogUnknownType(string dataType) + { + // Arrange + var data = CreatePayload(new { Id = 1 }); + _serviceProviderMock.Setup(sp => sp.GetService(It.IsAny())).Returns(null!); + + // Act + await _handler.ProcessRecord(dataType, data); + + // Assert + _loggerMock.Verify( + x => x.Log( + LogLevel.Error, + It.IsAny(), + It.Is((v, t) => v.ToString()!.Contains("Unknown reference data type")), + It.IsAny(), + It.IsAny>() + ), Times.Never); + } + + [TestMethod] + public async Task ProcessRecord_CaseInsensitiveDataType_ReturnsTrue() + { + // Arrange + var data = CreatePayload(new { GpPracticeCode = "Y12345" }); + var accessorMock = SetupAccessor(); + SetupBlobStorageDefaults("BsSelectGpPractice.json"); + + // Act + var result = await _handler.ProcessRecord("bsselectgppractice", data); + + // Assert + Assert.IsTrue(result); + accessorMock.Verify(a => a.InsertSingle(It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task ProcessRecord_InsertSingleReturnsFalse_ReturnsFalse() + { + // Arrange + var data = CreatePayload(new { GpPracticeCode = "Y99999" }); + SetupAccessor(insertResult: false); + SetupBlobStorageDefaults("BsSelectGpPractice.json"); + + // Act + var result = await _handler.ProcessRecord("BsSelectGpPractice", data); + + // Assert + Assert.IsFalse(result); + } +} + +internal class SqlExceptionWithNumber(int number) : Exception($"SQL error {number}") +{ + public int Number { get; } = number; +} diff --git a/tests/UnitTests/ScreeningDataServicesTests/ReferenceDataUpdaterTests/ReferenceDataUpdaterFunctionTests.cs b/tests/UnitTests/ScreeningDataServicesTests/ReferenceDataUpdaterTests/ReferenceDataUpdaterFunctionTests.cs new file mode 100644 index 000000000..164b32f74 --- /dev/null +++ b/tests/UnitTests/ScreeningDataServicesTests/ReferenceDataUpdaterTests/ReferenceDataUpdaterFunctionTests.cs @@ -0,0 +1,184 @@ +namespace NHS.CohortManager.Tests.UnitTests.ScreeningDataServicesTests; + +using System.Text.Json; +using Azure.Messaging.ServiceBus; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; +using Moq; +using Model; +using ReferenceDataUpdater; + +[TestClass] +public class ReferenceDataUpdaterFunctionTests +{ + private readonly Mock _insertHandlerMock = new(); + private readonly Mock> _loggerMock = new(); + private readonly Mock _messageActionsMock = new(); + private readonly ReferenceDataUpdaterFunction _function; + + private readonly ReferenceDataUpdateMessage _validUpdateMessage = new() + { + DataType = "BsSelectGpPractice", + Data = JsonSerializer.SerializeToElement(new { GpPracticeCode = "Y12345" }), + CorrelationId = "test-correlation-id", + Timestamp = DateTime.UtcNow + }; + + public ReferenceDataUpdaterFunctionTests() + { + _function = new ReferenceDataUpdaterFunction(_insertHandlerMock.Object, _loggerMock.Object); + } + + private static ServiceBusReceivedMessage CreateMessage(object body) + { + return ServiceBusModelFactory.ServiceBusReceivedMessage( + body: new BinaryData(JsonSerializer.Serialize(body)), + messageId: "test-message-id", + correlationId: "test-correlation-id" + ); + } + + private static ServiceBusReceivedMessage CreateMessage(string rawBody) + { + return ServiceBusModelFactory.ServiceBusReceivedMessage( + body: new BinaryData(rawBody), + messageId: "test-message-id", + correlationId: "test-correlation-id" + ); + } + + private void VerifyMessageCompleted() + { + _messageActionsMock.Verify( + x => x.CompleteMessageAsync(It.IsAny(), CancellationToken.None), + Times.Once); + _messageActionsMock.Verify( + x => x.DeadLetterMessageAsync(It.IsAny(), null, null, null, CancellationToken.None), + Times.Never); + } + + private void VerifyMessageDeadLettered() + { + _messageActionsMock.Verify( + x => x.DeadLetterMessageAsync(It.IsAny(), null, null, null, CancellationToken.None), + Times.Once); + _messageActionsMock.Verify( + x => x.CompleteMessageAsync(It.IsAny(), CancellationToken.None), + Times.Never); + } + + [TestMethod] + public async Task Run_ProcessRecordSucceeds_CompletesMessage() + { + // Arrange + var message = CreateMessage(_validUpdateMessage); + _insertHandlerMock.Setup(h => h.ProcessRecord(_validUpdateMessage.DataType, It.IsAny())) + .ReturnsAsync(true); + + // Act + await _function.Run(message, _messageActionsMock.Object); + + // Assert + VerifyMessageCompleted(); + } + + [TestMethod] + public async Task Run_ProcessRecordReturnsFalse_DeadLettersMessage() + { + // Arrange + var message = CreateMessage(_validUpdateMessage); + _insertHandlerMock.Setup(h => h.ProcessRecord(_validUpdateMessage.DataType, It.IsAny())) + .ReturnsAsync(false); + + // Act + await _function.Run(message, _messageActionsMock.Object); + + // Assert + VerifyMessageDeadLettered(); + } + + [TestMethod] + public async Task Run_InvalidJsonBody_DeadLettersMessage() + { + // Arrange + var message = CreateMessage("not valid json {{{"); + + // Act + await _function.Run(message, _messageActionsMock.Object); + + // Assert + VerifyMessageDeadLettered(); + } + + [TestMethod] + public async Task Run_NullDataType_DeadLettersMessage() + { + // Arrange + var updateMessage = new ReferenceDataUpdateMessage + { + DataType = null!, + Data = _validUpdateMessage.Data, + CorrelationId = "test-correlation-id", + Timestamp = DateTime.UtcNow + }; + var message = CreateMessage(updateMessage); + + // Act + await _function.Run(message, _messageActionsMock.Object); + + // Assert + VerifyMessageDeadLettered(); + _insertHandlerMock.Verify(h => h.ProcessRecord(It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task Run_WhitespaceDataType_DeadLettersMessage() + { + // Arrange + var updateMessage = new ReferenceDataUpdateMessage + { + DataType = " ", + Data = _validUpdateMessage.Data, + CorrelationId = "test-correlation-id", + Timestamp = DateTime.UtcNow + }; + var message = CreateMessage(updateMessage); + + // Act + await _function.Run(message, _messageActionsMock.Object); + + // Assert + VerifyMessageDeadLettered(); + _insertHandlerMock.Verify(h => h.ProcessRecord(It.IsAny(), It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task Run_ProcessRecordThrowsException_DeadLettersMessage() + { + // Arrange + var message = CreateMessage(_validUpdateMessage); + _insertHandlerMock.Setup(h => h.ProcessRecord(_validUpdateMessage.DataType, It.IsAny())) + .ThrowsAsync(new InvalidOperationException("Something went wrong")); + + // Act + await _function.Run(message, _messageActionsMock.Object); + + // Assert + VerifyMessageDeadLettered(); + } + + [TestMethod] + public async Task Run_NullMessageBody_DeadLettersMessage() + { + // Arrange + var message = CreateMessage("null"); + + // Act + await _function.Run(message, _messageActionsMock.Object); + + // Assert + _messageActionsMock.Verify( + x => x.DeadLetterMessageAsync(It.IsAny(), null, null, null, CancellationToken.None), + Times.Once); + } +} diff --git a/tests/UnitTests/ScreeningDataServicesTests/ReferenceDataUpdaterTests/ReferenceDataUpdaterTests.csproj b/tests/UnitTests/ScreeningDataServicesTests/ReferenceDataUpdaterTests/ReferenceDataUpdaterTests.csproj new file mode 100644 index 000000000..b8a01a7cb --- /dev/null +++ b/tests/UnitTests/ScreeningDataServicesTests/ReferenceDataUpdaterTests/ReferenceDataUpdaterTests.csproj @@ -0,0 +1,33 @@ + + + + net8.0 + enable + enable + false + true + + + + + + + + + + + + + + + + + + + + + + + + +