feat: update exception flag in cohort dist

application/CohortManager/compose.cohort-distribution.yaml

@@ -76,6 +76,7 @@ services:
       - AddCohortDistributionURL=http://localhost:7077/api/AddCohortDistributionData
       - ValidateCohortDistributionRecordURL=http://localhost:7084/api/ValidateCohortDistributionRecord
       - DtOsDatabaseConnectionString=Server=localhost,1433;Database=${DB_NAME};User Id=SA;Password=${PASSWORD};TrustServerCertificate=True
+      - ParticipantManagementUrl=http://localhost:7994/api/ParticipantManagementDataService
       - CohortQueueName=cohort-distribution-queue
       - CohortQueueNamePoison=cohort-distribution-queue-poison
       - IgnoreParticipantExceptions=false

application/CohortManager/compose.deps.yaml

@@ -4,7 +4,7 @@ services:
   azurite:
     container_name: azurite
     image: mcr.microsoft.com/azure-storage/azurite
-    command: azurite --silent
+    command: azurite --silent --skipApiVersionCheck
     network_mode: host
 
   azurite-setup:

application/CohortManager/src/Functions/CohortDistributionServices/CreateCohortDistribution/CreateCohortDistribution.cs

@@ -18,18 +18,21 @@ public class CreateCohortDistribution
     private readonly ICohortDistributionHelper _CohortDistributionHelper;
     private readonly IExceptionHandler _exceptionHandler;
     private readonly IAzureQueueStorageHelper _azureQueueStorageHelper;
+    private readonly IDataServiceClient<ParticipantManagement> _participantManagementClient;
 
     public CreateCohortDistribution(ILogger<CreateCohortDistribution> logger,
-           ICallFunction callFunction,
-           ICohortDistributionHelper CohortDistributionHelper,
-           IExceptionHandler exceptionHandler,
-           IAzureQueueStorageHelper azureQueueStorageHelper)
+                                    ICallFunction callFunction,
+                                    ICohortDistributionHelper CohortDistributionHelper,
+                                    IExceptionHandler exceptionHandler,
+                                    IAzureQueueStorageHelper azureQueueStorageHelper,
+                                    IDataServiceClient<ParticipantManagement> participantManagementClient)
     {
         _logger = logger;
         _callFunction = callFunction;
         _CohortDistributionHelper = CohortDistributionHelper;
         _exceptionHandler = exceptionHandler;
         _azureQueueStorageHelper = azureQueueStorageHelper;
+        _participantManagementClient = participantManagementClient;
     }
 
     [Function(nameof(CreateCohortDistribution))]
@@ -62,12 +65,12 @@ public async Task RunAsync([QueueTrigger("%CohortQueueName%", Connection = "Azur
                     return;
                 }
             }
-            
+
             // Check if participant has exceptions
             bool ignoreParticipantExceptions = Environment.GetEnvironmentVariable("IgnoreParticipantExceptions") == "true";
             _logger.LogInformation("Environment variable IgnoreParticipantExceptions is set to {IgnoreParticipantExceptions}", ignoreParticipantExceptions);
-            bool participantHasException = participantData.ExceptionFlag == 1;
 
+            bool participantHasException = participantData.ExceptionFlag == 1;
             if (participantHasException && !ignoreParticipantExceptions) // Will only run if IgnoreParticipantExceptions is false.
             {
                 await HandleErrorResponseAsync($"Unable to add to cohort distribution. As participant with ParticipantId: {participantData.ParticipantId}. Has an Exception against it",
@@ -77,13 +80,21 @@ await HandleErrorResponseAsync($"Unable to add to cohort distribution. As partic
 
             // Validation
             participantData.RecordType = basicParticipantCsvRecord.RecordType;
-            var validationRecordCreated = await _CohortDistributionHelper.ValidateCohortDistributionRecordAsync(basicParticipantCsvRecord.NhsNumber, basicParticipantCsvRecord.FileName, participantData);
+            var validationResponse = await _CohortDistributionHelper.ValidateCohortDistributionRecordAsync(basicParticipantCsvRecord.NhsNumber, basicParticipantCsvRecord.FileName, participantData);
 
-            if (validationRecordCreated && !ignoreParticipantExceptions)
+            // Update participant exception flag
+            if (validationResponse.CreatedException)
             {
-                var errorMessage = $"Validation error: A rule triggered a fatal error, preventing the cohort distribution record with participant Id {participantData.ParticipantId} from being added to the database";
+                var errorMessage = $"Participant {participantData.ParticipantId} triggered a validation rule, so will not be added to cohort distribution";
                 _logger.LogInformation(errorMessage);
                 await _exceptionHandler.CreateRecordValidationExceptionLog(participantData.NhsNumber, basicParticipantCsvRecord.FileName, errorMessage, serviceProvider, JsonSerializer.Serialize(participantData));
+
+                var participantMangement = await _participantManagementClient.GetSingle(participantData.ParticipantId);
+                participantMangement.ExceptionFlag = 1;
+
+                bool excpetionFlagUpdated = await _participantManagementClient.Update(participantMangement);
+                if (!excpetionFlagUpdated) throw new IOException("Failed to update exception flag");
+
                 return;
             }
             _logger.LogInformation("Validation has passed or exceptions are ignored, the record with participant id: {ParticipantId} will be added to the database", participantData.ParticipantId);
@@ -104,7 +115,7 @@ await HandleErrorResponseAsync($"Unable to add to cohort distribution. As partic
         }
         catch (Exception ex)
         {
-            var errorMessage = $"Failed during TransformParticipant or AddCohortDistribution Function.\nMessage: {ex.Message}\nStack Trace: {ex.StackTrace}";
+            var errorMessage = $"Create Cohort Distribution failed .\nMessage: {ex.Message}\nStack Trace: {ex.StackTrace}";
             _logger.LogError(ex, errorMessage);
             await _exceptionHandler.CreateSystemExceptionLogFromNhsNumber(ex, basicParticipantCsvRecord.NhsNumber, basicParticipantCsvRecord.FileName, "", JsonSerializer.Serialize(basicParticipantCsvRecord.ErrorRecord) ?? "N/A");
             throw;

application/CohortManager/src/Functions/CohortDistributionServices/CreateCohortDistribution/Program.cs

@@ -8,6 +8,9 @@
 
 var host = new HostBuilder()
     .ConfigureFunctionsWorkerDefaults()
+    .AddDataServicesHandler()
+        .AddDataService<ParticipantManagement>(Environment.GetEnvironmentVariable("ParticipantManagementUrl"))
+        .Build()
     .ConfigureServices(services =>
     {
         services.AddSingleton<ICallFunction, CallFunction>();

application/CohortManager/src/Functions/CohortDistributionServices/ValidateCohortDistributionRecord/ValidateCohortDistributionRecord.cs

@@ -63,11 +63,7 @@ public async Task<HttpResponseData> RunAsync([HttpTrigger(AuthorizationLevel.Ano
             var newParticipant = requestBody.CohortDistributionParticipant;
 
             var validationResult = await ValidateDataAsync(existingParticipant, newParticipant, requestBody.FileName);
-            if (validationResult.CreatedException)
-            {
-                return _createResponse.CreateHttpResponse(HttpStatusCode.Created, req, JsonSerializer.Serialize(validationResult));
-            }
-            return _createResponse.CreateHttpResponse(HttpStatusCode.OK, req);
+            return _createResponse.CreateHttpResponse(HttpStatusCode.OK, req, JsonSerializer.Serialize(validationResult));
 
         }
         catch (Exception ex)

application/CohortManager/src/Functions/Shared/Common/CohortDistributionHelper.cs

@@ -76,7 +76,7 @@ public async Task<CohortDistributionParticipant> TransformParticipantAsync(strin
         return null;
     }
 
-    public async Task<bool> ValidateCohortDistributionRecordAsync(string nhsNumber, string FileName, CohortDistributionParticipant cohortDistributionParticipant)
+    public async Task<ValidationExceptionLog> ValidateCohortDistributionRecordAsync(string nhsNumber, string FileName, CohortDistributionParticipant cohortDistributionParticipant)
     {
         var lookupValidationRequestBody = new ValidateCohortDistributionRecordBody()
         {
@@ -88,13 +88,7 @@ public async Task<bool> ValidateCohortDistributionRecordAsync(string nhsNumber,
 
         _logger.LogInformation("Called cohort validation service");
         var response = await GetResponseAsync(json, Environment.GetEnvironmentVariable("ValidateCohortDistributionRecordURL"));
-        if (!string.IsNullOrEmpty(response))
-        {
-            var responseObj = JsonSerializer.Deserialize<ValidationExceptionLog>(response);
-            return responseObj.IsFatal;
-        }
-        _logger.LogInformation("The response from the validation for cohort distribution returned null or empty");
-        return false;
+        return JsonSerializer.Deserialize<ValidationExceptionLog>(response);
     }
 
     private async Task<string> GetResponseAsync(string requestBodyJson, string functionURL)
@@ -104,7 +98,8 @@ private async Task<string> GetResponseAsync(string requestBodyJson, string funct
         {
             return "";
         }
-        if (response.StatusCode == HttpStatusCode.OK)
+
+        if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.Created)
         {
             var responseText = await _callFunction.GetResponseText(response);
             if (!string.IsNullOrEmpty(responseText))
@@ -113,7 +108,6 @@ private async Task<string> GetResponseAsync(string requestBodyJson, string funct
             }
 
         }
-
 
         return "";
     }

application/CohortManager/src/Functions/Shared/Common/Interfaces/ICohortDistributionHelper.cs

@@ -8,5 +8,5 @@ public interface ICohortDistributionHelper
     Task<CohortDistributionParticipant?> RetrieveParticipantDataAsync(CreateCohortDistributionRequestBody cohortDistributionRequestBody);
     Task<string> AllocateServiceProviderAsync(string nhsNumber, string screeningAcronym, string postCode, string errorRecord);
     Task<CohortDistributionParticipant> TransformParticipantAsync(string serviceProvider, CohortDistributionParticipant participantData);
-    Task<bool> ValidateCohortDistributionRecordAsync(string nhsNumber, string FileName, CohortDistributionParticipant cohortDistributionParticipant);
+    Task<ValidationExceptionLog> ValidateCohortDistributionRecordAsync(string nhsNumber, string FileName, CohortDistributionParticipant cohortDistributionParticipant);
 }

tests/UnitTests/CohortDistributionTests/CreateCohortDistributionTests/CreateCohortDistributionTests.cs

@@ -63,8 +63,21 @@ public CreateCohortDistributionTests()
             .Setup(x => x.RetrieveParticipantDataAsync(It.IsAny<CreateCohortDistributionRequestBody>()))
             .ReturnsAsync(_cohortDistributionParticipant);
 
+        _cohortDistributionHelper
+            .Setup(x => x.ValidateCohortDistributionRecordAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CohortDistributionParticipant>()))
+            .ReturnsAsync(new ValidationExceptionLog {CreatedException = false, IsFatal = false});
+
+        _participantManagementClientMock
+            .Setup(x => x.GetSingle(It.IsAny<string>()))
+            .ReturnsAsync(new ParticipantManagement {ExceptionFlag = 0});
+
+        _participantManagementClientMock
+            .Setup(x => x.Update(It.IsAny<ParticipantManagement>()))
+            .ReturnsAsync(true);
+
         _sut = new CreateCohortDistribution(_logger.Object, _callFunction.Object, _cohortDistributionHelper.Object,
-                                            _exceptionHandler.Object, _azureQueueStorageHelper.Object);
+                                            _exceptionHandler.Object, _azureQueueStorageHelper.Object,
+                                            _participantManagementClientMock.Object);
 
     }
 
@@ -114,7 +127,7 @@ public async Task RunAsync_RetrieveParticipantDataRequestFails_ReturnBadRequest(
         // Assert
         _logger.Verify(x => x.Log(It.Is<LogLevel>(l => l == LogLevel.Error),
             It.IsAny<EventId>(),
-            It.Is<It.IsAnyType>((v, t) => v.ToString().Contains("Failed during TransformParticipant or AddCohortDistribution Function.")),
+            It.Is<It.IsAnyType>((v, t) => v.ToString().Contains("Create Cohort Distribution failed")),
             It.IsAny<Exception>(),
             It.IsAny<Func<It.IsAnyType, Exception, string>>()),
         Times.Once);
@@ -162,7 +175,7 @@ public async Task RunAsync_AllocateServiceProviderToParticipantRequestFails_Retu
         // Assert
         _logger.Verify(x => x.Log(It.Is<LogLevel>(l => l == LogLevel.Error),
             It.IsAny<EventId>(),
-            It.Is<It.IsAnyType>((v, t) => v.ToString().Contains("Failed during TransformParticipant or AddCohortDistribution Function.")),
+            It.Is<It.IsAnyType>((v, t) => v.ToString().Contains("Create Cohort Distribution failed")),
             It.IsAny<Exception>(),
             It.IsAny<Func<It.IsAnyType, Exception, string>>()),
         Times.Once);
@@ -175,9 +188,6 @@ public async Task RunAsync_AllocateServiceProviderToParticipantRequestFails_Retu
     public async Task RunAsync_TransformDataServiceRequestFails_ReturnEarly()
     {
         // Arrange
-        _cohortDistributionHelper
-            .Setup(x => x.ValidateCohortDistributionRecordAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CohortDistributionParticipant>()))
-            .ReturnsAsync(false);
         _cohortDistributionHelper
             .Setup(x => x.AllocateServiceProviderAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
             .ReturnsAsync("");
@@ -203,9 +213,6 @@ public async Task RunAsync_AddCohortDistributionRequestFails_LogError()
         Exception caughtException = null;
         _request = _setupRequest.Setup(JsonSerializer.Serialize(_requestBody));
 
-        _cohortDistributionHelper
-            .Setup(x => x.ValidateCohortDistributionRecordAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CohortDistributionParticipant>()))
-            .ReturnsAsync(false);
         _cohortDistributionHelper
             .Setup(x => x.AllocateServiceProviderAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
             .ReturnsAsync("");
@@ -233,7 +240,7 @@ public async Task RunAsync_AddCohortDistributionRequestFails_LogError()
         // Assert
         _logger.Verify(x => x.Log(It.Is<LogLevel>(l => l == LogLevel.Error),
             It.IsAny<EventId>(),
-            It.Is<It.IsAnyType>((v, t) => v.ToString().Contains("Failed during TransformParticipant or AddCohortDistribution Function.")),
+            It.Is<It.IsAnyType>((v, t) => v.ToString().Contains("Create Cohort Distribution failed")),
             It.IsAny<Exception>(),
             It.IsAny<Func<It.IsAnyType, Exception, string>>()),
         Times.Once);
@@ -246,20 +253,13 @@ public async Task RunAsync_AddCohortDistributionRequestFails_LogError()
     public async Task RunAsync_AllSuccessfulRequests_AddsToCOhort()
     {
         // Arrange
-        _cohortDistributionHelper
-            .Setup(x => x.ValidateCohortDistributionRecordAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CohortDistributionParticipant>()))
-            .ReturnsAsync(false);
         _cohortDistributionHelper
             .Setup(x => x.AllocateServiceProviderAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
             .ReturnsAsync("");
         _cohortDistributionHelper
             .Setup(x => x.TransformParticipantAsync(It.IsAny<string>(), It.IsAny<CohortDistributionParticipant>()))
             .ReturnsAsync(new CohortDistributionParticipant());
 
-        _participantManagementClientMock
-            .Setup(c => c.GetSingleByFilter(It.IsAny<Expression<Func<ParticipantManagement, bool>>>()))
-            .ReturnsAsync(new ParticipantManagement { ExceptionFlag = 0 });
-
         _sendToCohortDistributionResponse
             .Setup(x => x.StatusCode)
             .Returns(HttpStatusCode.OK);
@@ -272,7 +272,9 @@ public async Task RunAsync_AllSuccessfulRequests_AddsToCOhort()
             .ReturnsAsync(new ParticipantManagement() { ExceptionFlag = 0 });
 
         var response = MockHelpers.CreateMockHttpResponseData(HttpStatusCode.OK);
-        _callFunction.Setup(call => call.SendPost(It.IsAny<string>(), It.IsAny<string>())).Returns(Task.FromResult(response));
+        _callFunction
+            .Setup(call => call.SendPost(It.IsAny<string>(), It.IsAny<string>()))
+            .ReturnsAsync(response);
 
         // Act
         await _sut.RunAsync(_requestBody);
@@ -315,12 +317,12 @@ public async Task RunAsync_ParticipantHasException_CreatesSystemExceptionLog()
     }
 
     [TestMethod]
-    public async Task RunAsync_ValidationRequestFailed_CreateValidationExceptionLog()
+    public async Task RunAsync_ValidationRuleTriggered_UpdateExceptionFlagAndLog()
     {
         // Arrange
         _cohortDistributionHelper
             .Setup(x => x.ValidateCohortDistributionRecordAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CohortDistributionParticipant>()))
-            .ReturnsAsync(true);
+            .ReturnsAsync(new ValidationExceptionLog { CreatedException = true, IsFatal = false });
         _cohortDistributionHelper
             .Setup(x => x.AllocateServiceProviderAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
             .ReturnsAsync("");
@@ -345,10 +347,13 @@ public async Task RunAsync_ValidationRequestFailed_CreateValidationExceptionLog(
             It.IsAny<string>(),
             It.IsAny<string>()
         ), Times.Once);
+
+        _participantManagementClientMock
+            .Verify(x => x.Update(It.IsAny<ParticipantManagement>()), Times.Once);
     }
 
     [TestMethod]
-    public async Task RunAsync_ParticipantHasExceptionAndEnvironmentVariableFalse_CreateSystemExceptionLog()
+    public async Task RunAsync_ParticipantHasExceptionAndEnvironmentVariableFalse_CreateExceptionAndReturn()
     {
         // Arrange
         Environment.SetEnvironmentVariable("IgnoreParticipantExceptions", "false");
@@ -370,5 +375,12 @@ public async Task RunAsync_ParticipantHasExceptionAndEnvironmentVariableFalse_Cr
          It.Is<It.IsAnyType>((v, t) => v.ToString().Contains($"Unable to add to cohort distribution")),
          It.IsAny<Exception>(),
          It.IsAny<Func<It.IsAnyType, Exception, string>>()));
+
+        _cohortDistributionHelper
+            .Verify(x => x.ValidateCohortDistributionRecordAsync(
+                It.IsAny<string>(),
+                It.IsAny<string>(),
+                It.IsAny<CohortDistributionParticipant>()),
+            Times.Never);
     }
 }

tests/UnitTests/CohortDistributionTests/ValidateCohortDistributionRecordTests/ValidateCohortDistributionRecordTests.cs

@@ -125,7 +125,7 @@ public async Task Run_RequestBodyInvalid_BadRequest()
     }
 
     [TestMethod]
-    public async Task Run_LookupValidationCreatesValidationError_Created()
+    public async Task Run_LookupValidationCreatesValidationError_ReturnOk()
     {
         // Arrange
         SetUpRequestBody(JsonSerializer.Serialize(_requestBody));
@@ -145,7 +145,7 @@ public async Task Run_LookupValidationCreatesValidationError_Created()
         var result = await _function.RunAsync(_request.Object);
 
         // Assert
-        Assert.AreEqual(HttpStatusCode.Created, result.StatusCode);
+        Assert.AreEqual(HttpStatusCode.OK, result.StatusCode);
     }
 
     [TestMethod]