Merge branch 'main' into feat/DTOSS-8367-get-acess-token-from-singedJWT

Author: SamRobinson75684

.azuredevops/pipelines/acr-image-promote-dev-to-nft.yaml

@@ -41,7 +41,7 @@ stages:
   jobs:
   - job: re_tag
     pool:
-      name: private-pool-prod-uks
+      name: private-pool-dev-uks
     displayName: Update/copy Docker images with new tag
     variables:
       - group: NFT_image_pipelines

.azuredevops/pipelines/acr-image-promote-nft-to-integration.yaml

@@ -41,7 +41,7 @@ stages:
   jobs:
   - job: re_tag
     pool:
-      name: private-pool-prod-uks
+      name: private-pool-dev-uks
     displayName: Update/copy Docker images with new tag
     variables:
       - group: INT_image_pipelines

application/CohortManager/compose.cohort-distribution.yaml

@@ -21,6 +21,7 @@ services:
       - LookupValidationURL=http://lookup-validation:7075/api/LookupValidation
       - StaticValidationURL=http://static-validation:7074/api/StaticValidation
       - ParticipantDemographicDataServiceURL=http://participant-demographic-data-service:7993/api/ParticipantDemographicDataService
+      - RemoveOldValidationRecordUrl=http://remove-validation-exception-data:7085/api/RemoveValidationExceptionData
       - IgnoreParticipantExceptions=false
       - IsExtractedToBSSelect=true
 

application/CohortManager/compose.core.yaml

@@ -156,6 +156,7 @@ services:
     build:
       context: ./src/Functions/
       dockerfile: ExceptionHandling/UpdateException/Dockerfile
+    profiles: [non-essential]
     environment:
       - ASPNETCORE_URLS=http://*:7070
       - DtOsDatabaseConnectionString=Server=db,1433;Database=${DB_NAME};User Id=SA;Password=${PASSWORD};TrustServerCertificate=True
@@ -187,11 +188,8 @@ services:
       dockerfile: ScreeningValidationService/StaticValidation/Dockerfile
     environment:
       - ASPNETCORE_URLS=http://*:7074
-      - CreateValidationExceptionURL=http://create-exception:7070/api/CreateValidationException
-      - BlobContainerName=rules
-      - AzureWebJobsStorage=${AZURITE_CONNECTION_STRING}
       - ExceptionFunctionURL=http://create-exception:7070/api/CreateException
-      - RemoveOldValidationRecord=http://remove-validation-exception-data:7085/api/RemoveValidationExceptionData
+      - AzureWebJobsStorage=${AZURITE_CONNECTION_STRING}
 
   lookup-validation:
     container_name: lookup-validation

application/CohortManager/src/Functions/CohortDistributionServices/DistributeParticipant/DistributeParticipantConfig.cs

@@ -8,7 +8,6 @@ public class DistributeParticipantConfig
     public string CohortDistributionTopic { get; set; }
     [Required]
     public string DistributeParticipantSubscription { get; set; }
-    public bool IgnoreParticipantExceptions { get; set; } = false;
     [Required]
     public string LookupValidationURL { get; set; }
     [Required]
@@ -21,5 +20,8 @@ public class DistributeParticipantConfig
     public string CohortDistributionDataServiceUrl { get; set; }
     [Required]
     public string ParticipantDemographicDataServiceUrl { get; set; }
+    [Required]
+    public string RemoveOldValidationRecordUrl { get; set; }
     public bool IsExtractedToBSSelect { get; set; } = false;
+    public bool IgnoreParticipantExceptions { get; set; } = false;
 }

application/CohortManager/src/Functions/CohortDistributionServices/DistributeParticipant/ValidateParticipant.cs

@@ -10,6 +10,8 @@ namespace NHS.CohortManager.CohortDistributionServices;
 using Model.Enums;
 using DataServices.Client;
 using Microsoft.Extensions.Options;
+using Newtonsoft.Json.Linq;
+using RulesEngine.Models;
 
 public class ValidateParticipant
 {
@@ -36,7 +38,7 @@ public ValidateParticipant(IDataServiceClient<CohortDistribution> cohortDistribu
     }
 
     /// <summary>
-    /// Orchestration for the validation and trasnformation process
+    /// Orchestrator for the validation and trasnformation process
     /// </summary>
     /// <param name="context">Context containing a validation record</param>
     /// <returns>The transformed <see cref="CohortDistributionParticipant"/>, or null if the validation/ transformation failed</returns>
@@ -52,21 +54,28 @@ public ValidateParticipant(IDataServiceClient<CohortDistribution> cohortDistribu
             var previousRecord = await context.CallActivityAsync<CohortDistributionParticipant>(nameof(GetCohortDistributionRecord), validationRecord.Participant.ParticipantId);
             validationRecord.PreviousParticipantRecord = previousRecord;
 
+            // Remove Previous Validation Errors from DB
+            await context.CallActivityAsync( nameof(RemoveOldValidationExceptions), new OldExceptionRecord()
+            {
+                NhsNumber = validationRecord.Participant.NhsNumber,
+                ScreeningName = validationRecord.Participant.ScreeningName
+            });
+            
             // Lookup & Static Validation
             _logger.LogInformation("Validating participant");
-            ValidationExceptionLog[] validationResults = await Task.WhenAll(
-                context.CallActivityAsync<ValidationExceptionLog>(nameof(StaticValidation), validationRecord),
-                context.CallActivityAsync<ValidationExceptionLog>(nameof(LookupValidation), validationRecord)
-            );
+            var staticTask = context.CallActivityAsync<List<ValidationRuleResult>>(nameof(StaticValidation), validationRecord);
+            var lookupTask = context.CallActivityAsync<List<ValidationRuleResult>>(nameof(LookupValidation), validationRecord);
+
+            await Task.WhenAll(staticTask, lookupTask);
 
-            var validationResult = new ValidationExceptionLog(validationResults[0], validationResults[1]);
+            var validationResult = staticTask.Result.Concat(lookupTask.Result).ToList();
 
             // Update exception flag and return
-            if (validationResult.CreatedException)
+            if (validationResult.Any())
             {
-                _logger.LogError("Participant {ParticipantId} triggered a validation rule", validationRecord.Participant.ParticipantId);
+                _logger.LogWarning("Participant {ParticipantId} triggered a validation rule", validationRecord.Participant.ParticipantId);
 
-                await context.CallActivityAsync(nameof(UpdateExceptionFlag), validationRecord.Participant.ParticipantId);
+                await context.CallActivityAsync(nameof(HandleValidationExceptions), new ValidationExceptionRecord(validationRecord, validationResult));
 
                 if (!_config.IgnoreParticipantExceptions)
                 {
@@ -115,13 +124,23 @@ public async Task<CohortDistributionParticipant> GetCohortDistributionRecord([Ac
         }
     }
 
+    /// <summary>
+    /// Calls the remove old validation exception function
+    /// </summary>
+    [Function(nameof(RemoveOldValidationExceptions))]
+    public async Task RemoveOldValidationExceptions([ActivityTrigger] OldExceptionRecord request)
+    {
+        string json = JsonSerializer.Serialize(request);
+        await _httpClient.SendPost(_config.RemoveOldValidationRecordUrl, json);
+    }
+
     /// <summary>
     /// Calls static validation
     /// </summary>
     /// <param name="validationRecord"></param>
     /// <returns>A <see cref="ValidationExceptionLog"/> representing if the participant has triggered a rule</returns>
     [Function(nameof(StaticValidation))]
-    public async Task<ValidationExceptionLog> StaticValidation([ActivityTrigger] ValidationRecord validationRecord)
+    public async Task<List<ValidationRuleResult>?> StaticValidation([ActivityTrigger] ValidationRecord validationRecord)
     {
         var request = new ParticipantCsvRecord
         {
@@ -133,9 +152,14 @@ public async Task<ValidationExceptionLog> StaticValidation([ActivityTrigger] Val
 
         var response = await _httpClient.SendPost(_config.StaticValidationURL, json);
         response.EnsureSuccessStatusCode();
+
+        if (response.StatusCode == HttpStatusCode.NoContent)
+        {
+            return new List<ValidationRuleResult>();
+        }
         string body = await _httpClient.GetResponseText(response);
 
-        var exceptionLog = JsonSerializer.Deserialize<ValidationExceptionLog>(body);
+        var exceptionLog = JsonSerializer.Deserialize<List<ValidationRuleResult>>(body);
 
         return exceptionLog;
     }
@@ -146,7 +170,7 @@ public async Task<ValidationExceptionLog> StaticValidation([ActivityTrigger] Val
     /// <param name="validationRecord"></param>
     /// <returns>A <see cref="ValidationExceptionLog"/> representing if the participant has triggered a rule</returns>
     [Function(nameof(LookupValidation))]
-    public async Task<ValidationExceptionLog> LookupValidation([ActivityTrigger] ValidationRecord validationRecord)
+    public async Task<List<ValidationRuleResult>> LookupValidation([ActivityTrigger] ValidationRecord validationRecord)
     {
         var request = new LookupValidationRequestBody
         {
@@ -159,9 +183,14 @@ public async Task<ValidationExceptionLog> LookupValidation([ActivityTrigger] Val
 
         var response = await _httpClient.SendPost(_config.LookupValidationURL, json);
         response.EnsureSuccessStatusCode();
+        if (response.StatusCode == HttpStatusCode.NoContent)
+        {
+            return new List<ValidationRuleResult>();
+        }
+
         string body = await _httpClient.GetResponseText(response);
 
-        var exceptionLog = JsonSerializer.Deserialize<ValidationExceptionLog>(body);
+        var exceptionLog = JsonSerializer.Deserialize<List<ValidationRuleResult>>(body);
         return exceptionLog;
     }
 
@@ -194,20 +223,30 @@ public async Task<ValidationExceptionLog> LookupValidation([ActivityTrigger] Val
     }
 
     /// <summary>
-    /// Updates the exception flag in participant management
+    /// Creates a validation excpetion and updates the exception flag
+    /// in participant management
     /// </summary>
-    /// <param name="participantId"></param>
     /// <exception cref="IOException">Thrown if the update fails</exception>
-    [Function(nameof(UpdateExceptionFlag))]
-    public async Task UpdateExceptionFlag([ActivityTrigger] string participantId)
+    [Function(nameof(HandleValidationExceptions))]
+    public async Task HandleValidationExceptions([ActivityTrigger] ValidationExceptionRecord validationExceptionRecord)
     {
-        var participantManagement = await _participantManagementClient.GetSingle(participantId);
+        // Send exceptions to DB
+        ParticipantCsvRecord participantRecord = new()
+        {
+            Participant = new Participant(validationExceptionRecord.ValidationRecord.Participant),
+            FileName = validationExceptionRecord.ValidationRecord.FileName
+        };
+
+        var exceptionCreated = await _exceptionHandler.CreateValidationExceptionLog(validationExceptionRecord.ValidationExceptions, participantRecord);
+
+        var participantManagement = await _participantManagementClient.GetSingle(participantRecord.Participant.ParticipantId);
         participantManagement.ExceptionFlag = 1;
 
         var exceptionFlagUpdated = await _participantManagementClient.Update(participantManagement);
         if (!exceptionFlagUpdated)
         {
             throw new IOException("Failed to update exception flag");
         }
+        _logger.LogInformation("Created validation exception and set exception flag to 1 for participant {ParticipantId}", participantRecord.Participant.ParticipantId);
     }
 }

application/CohortManager/src/Functions/CohortDistributionServices/DistributeParticipant/ValidationExceptionRecord.cs

@@ -0,0 +1,19 @@
+namespace NHS.CohortManager.CohortDistributionServices;
+
+using Model;
+
+/// <summary>
+/// Wrapper needed for handling validation exceptions
+/// (you cannot pass in mutiple params to durable function activites)
+/// </summary>
+public class ValidationExceptionRecord
+{
+    public ValidationRecord ValidationRecord { get; set; }
+    public List<ValidationRuleResult> ValidationExceptions { get; set; }
+
+    public ValidationExceptionRecord(ValidationRecord validationRecord, List<ValidationRuleResult> validationExceptions)
+    {
+        ValidationRecord = validationRecord;
+        ValidationExceptions = validationExceptions;
+    }
+}

application/CohortManager/src/Functions/ScreeningValidationService/LookupValidation/LookupValidation.cs

@@ -95,25 +95,15 @@ public async Task<HttpResponseData> RunAsync([HttpTrigger(AuthorizationLevel.Ano
             }
 
             // Validation rules are logically reversed
-            var validationErrors = resultList.Where(x => !x.IsSuccess);
+            var validationErrors = resultList.Where(x => !x.IsSuccess).Select(x => new ValidationRuleResult(x));
 
             if (validationErrors.Any())
             {
-                _logger.LogInformation("There was an error in the Validation Rules");
-                var participantCsvRecord = new ParticipantCsvRecord()
-                {
-                    Participant = newParticipant,
-                    FileName = requestBody.FileName
-                };
-                var exceptionCreated = await _handleException.CreateValidationExceptionLog(validationErrors, participantCsvRecord);
-                return _createResponse.CreateHttpResponse(HttpStatusCode.Created, req, JsonSerializer.Serialize(exceptionCreated));
+                string errors = JsonSerializer.Serialize(validationErrors);
+                return _createResponse.CreateHttpResponse(HttpStatusCode.OK, req, errors);
             }
 
-            return _createResponse.CreateHttpResponse(HttpStatusCode.OK, req, JsonSerializer.Serialize(new ValidationExceptionLog()
-            {
-                IsFatal = false,
-                CreatedException = false
-            }));
+            return _createResponse.CreateHttpResponse(HttpStatusCode.NoContent, req);
         }
         catch (Exception ex)
         {

application/CohortManager/src/Functions/ScreeningValidationService/StaticValidation/Program.cs

@@ -3,10 +3,8 @@
 using HealthChecks.Extensions;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
-using NHS.Screening.StaticValidation;
 
 var host = new HostBuilder()
-    .AddConfiguration<StaticValidationConfig>(out StaticValidationConfig config)
     .ConfigureFunctionsWorkerDefaults()
     .ConfigureServices(services =>
     {

application/CohortManager/src/Functions/ScreeningValidationService/StaticValidation/StaticValidation.cs

@@ -12,32 +12,22 @@ namespace NHS.CohortManager.ScreeningValidationService;
 using Microsoft.Extensions.Options;
 using Model;
 using Model.Enums;
-using NHS.Screening.StaticValidation;
 using RulesEngine.Models;
 
 public class StaticValidation
 {
     private readonly ILogger<StaticValidation> _logger;
     private readonly ICreateResponse _createResponse;
-    private readonly IExceptionHandler _handleException;
     private readonly IReadRules _readRules;
-    private readonly IHttpClientFunction _httpClientFunction;
-    private readonly StaticValidationConfig _config;
 
     public StaticValidation(
         ILogger<StaticValidation> logger,
-        IExceptionHandler handleException,
         ICreateResponse createResponse,
-        IReadRules readRules,
-        IHttpClientFunction httpClientFunction,
-        IOptions<StaticValidationConfig> staticValidationConfig)
+        IReadRules readRules)
     {
         _logger = logger;
-        _handleException = handleException;
         _createResponse = createResponse;
         _readRules = readRules;
-        _httpClientFunction = httpClientFunction;
-        _config = staticValidationConfig.Value;
     }
 
     // TODO: refactor to accept a cohort distribution participant
@@ -69,7 +59,7 @@ public async Task<HttpResponseData> RunAsync([HttpTrigger(AuthorizationLevel.Ano
             var re = new RulesEngine.RulesEngine(rules, reSettings);
 
             var ruleParameters = new[] {
-                new RuleParameter("participant", participantCsvRecord.Participant),
+                new RuleParameter("participant", participantCsvRecord.Participant)
             };
             var resultList = await re.ExecuteAllRulesAsync("Common", ruleParameters);
 
@@ -80,35 +70,20 @@ public async Task<HttpResponseData> RunAsync([HttpTrigger(AuthorizationLevel.Ano
                 resultList.AddRange(ActionResults);
             }
 
-            var validationErrors = resultList.Where(x => !x.IsSuccess);
+            var validationErrors = resultList.Where(x => !x.IsSuccess).Select(x => new ValidationRuleResult(x));
 
-            await RemoveOldValidationRecord(participantCsvRecord.Participant.NhsNumber, participantCsvRecord.Participant.ScreeningName);
             if (validationErrors.Any())
             {
-                var createExceptionLogResponse = await _handleException.CreateValidationExceptionLog(validationErrors, participantCsvRecord);
-                return _createResponse.CreateHttpResponse(HttpStatusCode.Created, req, JsonSerializer.Serialize(createExceptionLogResponse));
+                string errors = JsonSerializer.Serialize(validationErrors);
+                return _createResponse.CreateHttpResponse(HttpStatusCode.OK, req, errors);
             }
 
-            return _createResponse.CreateHttpResponse(HttpStatusCode.OK, req, JsonSerializer.Serialize(new ValidationExceptionLog()
-            {
-                IsFatal = false,
-                CreatedException = false
-            }));
+            return _createResponse.CreateHttpResponse(HttpStatusCode.NoContent, req);
         }
         catch (Exception ex)
         {
             _logger.LogError(ex, ex.Message);
             return _createResponse.CreateHttpResponse(HttpStatusCode.InternalServerError, req);
         }
     }
-
-    private async Task RemoveOldValidationRecord(string nhsNumber, string screeningName)
-    {
-        var OldExceptionRecordJson = JsonSerializer.Serialize(new OldExceptionRecord()
-        {
-            NhsNumber = nhsNumber,
-            ScreeningName = screeningName
-        });
-        await _httpClientFunction.SendPost(_config.RemoveOldValidationRecord, OldExceptionRecordJson);
-    }
 }