Merge branch 'main' into feat/dtoss-5226-production-environment-pipelines-3

rfk-nc · web-flow · commit 5081996db1fb · 2025-08-07T16:43:51.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -63,6 +63,7 @@ appsettings.json
 application/CohortManager/SampleData/
 *.csv
 *.parquet
+*.xml
 !tests/UnitTests/CaasIntegrationTests/receiveCaasFileTest/*.parquet
 
 # csv to sql statement scripts
diff --git a/application/CohortManager/compose.core.yaml b/application/CohortManager/compose.core.yaml
@@ -58,7 +58,7 @@ services:
       - ExceptionFunctionURL=http://create-exception:7070/api/CreateException
       - RetrievePdsDemographicURL=http://retrieve-pds-demographic:8082/api/RetrievePDSDemographic
       - UnsubscribeNemsSubscriptionUrl=http://manage-nems-subscription:9081/api/Unsubscribe
-      - ParticipantDemographicDataServiceURL=http://participant-demographic-data-service:7993/api/ParticipantDemographicDataService
+      - DemographicDataServiceURL=http://participant-demographic-data-service:7993/api/ParticipantDemographicDataService
       - ServiceBusConnectionString_client_internal=Endpoint=sb://service-bus;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;
       - ParticipantManagementTopic=participant-management-topic
 
diff --git a/application/CohortManager/src/Functions/DemographicServices/ManageNemsSubscription/ManageNemsSubscription.cs b/application/CohortManager/src/Functions/DemographicServices/ManageNemsSubscription/ManageNemsSubscription.cs
@@ -49,13 +49,13 @@ public async Task<HttpResponseData> Subscribe([HttpTrigger(AuthorizationLevel.An
 
             string? nhsNumber = req.Query["nhsNumber"];
 
-            if (!ValidationHelper.ValidateNHSNumber(nhsNumber))
+            if (!ValidationHelper.ValidateNHSNumber(nhsNumber!))
             {
                 _logger.LogError("NHS number is required and must be valid format");
                 return await _createResponse.CreateHttpResponseWithBodyAsync(HttpStatusCode.BadRequest, req, "NHS number is required and must be valid format.");
             }
 
-            var result = await _subscriptionManager.CreateAndSendSubscriptionAsync(nhsNumber);
+            var result = await _subscriptionManager.CreateAndSendSubscriptionAsync(nhsNumber!);
 
             if (!result.Success)
             {
diff --git a/application/CohortManager/src/Functions/DemographicServices/RetrievePDSDemographic/RetrievePDSDemographic.cs b/application/CohortManager/src/Functions/DemographicServices/RetrievePDSDemographic/RetrievePDSDemographic.cs
@@ -12,6 +12,8 @@ namespace NHS.CohortManager.DemographicServices;
 using Microsoft.Extensions.Options;
 using System.Threading.Tasks;
 using Model;
+using System.Net.Http.Json;
+using Hl7.Fhir.Support;
 
 public class RetrievePdsDemographic
 {
@@ -71,12 +73,8 @@ public async Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymou
 
             if (response.StatusCode == HttpStatusCode.NotFound)
             {
-                var demographicRecordDeletedFromDatabase = await DeleteDemographicRecord(nhsNumber);
-                if (!demographicRecordDeletedFromDatabase)
-                {
-                    return _createResponse.CreateHttpResponse(HttpStatusCode.NotFound, req, "could not delete record from database. See logs for more details.");
-                }
-                return _createResponse.CreateHttpResponse(HttpStatusCode.NotFound, req, "Record not found in PDS and successfully removed from cohort manager database.");
+                var pdsErrorResponse = await response.Content.ReadAsStringAsync();
+                return _createResponse.CreateHttpResponse(HttpStatusCode.NotFound, req, pdsErrorResponse);
             }
 
             response.EnsureSuccessStatusCode();
@@ -129,36 +127,4 @@ private async Task<bool> UpsertDemographicRecordFromPDS(ParticipantDemographic p
         _logger.LogError("Failed to update Participant Demographic.");
         return false;
     }
-
-    private async Task<bool> DeleteDemographicRecord(string nhsNumber)
-    {
-        var nhsNumberParsed = long.TryParse(nhsNumber, out long parsedNhsNumber);
-        if (!nhsNumberParsed)
-        {
-            _logger.LogError("could not parse nhs number when trying to get record for deletion.");
-            return false;
-        }
-        var oldParticipantDemographic = await _participantDemographicClient.GetSingleByFilter(i => i.NhsNumber == parsedNhsNumber);
-
-        if (oldParticipantDemographic == null)
-        {
-
-            _logger.LogWarning("Failed to delete Participant Demographic as record did not exist in database.");
-            return false;
-        }
-
-        _logger.LogInformation("Participant Demographic record found, attempting to delete  Participant Demographic.");
-        bool updateSuccess = await _participantDemographicClient.Delete(oldParticipantDemographic.ParticipantId.ToString());
-
-        if (updateSuccess)
-        {
-            _logger.LogInformation("Successfully deleted Participant Demographic.");
-            return true;
-        }
-
-        _logger.LogError("Failed to delete Participant Demographic.");
-        return false;
-    }
-
-
 }
diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/PdsErroResponseClasses/PDSCoding.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/PdsErroResponseClasses/PDSCoding.cs
@@ -0,0 +1,9 @@
+namespace NHS.Screening.ProcessNemsUpdate;
+
+public class PdsCoding
+{
+    public string? code { get; set; }
+    public string? display { get; set; }
+    public string? system { get; set; }
+    public string? version { get; set; }
+}
diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/PdsErroResponseClasses/PDSErrorRespone.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/PdsErroResponseClasses/PDSErrorRespone.cs
@@ -0,0 +1,7 @@
+namespace NHS.Screening.ProcessNemsUpdate;
+
+public class PdsErrorResponse
+{
+    public List<PdsIssue>? issue { get; set; }
+    public string? resourceType { get; set; }
+}
diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/PdsErroResponseClasses/PDSIssue.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/PdsErroResponseClasses/PDSIssue.cs
@@ -0,0 +1,8 @@
+namespace NHS.Screening.ProcessNemsUpdate;
+
+public class PdsIssue
+{
+    public string? code { get; set; }
+    public PdsErrorDetails? details { get; set; }
+    public string? severity { get; set; }
+}
diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/PdsErroResponseClasses/PdsErrorDetails.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/PdsErroResponseClasses/PdsErrorDetails.cs
@@ -0,0 +1,7 @@
+namespace NHS.Screening.ProcessNemsUpdate;
+
+public class PdsErrorDetails
+{
+    public List<PdsCoding>? coding { get; set; }
+}
+
diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdate.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdate.cs
@@ -2,6 +2,7 @@
 
 using System.Collections.Concurrent;
 using System.Collections.Specialized;
+using System.Net.Http.Json;
 using System.Text;
 using System.Text.Json;
 using Common;
@@ -11,6 +12,7 @@
 using Microsoft.Extensions.Options;
 using Model;
 using DataServices.Client;
+using FluentValidation.Validators;
 
 public class ProcessNemsUpdate
 {
@@ -61,61 +63,35 @@ public async Task Run([BlobTrigger("nems-updates/{name}", Connection = "nemsmesh
     {
         try
         {
-            string? nhsNumber = await GetNhsNumberFromFile(blobStream, name);
+            var nhsNumber = await GetNhsNumberFromFile(blobStream, name);
 
-            if (nhsNumber == null)
-            {
-                _logger.LogInformation("There is no NHS number, unable to continue.");
-                return;
-            }
-
-            if (!ValidationHelper.ValidateNHSNumber(nhsNumber))
+            if (!ValidationHelper.ValidateNHSNumber(nhsNumber!))
             {
+                _logger.LogError("There was a problem parsing the NHS number from blob store in the ProcessNemsUpdate function");
                 throw new InvalidDataException("Invalid NHS Number");
             }
-            nhsNumberLong = long.Parse(nhsNumber);
-
-            string? pdsRecord = await RetrievePdsRecord(nhsNumber);
+            nhsNumberLong = long.Parse(nhsNumber!);
 
-            if (pdsRecord == null)
+            var pdsResponse = await RetrievePdsRecord(nhsNumber!);
+            if (pdsResponse!.StatusCode == System.Net.HttpStatusCode.NotFound)
             {
-                _logger.LogInformation("There is no PDS record, unable to continue.");
+                await ProcessPdsResponse(pdsResponse, nhsNumber!);
+                // we can stop processing here as we know that not found means the participant ether needed an update or they were actually not found
                 return;
             }
 
-            var retrievedPdsRecord = JsonSerializer.Deserialize<PdsDemographic>(pdsRecord);
+            pdsResponse.EnsureSuccessStatusCode();
+
+            var retrievedPdsRecord = await pdsResponse.Content.ReadFromJsonAsync<PdsDemographic>();
 
             if (retrievedPdsRecord?.NhsNumber == nhsNumber)
             {
                 _logger.LogInformation("NHS numbers match, processing the retrieved PDS record.");
-                await ProcessRecord(retrievedPdsRecord);
+                await ProcessRecord(new Participant(retrievedPdsRecord!));
             }
-
             else
             {
-                var supersededRecord = new PdsDemographic()
-                {
-                    NhsNumber = nhsNumber,
-                    SupersededByNhsNumber = retrievedPdsRecord?.NhsNumber,
-                    PrimaryCareProvider = null,
-                    ReasonForRemoval = "ORR",
-                    RemovalEffectiveFromDate = DateTime.UtcNow.Date.ToString("yyyyMMdd")
-                };
-
-                _logger.LogInformation("NHS numbers do not match, processing the superseded record.");
-                await ProcessRecord(supersededRecord);
-
-                /*information exception raised for RuleId 60 and Rule name 'SupersededNhsNumber'*/
-                var ruleId = 60;  // Rule 60 is for Superseded rule
-                var ruleName = "SupersededNhsNumber"; //Superseded rule name
-                await _exceptionHandler.CreateTransformExecutedExceptions(supersededRecord.ToCohortDistributionParticipant(), ruleName, ruleId);
-
-                var unsubscribedFromNems = await UnsubscribeNems(nhsNumber);
-
-                if (unsubscribedFromNems)
-                {
-                    _logger.LogInformation("Successfully unsubscribed from NEMS.");
-                }
+                await UnsubscribeFromNems(nhsNumber!, retrievedPdsRecord!);
             }
 
         }
@@ -126,6 +102,56 @@ public async Task Run([BlobTrigger("nems-updates/{name}", Connection = "nemsmesh
 
     }
 
+    private async Task ProcessPdsResponse(HttpResponseMessage pdsResponse, string nhsNumber)
+    {
+        var errorResponse = await pdsResponse!.Content.ReadFromJsonAsync<PdsErrorResponse>();
+        // we now create a record as an update record and send to the manage participant function. Reason for removal for date should be today and the reason for remove of ORR
+        if (errorResponse!.issue!.FirstOrDefault()!.details!.coding!.FirstOrDefault()!.code == "INVALIDATED_RESOURCE")
+        {
+            var pdsDemographic = new PdsDemographic()
+            {
+                NhsNumber = nhsNumber,
+                PrimaryCareProvider = null,
+                ReasonForRemoval = "ORR",
+                RemovalEffectiveFromDate = DateTime.UtcNow.Date.ToString("yyyyMMdd")
+            };
+            var participant = new Participant(pdsDemographic);
+            participant.RecordType = Actions.Removed;
+            //sends record for an update
+            await ProcessRecord(participant);
+            return;
+        }
+        _logger.LogError("the PDS function has returned a 404 error. function now stopping processing");
+
+    }
+
+    private async Task UnsubscribeFromNems(string nhsNumber, PdsDemographic retrievedPdsRecord)
+    {
+        var supersededRecord = new PdsDemographic()
+        {
+            NhsNumber = nhsNumber,
+            SupersededByNhsNumber = retrievedPdsRecord?.NhsNumber,
+            PrimaryCareProvider = null,
+            ReasonForRemoval = "ORR",
+            RemovalEffectiveFromDate = DateTime.UtcNow.Date.ToString("yyyyMMdd")
+        };
+
+        _logger.LogInformation("NHS numbers do not match, processing the superseded record.");
+        await ProcessRecord(new Participant(supersededRecord));
+
+        /*information exception raised for RuleId 60 and Rule name 'SupersededNhsNumber'*/
+        var ruleId = 60;  // Rule 60 is for Superseded rule
+        var ruleName = "SupersededNhsNumber"; //Superseded rule name
+        await _exceptionHandler.CreateTransformExecutedExceptions(supersededRecord.ToCohortDistributionParticipant(), ruleName, ruleId);
+
+        var unsubscribedFromNems = await UnsubscribeNems(nhsNumber);
+
+        if (unsubscribedFromNems)
+        {
+            _logger.LogInformation("Successfully unsubscribed from NEMS.");
+        }
+    }
+
     private async Task<string?> GetNhsNumberFromFile(Stream blobStream, string name)
     {
         try
@@ -139,7 +165,7 @@ public async Task Run([BlobTrigger("nems-updates/{name}", Connection = "nemsmesh
             }
 
             // Determine format based on file extension and call appropriate parser
-            if (name.EndsWith(".xml", StringComparison.OrdinalIgnoreCase))
+            if (name.EndsWith(".xml", System.StringComparison.OrdinalIgnoreCase))
             {
                 return _fhirPatientDemographicMapper.ParseFhirXmlNhsNumber(blobJson);
             }
@@ -156,41 +182,32 @@ public async Task Run([BlobTrigger("nems-updates/{name}", Connection = "nemsmesh
         }
     }
 
-    private async Task<string?> RetrievePdsRecord(string nhsNumber)
+    private async Task<HttpResponseMessage> RetrievePdsRecord(string nhsNumber)
     {
-        try
+        var queryParams = new Dictionary<string, string>()
         {
-            var queryParams = new Dictionary<string, string>()
-            {
-                {"nhsNumber", nhsNumber }
-            };
+            {"nhsNumber", nhsNumber }
+        };
 
-            return await _httpClientFunction.SendGet(_config.RetrievePdsDemographicURL, queryParams);
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "There was an error retrieving the PDS record.");
-            return null;
-        }
+        return await _httpClientFunction.SendGetHttpResponse(_config.RetrievePdsDemographicURL, queryParams);
     }
 
-    private async Task ProcessRecord(PdsDemographic pdsDemographic)
+    private async Task ProcessRecord(Participant participant)
     {
         var updateRecord = new ConcurrentQueue<BasicParticipantCsvRecord>();
 
-        var participant = new Participant(pdsDemographic);
-
         // TODO validate all dates in record before enqueuing
-
         var existingParticipant = await _participantDemographic.GetSingleByFilter(x => x.NhsNumber == nhsNumberLong);
+
+
         if (existingParticipant == null)
         {
             participant.RecordType = Actions.New;
             _logger.LogWarning("The participant doesn't exists in Cohort Manager.A new record will be created in Cohort Manager.");
         }
         else
         {
-            participant.RecordType = Actions.Amended;
+            participant.RecordType = participant.RecordType != Actions.Removed ? Actions.Amended : participant.RecordType;
             _logger.LogWarning("The participant already exists in Cohort Manager. Existing record will get updated.");
         }
 
@@ -204,6 +221,7 @@ private async Task ProcessRecord(PdsDemographic pdsDemographic)
         updateRecord.Enqueue(basicParticipantCsvRecord);
 
         _logger.LogInformation("Sending record to the update queue.");
+
         await _addBatchToQueue.ProcessBatch(updateRecord, _config.ParticipantManagementTopic);
     }
 
diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdateConfig.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/ProcessNemsUpdateConfig.cs
@@ -7,7 +7,14 @@ public class ProcessNemsUpdateConfig
     public required string RetrievePdsDemographicURL { get; set; }
     public required string NemsMessages { get; set; }
     public required string UnsubscribeNemsSubscriptionUrl { get; set; }
-    public required string ParticipantDemographicDataServiceURL { get; set; }
+
+    [Required]
     public required string ServiceBusConnectionString_client_internal { get; set; }
+
+    [Required]
     public required string ParticipantManagementTopic { get; set; }
+
+    [Required]
+    public required string DemographicDataServiceURL { get; set; }
+
 }
diff --git a/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/Program.cs b/application/CohortManager/src/Functions/NemsSubscriptionService/ProcessNemsUpdate/Program.cs
@@ -9,10 +9,10 @@
 
 var host = new HostBuilder()
     .AddConfiguration<ProcessNemsUpdateConfig>(out ProcessNemsUpdateConfig config)
-        .AddDataServicesHandler()
-        .AddDataService<ParticipantDemographic>(config.ParticipantDemographicDataServiceURL)
-        .Build()
     .ConfigureFunctionsWorkerDefaults()
+      .AddDataServicesHandler()
+        .AddDataService<ParticipantDemographic>(config.DemographicDataServiceURL)
+        .Build()
     .ConfigureServices(services =>
     {
         services.AddSingleton<IFhirPatientDemographicMapper, FhirPatientDemographicMapper>();
@@ -23,7 +23,6 @@
     .AddTelemetry()
     .AddExceptionHandler()
     .AddHttpClient()
-    .AddAzureQueues()
     .AddServiceBusClient(config.ServiceBusConnectionString_client_internal)
     .Build();
 
diff --git a/application/CohortManager/src/Functions/Shared/Common/HttpClientFunction.cs b/application/CohortManager/src/Functions/Shared/Common/HttpClientFunction.cs
diff --git a/application/CohortManager/src/Functions/Shared/Common/Interfaces/IHttpClientFunction.cs b/application/CohortManager/src/Functions/Shared/Common/Interfaces/IHttpClientFunction.cs
diff --git a/application/CohortManager/src/Functions/Shared/Common/PdsHttpClientMock.cs b/application/CohortManager/src/Functions/Shared/Common/PdsHttpClientMock.cs
diff --git a/application/CohortManager/src/Functions/Shared/Model/CohortDistributionParticipant.cs b/application/CohortManager/src/Functions/Shared/Model/CohortDistributionParticipant.cs
diff --git a/tests/UnitTests/NemsSubscriptionServiceTests/ProcessNemsUpdateTests/ProcessNemsUpdateTests.cs b/tests/UnitTests/NemsSubscriptionServiceTests/ProcessNemsUpdateTests/ProcessNemsUpdateTests.cs

Original file line number	Diff line number	Diff line change
`@@ -49,13 +49,13 @@ public async Task<HttpResponseData> Subscribe([HttpTrigger(AuthorizationLevel.An`
`49`	`49`
`50`	`50`	`string? nhsNumber = req.Query["nhsNumber"];`
`51`	`51`
`52`		`- if (!ValidationHelper.ValidateNHSNumber(nhsNumber))`
	`52`	`+ if (!ValidationHelper.ValidateNHSNumber(nhsNumber!))`
`53`	`53`	`{`
`54`	`54`	`_logger.LogError("NHS number is required and must be valid format");`
`55`	`55`	`return await _createResponse.CreateHttpResponseWithBodyAsync(HttpStatusCode.BadRequest, req, "NHS number is required and must be valid format.");`
`56`	`56`	`}`
`57`	`57`
`58`		`- var result = await _subscriptionManager.CreateAndSendSubscriptionAsync(nhsNumber);`
	`58`	`+ var result = await _subscriptionManager.CreateAndSendSubscriptionAsync(nhsNumber!);`
`59`	`59`
`60`	`60`	`if (!result.Success)`
`61`	`61`	`{`