feat: dtoss 3660 nhs id in participant man demograp

.gitignore

@@ -84,4 +84,4 @@ application/CohortManager/.vscode/
 # Ignore generated feature bindings
 *.feature.cs
 tests/smoke-tests/dtos-cohort-manager-smoke-tests/Config/appsettings-local.json
-*.crt
+tests/e2e-tests/dtos-cohort-manager-e2e-tests/Config/appsettings-local.json

application/CohortManager/compose.deps.yaml

@@ -43,4 +43,4 @@ services:
         condition: service_healthy
     environment:
       - PASSWORD=${PASSWORD}
-      - DB_NAME=${DB_NAME}
+      - DB_NAME=${DB_NAME}

scripts/config/gitleaks.toml

@@ -30,7 +30,8 @@ paths = [
   '''tests/CaasIntegrationTests/receiveCaasFileTest/receiveCaasFileTest.csproj''',
   '''tests/IntegrationTests/integration.csproj''',
   '''application/CohortManager/src/Functions/Shared/Model/Model.csproj''',
-  '''tests/smoke-tests/dtos-cohort-manager-smoke-tests/'''
+  '''tests/smoke-tests/dtos-cohort-manager-smoke-tests/''',
+  '''tests/e2e-tests/dtos-cohort-manager-e2e-tests/'''
 ]
 
 

tests/e2e-tests/dtos-cohort-manager-e2e-tests/.gitignore

@@ -0,0 +1,87 @@
+# WARNING: Please, DO NOT edit this section of the file! It is maintained in the repository template.
+
+.scannerwork
+*cloc*report*.json
+*sbom*report*.json
+*vulnerabilities*report*.json
+*report*json.zip
+.version
+
+*.code-workspace
+!project.code-workspace
+
+# Please, add your custom content below!
+
+# keys
+
+*.pfx
+*.pem
+*.key
+
+# User-specific files
+*.suo
+*.user
+
+# Visual Studio 2015/2017 cache/options directory
+.vs/
+
+# Visual Studio 2017 auto generated files
+Generated\ Files/
+
+# Text Editors
+.vscode/*
+!.vscode/tasks.json
+_ReSharper.*/
+*/*/.idea/*
+
+# Azurite
+__azurite*
+__blobstorage__
+__queuestorage__
+
+# Tests
+TestResults
+
+# Builds
+bin
+obj
+OutputDirectory
+
+# Mac
+.DS_Store
+
+# User Settings
+local.settings.json
+
+# Docker
+.env
+
+# sample data
+application/CohortManager/SampleData/
+*.csv
+*.parquet
+!tests/UnitTests/CaasIntegrationTests/receiveCaasFileTest/*.parquet
+!tests/smoke-tests/dtos-cohort-manager-smoke-tests/TestFiles/**/*.parquet
+
+# csv to sql statement scripts
+bs_select_gp_practice_lpk.txt
+current_posting.txt
+select_outcode_mapping_lkp_output.txt
+
+# Python
+*.pyc
+
+
+recreate_local_settings.sh
+
+.history
+application/CohortManager/Set-up/SampleData/CaaSSchema02.json
+infrastructure/.terraform.lock.hcl
+infrastructure/tf-core/.terraform.lock.hcl
+infrastructure/tf-audit/.terraform.lock.hcl
+application/CohortManager/.vscode/
+
+# Ignore generated feature bindings
+*.feature.cs
+tests/smoke-tests/dtos-cohort-manager-smoke-tests/Config/appsettings-local.json
+tests/e2e-tests/dtos-cohort-manager-e2e-tests/Config/appsettings-local.json

tests/e2e-tests/dtos-cohort-manager-e2e-tests/Config/AppSettings.cs

@@ -0,0 +1,20 @@
+using dtos_cohort_manager_e2e_tests.Config;
+public class AzureSettings
+{
+    public bool IsCloudEnvironment { get; set; }
+}
+
+public class AppSettings
+{
+    public ConnectionStrings ConnectionStrings { get; set; }
+    public AzureSettings AzureSettings { get; set; }
+    public string? ManagedIdentityClientId { get; set; }
+    public FilePaths FilePaths { get; set; }
+    public string BlobContainerName { get; set; }
+    public string AzureWebJobsStorage { get; set; }
+}
+
+public class ConnectionStrings
+{
+    public string DtOsDatabaseConnectionString { get; set; }
+}

tests/e2e-tests/dtos-cohort-manager-e2e-tests/Config/FilePaths.cs

@@ -0,0 +1,8 @@
+namespace dtos_cohort_manager_e2e_tests.Config;
+
+public class FilePaths
+{
+    public string Add { get; set; }
+    public string Amended { get; set; }
+    public string Remove { get; set; }
+}

tests/e2e-tests/dtos-cohort-manager-e2e-tests/Config/appsettings.json

@@ -0,0 +1,18 @@
+{
+  "AppSettings": {
+    "ConnectionStrings": {
+      "DtOsDatabaseConnectionString": ""
+    },
+    "AzureSettings": {
+    "IsCloudEnvironment": true
+  },
+ "FilePaths": {
+  "Add": "TestFiles/add/",
+  "Amended": "TestFiles/amended/",
+  "Remove": "TestFiles/remove/"
+},
+    "CloudFileStorageConnectionString": "",
+    "BlobContainerName": "inbound",
+    "ManagedIdentityClientId": ""
+  }
+}

tests/e2e-tests/dtos-cohort-manager-e2e-tests/Contexts/SmokeTestsContext.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using CsvHelper;
+using dtos_cohort_manager_e2e_tests.Models;
+
+namespace dtos_cohort_manager_e2e_tests.Contexts;
+
+public class EndToEndTestsContext
+{
+    public string FilePath { get; set; }
+
+    public RecordTypesEnum RecordType { get; set; }
+
+    public List<string>? NhsNumbers { get; set; }
+}

tests/e2e-tests/dtos-cohort-manager-e2e-tests/Features/Epic1_AutomatedRegressionSuite.feature

@@ -0,0 +1,90 @@
+Feature: Epic1_AutomatedRegressionSuite
+DTOSS Regression TEST PACK.
+
+  Background:
+    Given the database is cleaned of all records for NHS Numbers: 1111110662, 2222211794,2312514176,2612314172,2612514171
+    And the application is properly configured
+
+  @DTOSS-7583 @Regression
+  Scenario: Verify NHS data propagation across participant tables after file upload for ADD record
+    Given file <FileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then verify the NHS numbers in Participant_Management and Participant_Demographic table should match the file data
+
+    Examples:
+      | FileName                                             | RecordType | NhsNumbers             |
+      | ADD_2_RECORDS_-_CAAS_BREAST_SCREENING_COHORT.parquet | Add        | 1111110662, 2222211794 |
+
+  @DTOSS-7583 @Regression
+  Scenario: Verify NHS data propagation across participant tables after file upload for AMENDED record
+    Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    And the file is uploaded to the Blob Storage container
+    And the NHS numbers in the database should match the file data
+    And file <AmendedFileName> exists in the configured location for "Amended" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then verify the NhsNumbers in Participant_Management table should match <RecordType>
+    And the Participant_Demographic table should match the <AmendedGivenName> for the NHS Number
+
+    Examples:
+      | AddFileName                                        | AmendedFileName                                        | NhsNumbers | AmendedGivenName | RecordType |
+      | ADD1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | AMENDED1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2312514176 | AMENDEDNewTest1  | Amended    |
+
+  @DTOSS-7584 @Regression
+  Scenario: Confirm NHS Number Count Integrity Across Participant Tables After Processing for ADD record
+    Given file <FileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then the NHS Number should have exactly 1 record in Participant_Management
+    And the NHS Number should have exactly 1 record in Participant_Demographic
+
+    Examples:
+      | FileName                                             | RecordType | NhsNumbers             |
+      | ADD_2_RECORDS_-_CAAS_BREAST_SCREENING_COHORT.parquet | Add        | 1111110662, 2222211794 |
+
+  @DTOSS-7584 @Regression
+  Scenario: Confirm NHS Number Count Integrity Across Participant Tables After Processing for AMENDED record
+    Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    And the file is uploaded to the Blob Storage container
+    And the NHS numbers in the database should match the file data
+    And file <AmendedFileName> exists in the configured location for "Amended" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then the NHS Number should have exactly 1 record in Participant_Management
+    And the NHS Number should have exactly 1 record in Participant_Demographic
+
+    Examples:
+      | AddFileName                                        | AmendedFileName                                        | NhsNumbers |
+      | ADD1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | AMENDED1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2312514176 |
+
+  @DTOSS-7585 @Regression
+  Scenario: Verify exception records doesn't end up in Participant_Demographic and Participant_Management table for ADD record
+    Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then the Exception table should contain the below details for the NHS Number
+      | FieldName        | FieldValue                                     |
+      | RULE_ID          |                                             36 |
+      | RULE_DESCRIPTION | Invalid primary care provider GP practice code |
+    Then the NHS Number should have exactly 1 record in Participant_Management
+    And the NHS Number should have exactly 1 record in Participant_Demographic
+    And the NHS Number should have exactly 0 record in Cohort_Distribution table
+
+    Examples:
+      | AddFileName                                             | NhsNumbers |
+      | Exception_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2612314172 |
+
+  @DTOSS-7585 @Regression
+  Scenario: Verify exception records doesn't end up in Participant_Demographic and Participant_Management table for AMENDED record
+    Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    And the file is uploaded to the Blob Storage container
+    And the NHS numbers in the database should match the file data
+    And file <AmendedFileName> exists in the configured location for "Amended" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then the Exception table should contain the below details for the NHS Number
+      | FieldName        | FieldValue            |
+      | RULE_ID          |                    17 |
+      | RULE_DESCRIPTION | Date of birth invalid |
+    Then the NHS Number should have exactly 1 record in Participant_Management
+    And the NHS Number should have exactly 1 record in Participant_Demographic
+    And the NHS Number should have exactly 1 record in Cohort_Distribution table
+
+    Examples:
+      | AddFileName                                       | AmendedFileName                                       | NhsNumbers |
+      | ADD_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | AMENDED_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2612514171 |

tests/e2e-tests/dtos-cohort-manager-e2e-tests/Helpers/BlobStorageHelper.cs

@@ -0,0 +1,38 @@
+using System.IO;
+using System.Threading.Tasks;
+using Azure.Storage.Blobs;
+using Microsoft.Extensions.Logging;
+
+namespace dtos_cohort_manager_e2e_tests.Helpers;
+
+public class BlobStorageHelper
+{
+    private readonly BlobServiceClient _blobServiceClient;
+    private readonly ILogger<BlobStorageHelper> _logger;
+
+
+    public BlobStorageHelper(BlobServiceClient blobServiceClient, ILogger<BlobStorageHelper> logger)
+    {
+        _blobServiceClient = blobServiceClient;
+        _logger = logger;
+    }
+
+    public async Task UploadFileToBlobStorageAsync(string filePath, string blobContainerName)
+    {
+        if (!File.Exists(filePath))
+        {
+            _logger.LogError($"File not found at {filePath}");
+            return;
+        }
+
+        _logger.LogInformation("Uploading file {FilePath} to blob storage", filePath);
+
+        var blobContainerClient = _blobServiceClient.GetBlobContainerClient(blobContainerName);
+        await blobContainerClient.CreateIfNotExistsAsync();
+
+        var blobClient = blobContainerClient.GetBlobClient(Path.GetFileName(filePath));
+        await blobClient.UploadAsync(File.OpenRead(filePath), true);
+
+        _logger.LogInformation("File uploaded successfully");
+    }
+}

tests/e2e-tests/dtos-cohort-manager-e2e-tests/Helpers/CsvHelper.cs

@@ -0,0 +1,47 @@
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using CsvHelper;
+using CsvHelper.Configuration;
+
+namespace dtos_cohort_manager_e2e_tests.Helpers
+{
+    public static class CsvHelperService
+    {
+        public static List<string> ExtractNhsNumbersFromCsv(string filePath)
+        {
+            using var reader = new StreamReader(filePath);
+            using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture));
+            var records = new List<string>();
+            csv.Read();
+            csv.ReadHeader();
+            while (csv.Read())
+            {
+                records.Add(csv.GetField(3));
+            }
+            return records;
+        }
+
+        public static List<Dictionary<string, string>> ReadCsv(string filePath)
+        {
+            using var reader = new StreamReader(filePath);
+            using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture));
+            var records = new List<Dictionary<string, string>>();
+            csv.Read();
+            csv.ReadHeader();
+            var headers = csv.HeaderRecord;
+
+            while (csv.Read())
+            {
+                var record = new Dictionary<string, string>();
+                foreach (var header in headers)
+                {
+                    record[header] = csv.GetField(header);
+                }
+                records.Add(record);
+            }
+
+            return records;
+        }
+    }
+}