feat: dtoss 3660 nhs id in participant man demograp

.gitignore

@@ -62,6 +62,7 @@ application/CohortManager/SampleData/
 *.parquet
 !tests/UnitTests/CaasIntegrationTests/receiveCaasFileTest/*.parquet
 !tests/smoke-tests/dtos-cohort-manager-smoke-tests/TestFiles/**/*.parquet
+!tests/e2e-tests/NHS.CohortManager.EndToEndTests/TestFiles/**/*.parquet
 
 # csv to sql statement scripts
 bs_select_gp_practice_lpk.txt
@@ -84,4 +85,5 @@ application/CohortManager/.vscode/
 # Ignore generated feature bindings
 *.feature.cs
 tests/smoke-tests/dtos-cohort-manager-smoke-tests/Config/appsettings-local.json
+tests/e2e-tests/NHS.CohortManager.EndToEndTests/Config/appsettings-local.json
 *.crt

application/CohortManager/compose.deps.yaml

@@ -43,4 +43,4 @@ services:
         condition: service_healthy
     environment:
       - PASSWORD=${PASSWORD}
-      - DB_NAME=${DB_NAME}
+      - DB_NAME=${DB_NAME}

scripts/config/gitleaks.toml

@@ -30,7 +30,9 @@ paths = [
   '''tests/CaasIntegrationTests/receiveCaasFileTest/receiveCaasFileTest.csproj''',
   '''tests/IntegrationTests/integration.csproj''',
   '''application/CohortManager/src/Functions/Shared/Model/Model.csproj''',
-  '''tests/smoke-tests/dtos-cohort-manager-smoke-tests/'''
+  '''tests/smoke-tests/dtos-cohort-manager-smoke-tests/''',
+'''tests/e2e-tests/NHS.CohortManager.EndToEndTests/NHS.CohortManager.EndToEndTests.csproj''',
+'''tests/e2e-tests/dtos-cohort-manager-e2e-tests/dtos_cohort_manager_e2e_tests.csproj'''
 ]
 
 

tests/e2e-tests/NHS.CohortManager.EndToEndTests/Config/AppSettings.cs

@@ -0,0 +1,20 @@
+using NHS.CohortManager.EndToEndTests.Config;
+public class AzureSettings
+{
+    public bool IsCloudEnvironment { get; set; }
+}
+
+public class AppSettings
+{
+    public ConnectionStrings ConnectionStrings { get; set; }
+    public AzureSettings AzureSettings { get; set; }
+    public string? ManagedIdentityClientId { get; set; }
+    public FilePaths FilePaths { get; set; }
+    public string BlobContainerName { get; set; }
+    public string AzureWebJobsStorage { get; set; }
+}
+
+public class ConnectionStrings
+{
+    public string DtOsDatabaseConnectionString { get; set; }
+}

tests/e2e-tests/NHS.CohortManager.EndToEndTests/Config/FilePaths.cs

@@ -0,0 +1,8 @@
+namespace NHS.CohortManager.EndToEndTests.Config;
+
+public class FilePaths
+{
+    public string Add { get; set; }
+    public string Amended { get; set; }
+    public string Remove { get; set; }
+}

tests/e2e-tests/NHS.CohortManager.EndToEndTests/Config/appsettings-example.json

@@ -0,0 +1,18 @@
+{
+  "AppSettings": {
+    "ConnectionStrings": {
+      "DtOsDatabaseConnectionString": ""
+    },
+    "AzureSettings": {
+    "IsCloudEnvironment": true
+  },
+ "FilePaths": {
+  "Add": "TestFiles/add/",
+  "Amended": "TestFiles/amended/",
+  "Remove": "TestFiles/remove/"
+},
+    "CloudFileStorageConnectionString": "",
+    "BlobContainerName": "inbound",
+    "ManagedIdentityClientId": ""
+  }
+}

tests/e2e-tests/NHS.CohortManager.EndToEndTests/Contexts/EndToEndTestsContext.cs

@@ -0,0 +1,20 @@
+namespace NHS.CohortManager.EndToEndTests.Contexts;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using CsvHelper;
+using NHS.CohortManager.EndToEndTests.Models;
+
+
+
+public class EndToEndTestsContext
+{
+    public string FilePath { get; set; }
+
+    public RecordTypesEnum RecordType { get; set; }
+
+    public List<string>? NhsNumbers { get; set; }
+}

tests/e2e-tests/NHS.CohortManager.EndToEndTests/Features/Epic1_AutomatedRegressionSuite.feature

@@ -0,0 +1,90 @@
+Feature: Epic1_AutomatedRegressionSuite
+DTOSS Regression TEST PACK.
+
+  Background:
+    Given the database is cleaned of all records for NHS Numbers: 1111110662, 2222211794,2312514176,2612314172,2612514171
+    And the application is properly configured
+
+  @DTOSS-7583 @Regression
+  Scenario: Verify ADD records reach the participant tables
+    Given file <FileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then verify the NHS numbers in Participant_Management and Participant_Demographic table should match the file data
+
+    Examples:
+      | FileName                                             | RecordType | NhsNumbers             |
+      | ADD_2_RECORDS_-_CAAS_BREAST_SCREENING_COHORT.parquet | Add        | 1111110662, 2222211794 |
+
+  @DTOSS-7583 @Regression
+  Scenario: Verify AMENDED records reach the participant tables
+    Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    And the file is uploaded to the Blob Storage container
+    And the NHS numbers in the database should match the file data
+    And file <AmendedFileName> exists in the configured location for "Amended" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then verify the NhsNumbers in Participant_Management table should match <RecordType>
+    And the Participant_Demographic table should match the <AmendedGivenName> for the NHS Number
+
+    Examples:
+      | AddFileName                                        | AmendedFileName                                        | NhsNumbers | AmendedGivenName | RecordType |
+      | ADD1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | AMENDED1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2312514176 | AMENDEDNewTest1  | Amended    |
+
+  @DTOSS-7584 @Regression
+  Scenario: Confirm NHS Number Count Integrity Across Participant Tables After Processing for ADD record
+    Given file <FileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then the NHS Number should have exactly 1 record in Participant_Management
+    And the NHS Number should have exactly 1 record in Participant_Demographic
+
+    Examples:
+      | FileName                                             | RecordType | NhsNumbers             |
+      | ADD_2_RECORDS_-_CAAS_BREAST_SCREENING_COHORT.parquet | Add        | 1111110662, 2222211794 |
+
+  @DTOSS-7584 @Regression
+  Scenario: Confirm NHS Number Count Integrity Across Participant Tables After Processing for AMENDED record
+    Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    And the file is uploaded to the Blob Storage container
+    And the NHS numbers in the database should match the file data
+    And file <AmendedFileName> exists in the configured location for "Amended" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then the NHS Number should have exactly 1 record in Participant_Management
+    And the NHS Number should have exactly 1 record in Participant_Demographic
+
+    Examples:
+      | AddFileName                                        | AmendedFileName                                        | NhsNumbers |
+      | ADD1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | AMENDED1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2312514176 |
+
+  @DTOSS-7585 @Regression
+  Scenario: Verify ADD records that trigger a non-fatal validation rule reach internal participant tables but not Cohort distribution
+    Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then the Exception table should contain the below details for the NHS Number
+      | FieldName        | FieldValue                                     |
+      | RULE_ID          |                                             36 |
+      | RULE_DESCRIPTION | Invalid primary care provider GP practice code |
+    Then the NHS Number should have exactly 1 record in Participant_Management
+    And the NHS Number should have exactly 1 record in Participant_Demographic
+    And the NHS Number should have exactly 0 record in Cohort_Distribution table
+
+    Examples:
+      | AddFileName                                             | NhsNumbers |
+      | Exception_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2612314172 |
+
+  @DTOSS-7585 @Regression
+  Scenario: Verify AMENDED records with non-fatal validation issues reach participant tables with partial Cohort distribution entries
+    Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
+    And the file is uploaded to the Blob Storage container
+    And the NHS numbers in the database should match the file data
+    And file <AmendedFileName> exists in the configured location for "Amended" with NHS numbers : <NhsNumbers>
+    When the file is uploaded to the Blob Storage container
+    Then the Exception table should contain the below details for the NHS Number
+      | FieldName        | FieldValue            |
+      | RULE_ID          |                    17 |
+      | RULE_DESCRIPTION | Date of birth invalid |
+    Then the NHS Number should have exactly 1 record in Participant_Management
+    And the NHS Number should have exactly 1 record in Participant_Demographic
+    And the NHS Number should have exactly 1 record in Cohort_Distribution table
+
+    Examples:
+      | AddFileName                                       | AmendedFileName                                       | NhsNumbers |
+      | ADD_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | AMENDED_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2612514171 |

tests/e2e-tests/NHS.CohortManager.EndToEndTests/Helpers/BlobStorageHelper.cs

@@ -0,0 +1,40 @@
+namespace NHS.CohortManager.EndToEndTests.Helpers;
+
+using System.IO;
+using System.Threading.Tasks;
+using Azure.Storage.Blobs;
+using Microsoft.Extensions.Logging;
+
+
+
+public class BlobStorageHelper
+{
+    private readonly BlobServiceClient _blobServiceClient;
+    private readonly ILogger<BlobStorageHelper> _logger;
+
+
+    public BlobStorageHelper(BlobServiceClient blobServiceClient, ILogger<BlobStorageHelper> logger)
+    {
+        _blobServiceClient = blobServiceClient;
+        _logger = logger;
+    }
+
+    public async Task UploadFileToBlobStorageAsync(string filePath, string blobContainerName)
+    {
+        if (!File.Exists(filePath))
+        {
+            _logger.LogError($"File not found at {filePath}");
+            return;
+        }
+
+        _logger.LogInformation("Uploading file {FilePath} to blob storage", filePath);
+
+        var blobContainerClient = _blobServiceClient.GetBlobContainerClient(blobContainerName);
+        await blobContainerClient.CreateIfNotExistsAsync();
+
+        var blobClient = blobContainerClient.GetBlobClient(Path.GetFileName(filePath));
+        await blobClient.UploadAsync(File.OpenRead(filePath), true);
+
+        _logger.LogInformation("File uploaded successfully");
+    }
+}

tests/e2e-tests/NHS.CohortManager.EndToEndTests/Helpers/CsvHelper.cs

@@ -0,0 +1,49 @@
+namespace NHS.CohortManager.EndToEndTests.Helpers;
+
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using CsvHelper;
+using CsvHelper.Configuration;
+
+
+
+public static class CsvHelperService
+{
+    public static List<string> ExtractNhsNumbersFromCsv(string filePath)
+    {
+        using var reader = new StreamReader(filePath);
+        using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture));
+        var records = new List<string>();
+        csv.Read();
+        csv.ReadHeader();
+        while (csv.Read())
+        {
+            records.Add(csv.GetField(3));
+        }
+        return records;
+    }
+
+    public static List<Dictionary<string, string>> ReadCsv(string filePath)
+    {
+        using var reader = new StreamReader(filePath);
+        using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture));
+        var records = new List<Dictionary<string, string>>();
+        csv.Read();
+        csv.ReadHeader();
+        var headers = csv.HeaderRecord;
+
+        while (csv.Read())
+        {
+            var record = new Dictionary<string, string>();
+            foreach (var header in headers)
+            {
+                record[header] = csv.GetField(header);
+            }
+            records.Add(record);
+        }
+
+        return records;
+    }
+}
+

tests/e2e-tests/NHS.CohortManager.EndToEndTests/Helpers/DatabaseHelper.cs

@@ -0,0 +1,65 @@
+namespace NHS.CohortManager.EndToEndTests.Helpers;
+
+using Azure.Core;
+using Azure.Identity;
+using Microsoft.Data.SqlClient;
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+
+public static class DatabaseHelper
+{
+    // Whitelist of allowed table names
+    private static readonly HashSet<string> AllowedTables = new HashSet<string>
+    {
+        "PARTICIPANT_MANAGEMENT",
+        "PARTICIPANT_DEMOGRAPHIC",
+        "BS_COHORT_DISTRIBUTION",
+    };
+
+    public static async Task<int> ExecuteNonQueryAsync(
+     SqlConnectionWithAuthentication sqlAuthConnection,
+     string query,
+     params SqlParameter[] parameters)
+    {
+        await using var connection = await sqlAuthConnection.GetOpenConnectionAsync();
+        await using var command = new SqlCommand(query, connection);
+
+        command.Parameters.AddRange(parameters);
+
+        return await command.ExecuteNonQueryAsync();
+    }
+
+    public static async Task<int> GetRecordCountAsync(SqlConnectionWithAuthentication sqlConnectionWithAuthentication, string tableName)
+    {
+        // Check if the table name is in the whitelist
+        if (!AllowedTables.Contains(tableName.ToUpper()))
+        {
+            throw new ArgumentException($"Table '{tableName}' is not in the list of allowed tables.");
+        }
+
+        // Get the open connection (with token if using Managed Identity)
+        using var connection = await sqlConnectionWithAuthentication.GetOpenConnectionAsync();
+
+        // Check if the table actually exists in the database
+        if (!await TableExistsAsync(connection, tableName))
+        {
+            throw new ArgumentException($"Table '{tableName}' does not exist in the database.");
+        }
+
+        var query = "SELECT COUNT(*) FROM " + tableName;
+        using var command = new SqlCommand(query, connection);
+        return (int)await command.ExecuteScalarAsync();
+    }
+
+    private static async Task<bool> TableExistsAsync(SqlConnection connection, string tableName)
+    {
+        var query = "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = @TableName";
+        using (var command = new SqlCommand(query, connection))
+        {
+            command.Parameters.AddWithValue("@TableName", tableName);
+            return (int)await command.ExecuteScalarAsync() > 0;
+        }
+    }
+}