Skip to content

Commit 99578b1

Browse files
charanNHScharanNHS
authored
feat: dtoss 3660 nhs id in participant man demograp (#826)
* Full tested feature and working * Removed api version check in deps yaml * fixed format * Fix issue compose core * fixed cucumber gherkin issue * copied necessary classes from smoke tests project * Added gitignore * Removed reference of epic1 in smoke tests solution * Added end to end tests to gitleaks file * Renamed Smoke context to end to end context * Removed gitginore and using the one on root * Added check to include test files in gitignore * Removed unused methods in datavalidation helper and included just Testfiles folder * Just ignored csproj file * Moved Private methods to the bottom of the class * Renamed name spaces and project * Edited gitignore file * Including old project name * Renamed feature file * Renamed Scenarios with meaningful name * Renamed scenarios to will suggestions * Added missing crt back to gitignore * Added old proj reference * Renamed appsettings.json
1 parent 04980d6 commit 99578b1

43 files changed

Lines changed: 1378 additions & 130 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ application/CohortManager/SampleData/
6262
*.parquet
6363
!tests/UnitTests/CaasIntegrationTests/receiveCaasFileTest/*.parquet
6464
!tests/smoke-tests/dtos-cohort-manager-smoke-tests/TestFiles/**/*.parquet
65+
!tests/e2e-tests/NHS.CohortManager.EndToEndTests/TestFiles/**/*.parquet
6566

6667
# csv to sql statement scripts
6768
bs_select_gp_practice_lpk.txt
@@ -84,4 +85,5 @@ application/CohortManager/.vscode/
8485
# Ignore generated feature bindings
8586
*.feature.cs
8687
tests/smoke-tests/dtos-cohort-manager-smoke-tests/Config/appsettings-local.json
88+
tests/e2e-tests/NHS.CohortManager.EndToEndTests/Config/appsettings-local.json
8789
*.crt

‎application/CohortManager/compose.deps.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,4 +43,4 @@ services:
4343
condition: service_healthy
4444
environment:
4545
- PASSWORD=${PASSWORD}
46-
- DB_NAME=${DB_NAME}
46+
- DB_NAME=${DB_NAME}

‎scripts/config/gitleaks.toml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ paths = [
3030
'''tests/CaasIntegrationTests/receiveCaasFileTest/receiveCaasFileTest.csproj''',
3131
'''tests/IntegrationTests/integration.csproj''',
3232
'''application/CohortManager/src/Functions/Shared/Model/Model.csproj''',
33-
'''tests/smoke-tests/dtos-cohort-manager-smoke-tests/'''
33+
'''tests/smoke-tests/dtos-cohort-manager-smoke-tests/''',
34+
'''tests/e2e-tests/NHS.CohortManager.EndToEndTests/NHS.CohortManager.EndToEndTests.csproj''',
35+
'''tests/e2e-tests/dtos-cohort-manager-e2e-tests/dtos_cohort_manager_e2e_tests.csproj'''
3436
]
3537

3638

‎tests/e2e-tests/NHS.CohortManager.EndToEndTests/Config/AppSettings.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
using NHS.CohortManager.EndToEndTests.Config;
2+
public class AzureSettings
3+
{
4+
public bool IsCloudEnvironment { get; set; }
5+
}
6+
7+
public class AppSettings
8+
{
9+
public ConnectionStrings ConnectionStrings { get; set; }
10+
public AzureSettings AzureSettings { get; set; }
11+
public string? ManagedIdentityClientId { get; set; }
12+
public FilePaths FilePaths { get; set; }
13+
public string BlobContainerName { get; set; }
14+
public string AzureWebJobsStorage { get; set; }
15+
}
16+
17+
public class ConnectionStrings
18+
{
19+
public string DtOsDatabaseConnectionString { get; set; }
20+
}

‎tests/e2e-tests/NHS.CohortManager.EndToEndTests/Config/FilePaths.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
namespace NHS.CohortManager.EndToEndTests.Config;
2+
3+
public class FilePaths
4+
{
5+
public string Add { get; set; }
6+
public string Amended { get; set; }
7+
public string Remove { get; set; }
8+
}

‎tests/e2e-tests/NHS.CohortManager.EndToEndTests/Config/appsettings-example.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"AppSettings": {
3+
"ConnectionStrings": {
4+
"DtOsDatabaseConnectionString": ""
5+
},
6+
"AzureSettings": {
7+
"IsCloudEnvironment": true
8+
},
9+
"FilePaths": {
10+
"Add": "TestFiles/add/",
11+
"Amended": "TestFiles/amended/",
12+
"Remove": "TestFiles/remove/"
13+
},
14+
"CloudFileStorageConnectionString": "",
15+
"BlobContainerName": "inbound",
16+
"ManagedIdentityClientId": ""
17+
}
18+
}

‎tests/e2e-tests/NHS.CohortManager.EndToEndTests/Contexts/EndToEndTestsContext.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
namespace NHS.CohortManager.EndToEndTests.Contexts;
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Linq;
6+
using System.Text;
7+
using System.Threading.Tasks;
8+
using CsvHelper;
9+
using NHS.CohortManager.EndToEndTests.Models;
10+
11+
12+
13+
public class EndToEndTestsContext
14+
{
15+
public string FilePath { get; set; }
16+
17+
public RecordTypesEnum RecordType { get; set; }
18+
19+
public List<string>? NhsNumbers { get; set; }
20+
}

‎tests/e2e-tests/NHS.CohortManager.EndToEndTests/Features/Epic1_AutomatedRegressionSuite.feature‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
Feature: Epic1_AutomatedRegressionSuite
2+
DTOSS Regression TEST PACK.
3+
4+
Background:
5+
Given the database is cleaned of all records for NHS Numbers: 1111110662, 2222211794,2312514176,2612314172,2612514171
6+
And the application is properly configured
7+
8+
@DTOSS-7583 @Regression
9+
Scenario: Verify ADD records reach the participant tables
10+
Given file <FileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
11+
When the file is uploaded to the Blob Storage container
12+
Then verify the NHS numbers in Participant_Management and Participant_Demographic table should match the file data
13+
14+
Examples:
15+
| FileName | RecordType | NhsNumbers |
16+
| ADD_2_RECORDS_-_CAAS_BREAST_SCREENING_COHORT.parquet | Add | 1111110662, 2222211794 |
17+
18+
@DTOSS-7583 @Regression
19+
Scenario: Verify AMENDED records reach the participant tables
20+
Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
21+
And the file is uploaded to the Blob Storage container
22+
And the NHS numbers in the database should match the file data
23+
And file <AmendedFileName> exists in the configured location for "Amended" with NHS numbers : <NhsNumbers>
24+
When the file is uploaded to the Blob Storage container
25+
Then verify the NhsNumbers in Participant_Management table should match <RecordType>
26+
And the Participant_Demographic table should match the <AmendedGivenName> for the NHS Number
27+
28+
Examples:
29+
| AddFileName | AmendedFileName | NhsNumbers | AmendedGivenName | RecordType |
30+
| ADD1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | AMENDED1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2312514176 | AMENDEDNewTest1 | Amended |
31+
32+
@DTOSS-7584 @Regression
33+
Scenario: Confirm NHS Number Count Integrity Across Participant Tables After Processing for ADD record
34+
Given file <FileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
35+
When the file is uploaded to the Blob Storage container
36+
Then the NHS Number should have exactly 1 record in Participant_Management
37+
And the NHS Number should have exactly 1 record in Participant_Demographic
38+
39+
Examples:
40+
| FileName | RecordType | NhsNumbers |
41+
| ADD_2_RECORDS_-_CAAS_BREAST_SCREENING_COHORT.parquet | Add | 1111110662, 2222211794 |
42+
43+
@DTOSS-7584 @Regression
44+
Scenario: Confirm NHS Number Count Integrity Across Participant Tables After Processing for AMENDED record
45+
Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
46+
And the file is uploaded to the Blob Storage container
47+
And the NHS numbers in the database should match the file data
48+
And file <AmendedFileName> exists in the configured location for "Amended" with NHS numbers : <NhsNumbers>
49+
When the file is uploaded to the Blob Storage container
50+
Then the NHS Number should have exactly 1 record in Participant_Management
51+
And the NHS Number should have exactly 1 record in Participant_Demographic
52+
53+
Examples:
54+
| AddFileName | AmendedFileName | NhsNumbers |
55+
| ADD1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | AMENDED1_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2312514176 |
56+
57+
@DTOSS-7585 @Regression
58+
Scenario: Verify ADD records that trigger a non-fatal validation rule reach internal participant tables but not Cohort distribution
59+
Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
60+
When the file is uploaded to the Blob Storage container
61+
Then the Exception table should contain the below details for the NHS Number
62+
| FieldName | FieldValue |
63+
| RULE_ID | 36 |
64+
| RULE_DESCRIPTION | Invalid primary care provider GP practice code |
65+
Then the NHS Number should have exactly 1 record in Participant_Management
66+
And the NHS Number should have exactly 1 record in Participant_Demographic
67+
And the NHS Number should have exactly 0 record in Cohort_Distribution table
68+
69+
Examples:
70+
| AddFileName | NhsNumbers |
71+
| Exception_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2612314172 |
72+
73+
@DTOSS-7585 @Regression
74+
Scenario: Verify AMENDED records with non-fatal validation issues reach participant tables with partial Cohort distribution entries
75+
Given file <AddFileName> exists in the configured location for "Add" with NHS numbers : <NhsNumbers>
76+
And the file is uploaded to the Blob Storage container
77+
And the NHS numbers in the database should match the file data
78+
And file <AmendedFileName> exists in the configured location for "Amended" with NHS numbers : <NhsNumbers>
79+
When the file is uploaded to the Blob Storage container
80+
Then the Exception table should contain the below details for the NHS Number
81+
| FieldName | FieldValue |
82+
| RULE_ID | 17 |
83+
| RULE_DESCRIPTION | Date of birth invalid |
84+
Then the NHS Number should have exactly 1 record in Participant_Management
85+
And the NHS Number should have exactly 1 record in Participant_Demographic
86+
And the NHS Number should have exactly 1 record in Cohort_Distribution table
87+
88+
Examples:
89+
| AddFileName | AmendedFileName | NhsNumbers |
90+
| ADD_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | AMENDED_1B8F53_-_CAAS_BREAST_SCREENING_COHORT.parquet | 2612514171 |

‎tests/e2e-tests/NHS.CohortManager.EndToEndTests/Helpers/BlobStorageHelper.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
namespace NHS.CohortManager.EndToEndTests.Helpers;
2+
3+
using System.IO;
4+
using System.Threading.Tasks;
5+
using Azure.Storage.Blobs;
6+
using Microsoft.Extensions.Logging;
7+
8+
9+
10+
public class BlobStorageHelper
11+
{
12+
private readonly BlobServiceClient _blobServiceClient;
13+
private readonly ILogger<BlobStorageHelper> _logger;
14+
15+
16+
public BlobStorageHelper(BlobServiceClient blobServiceClient, ILogger<BlobStorageHelper> logger)
17+
{
18+
_blobServiceClient = blobServiceClient;
19+
_logger = logger;
20+
}
21+
22+
public async Task UploadFileToBlobStorageAsync(string filePath, string blobContainerName)
23+
{
24+
if (!File.Exists(filePath))
25+
{
26+
_logger.LogError($"File not found at {filePath}");
27+
return;
28+
}
29+
30+
_logger.LogInformation("Uploading file {FilePath} to blob storage", filePath);
31+
32+
var blobContainerClient = _blobServiceClient.GetBlobContainerClient(blobContainerName);
33+
await blobContainerClient.CreateIfNotExistsAsync();
34+
35+
var blobClient = blobContainerClient.GetBlobClient(Path.GetFileName(filePath));
36+
await blobClient.UploadAsync(File.OpenRead(filePath), true);
37+
38+
_logger.LogInformation("File uploaded successfully");
39+
}
40+
}

‎tests/e2e-tests/NHS.CohortManager.EndToEndTests/Helpers/CsvHelper.cs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
namespace NHS.CohortManager.EndToEndTests.Helpers;
2+
3+
using System.Collections.Generic;
4+
using System.Globalization;
5+
using System.IO;
6+
using CsvHelper;
7+
using CsvHelper.Configuration;
8+
9+
10+
11+
public static class CsvHelperService
12+
{
13+
public static List<string> ExtractNhsNumbersFromCsv(string filePath)
14+
{
15+
using var reader = new StreamReader(filePath);
16+
using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture));
17+
var records = new List<string>();
18+
csv.Read();
19+
csv.ReadHeader();
20+
while (csv.Read())
21+
{
22+
records.Add(csv.GetField(3));
23+
}
24+
return records;
25+
}
26+
27+
public static List<Dictionary<string, string>> ReadCsv(string filePath)
28+
{
29+
using var reader = new StreamReader(filePath);
30+
using var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture));
31+
var records = new List<Dictionary<string, string>>();
32+
csv.Read();
33+
csv.ReadHeader();
34+
var headers = csv.HeaderRecord;
35+
36+
while (csv.Read())
37+
{
38+
var record = new Dictionary<string, string>();
39+
foreach (var header in headers)
40+
{
41+
record[header] = csv.GetField(header);
42+
}
43+
records.Add(record);
44+
}
45+
46+
return records;
47+
}
48+
}
49+

0 commit comments

Comments
 (0)