feat: transform function skeleton and blob download functionality

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileParser.cs

@@ -0,0 +1,12 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
+
+public class FileParser : IFileParser
+{
+    public ParsedFile Parse(Stream stream)
+    {
+        // TODO - implement this
+        throw new NotImplementedException();
+    }
+}

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/FileTransformer.cs

@@ -0,0 +1,35 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation;
+using ServiceLayer.Mesh.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
+
+// TODO - NBSS appointment file specific implementation of IFileTransformer. To orchestrate parsing, validation and staging of data (delegated to separate classes)
+public class FileTransformer : IFileTransformer
+{
+    private readonly IFileParser _fileParser;
+    private readonly IValidationRunner _validationRunner;
+    private readonly IStagingPersister _stagingPersister;
+
+    public FileTransformer(IFileParser fileParser, IValidationRunner validationRunner, IStagingPersister stagingPersister)
+    {
+        _fileParser = fileParser;
+        _validationRunner = validationRunner;
+        _stagingPersister = stagingPersister;
+    }
+
+    public MeshFileType HandlesFileType => MeshFileType.NbssAppointmentEvents;
+
+    public async Task<IList<ValidationError>> TransformFileAsync(Stream stream, MeshFile metaData)
+    {
+        // TODO - consider whether we should wrap this parsing in a try-catch and return a List<ValidationError> in case of any unforeseen parsing issues (file is totally unlike anything we expect)
+        var parsed = _fileParser.Parse(stream);
+
+        var validationErrors = _validationRunner.Validate(parsed);
+        if (!validationErrors.Any())
+        {
+            await _stagingPersister.WriteStagedData(parsed);
+        }
+
+        return validationErrors;
+    }
+}

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/IFileParser.cs

@@ -0,0 +1,8 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
+
+public interface IFileParser
+{
+    ParsedFile Parse(Stream stream);
+}

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/IStagingPersister.cs

@@ -0,0 +1,9 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
+
+// TODO - interface for class to take validated AppointmentEventsFile and save the records to NbssAppointmentEvents table
+public interface IStagingPersister
+{
+    Task WriteStagedData(ParsedFile parsedFile);
+}

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/FileControlRecord.cs

@@ -0,0 +1,14 @@
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+public class FileControlRecord
+{
+    public string? RecordTypeIdentifier { get; set; }
+
+    public string? ExtractId { get; set; }
+
+    public string? TransferStartDate { get; set; }
+
+    public string? TransferStartTime { get; set; }
+
+    public string? RecordCount { get; set; }
+}

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/FileDataRecord.cs

@@ -0,0 +1,9 @@
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+public class FileDataRecord
+{
+    public int RowNumber { get; set; }
+    public Dictionary<string, string> Fields { get; } = [];
+
+    public string? this[string fieldName] => Fields.GetValueOrDefault(fieldName);
+}

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Models/ParsedFile.cs

@@ -0,0 +1,9 @@
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+public class ParsedFile
+{
+    public FileControlRecord? FileHeader { get; set; }
+    public FileControlRecord? FileTrailer { get; set; }
+    public required List<string> ColumnHeadings { get; set; } = [];
+    public required List<FileDataRecord> DataRecords { get; set; } = [];
+}

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/StagingPersister.cs

@@ -0,0 +1,13 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents;
+
+// TODO - class to take validated AppointmentEventsFile and save the records to NbssAppointmentEvents table
+public class StagingPersister : IStagingPersister
+{
+    public Task WriteStagedData(ParsedFile parsedFile)
+    {
+        // TODO - implement this
+        throw new NotImplementedException();
+    }
+}

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/IFileValidator.cs

@@ -0,0 +1,9 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation;
+
+// TODO - create a whole bunch of implementations of this to perform the validation against NBSS Appointment events files
+public interface IFileValidator
+{
+    IEnumerable<ValidationError> Validate(ParsedFile file);
+}

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/IRecordValidator.cs

@@ -0,0 +1,9 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation;
+
+// TODO - create a whole bunch of implementations of this to perform the validation against NBSS Appointment events records
+public interface IRecordValidator
+{
+    IEnumerable<ValidationError> Validate(FileDataRecord fileDataRecord);
+}

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/IValidationRunner.cs

@@ -0,0 +1,10 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation;
+
+public interface IValidationRunner
+{
+    IList<ValidationError> Validate(ParsedFile file);
+}
+
+

src/ServiceLayer.Mesh/FileTypes/NbssAppointmentEvents/Validation/ValidationRunner.cs

@@ -0,0 +1,29 @@
+using ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Models;
+
+namespace ServiceLayer.Mesh.FileTypes.NbssAppointmentEvents.Validation;
+
+public class ValidationRunner(
+    IEnumerable<IFileValidator> fileValidators,
+    IEnumerable<IRecordValidator> recordValidators)
+    : IValidationRunner
+{
+    public IList<ValidationError> Validate(ParsedFile file)
+    {
+        var errors = new List<ValidationError>();
+
+        foreach (var dataRecord in file.DataRecords)
+        {
+            foreach (var recordValidator in recordValidators)
+            {
+                errors.AddRange(recordValidator.Validate(dataRecord));
+            }
+        }
+
+        foreach (var validator in fileValidators)
+        {
+            errors.AddRange(validator.Validate(file));
+        }
+
+        return errors;
+    }
+}

src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs

@@ -1,6 +1,50 @@
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using ServiceLayer.Mesh.Data;
+using ServiceLayer.Mesh.Messaging;
+using ServiceLayer.Mesh.Models;
+using ServiceLayer.Mesh.Storage;
+
 namespace ServiceLayer.Mesh.Functions;
 
-public class FileTransformFunction
+public class FileTransformFunction(
+    ILogger<FileTransformFunction> logger,
+    ServiceLayerDbContext serviceLayerDbContext,
+    IMeshFilesBlobStore meshFileBlobStore)
 {
+    [Function("FileTransformFunction")]
+    public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQueueMessage message)
+    {
+        await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
+
+        var file = await serviceLayerDbContext.MeshFiles.FirstOrDefaultAsync(f => f.FileId == message.FileId);
+
+        if (file == null)
+        {
+            logger.LogWarning("File with id: {fileId} not found in MeshFiles table.", message.FileId);
+            return;
+        }
+
+        if (file.Status != MeshFileStatus.Extracted)
+        {
+            logger.LogWarning("File with id: {fileId} found in MeshFiles table but is not suitable for transformation. Status: {status}", message.FileId, file.Status);
+            return;
+        }
+
+        await UpdateFileStatusForTransformation(file);
+        await transaction.CommitAsync();
+
+        var fileContent = await meshFileBlobStore.DownloadAsync(file);
+
+        // TODO - take dependency on IEnumerable<IFileTransformer>.
+        // After initial common checks against database, find the appropriate implementation of IFileTransformer to handle the functionality that differs between file type.
+    }
 
+    private async Task UpdateFileStatusForTransformation(MeshFile file)
+    {
+        file.Status = MeshFileStatus.Transforming;
+        file.LastUpdatedUtc = DateTime.UtcNow;
+        await serviceLayerDbContext.SaveChangesAsync();
+    }
 }

src/ServiceLayer.Mesh/IFileTransformer.cs

@@ -0,0 +1,9 @@
+using ServiceLayer.Mesh.Models;
+
+namespace ServiceLayer.Mesh;
+
+public interface IFileTransformer
+{
+    MeshFileType HandlesFileType { get; }
+    Task<IList<ValidationError>> TransformFileAsync(Stream stream, MeshFile metaData);
+}

src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs

@@ -5,9 +5,17 @@ namespace ServiceLayer.Mesh.Storage;
 
 public class MeshFilesBlobStore(BlobContainerClient blobContainerClient) : IMeshFilesBlobStore
 {
-    public Task<Stream> DownloadAsync(MeshFile file)
+    public async Task<Stream> DownloadAsync(MeshFile file)
     {
-        throw new NotImplementedException();
+        var blobClient = blobContainerClient.GetBlobClient(file.BlobPath);
+
+        var dataStream = new MemoryStream();
+
+        await blobClient.DownloadToAsync(dataStream);
+
+        dataStream.Close();
+
+        return dataStream;
     }
 
     public async Task<string> UploadAsync(MeshFile file, byte[] data)

src/ServiceLayer.Mesh/ValidationError.cs

@@ -0,0 +1,12 @@
+namespace ServiceLayer.Mesh;
+
+public class ValidationError
+{
+    public int? RowNumber { get; set; }
+
+    public required string Field { get; set; }
+
+    public required string Code { get; set; }
+
+    public required string Error { get; set; }
+}

tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs

@@ -0,0 +1,126 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Moq;
+using ServiceLayer.Mesh.Data;
+using ServiceLayer.Mesh.Functions;
+using ServiceLayer.Mesh.Messaging;
+using ServiceLayer.Mesh.Models;
+using ServiceLayer.Mesh.Storage;
+
+namespace ServiceLayer.Mesh.Tests.Functions;
+
+public class FileTransformFunctionTests
+{
+    private readonly Mock<ILogger<FileTransformFunction>> _loggerMock;
+    private readonly Mock<IMeshFilesBlobStore> _blobStoreMock;
+    private readonly ServiceLayerDbContext _dbContext;
+    private readonly FileTransformFunction _function;
+
+    public FileTransformFunctionTests()
+    {
+        _loggerMock = new Mock<ILogger<FileTransformFunction>>();
+        _blobStoreMock = new Mock<IMeshFilesBlobStore>();
+
+        var options = new DbContextOptionsBuilder<ServiceLayerDbContext>()
+            .UseInMemoryDatabase(Guid.NewGuid().ToString())
+            .ConfigureWarnings(warnings =>
+                warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning))
+            .Options;
+        _dbContext = new ServiceLayerDbContext(options);
+
+        _function = new FileTransformFunction(
+            _loggerMock.Object,
+            _dbContext,
+            _blobStoreMock.Object
+        );
+    }
+
+    [Fact]
+    public async Task Run_FileNotFound_ExitsSilently()
+    {
+        // Arrange
+        var message = new FileTransformQueueMessage { FileId = "nonexistent-file" };
+
+        // Act
+        await _function.Run(message);
+
+        // Assert
+        _loggerMock.Verify(
+            x => x.Log(
+                LogLevel.Warning,
+                It.IsAny<EventId>(),
+                It.Is<It.IsAnyType>((v, t) => v.ToString() == $"File with id: {message.FileId} not found in MeshFiles table."),
+                null,
+                It.IsAny<Func<It.IsAnyType, Exception?, string>>()
+            ), Times.Once);
+
+        Assert.Equal(0, _dbContext.MeshFiles.Count());
+        _blobStoreMock.Verify(x => x.DownloadAsync(It.IsAny<MeshFile>()), Times.Never);
+    }
+
+    [Fact]
+    public async Task Run_FileStatusInvalid_ExitsSilently()
+    {
+        // Arrange
+        var file = new MeshFile
+        {
+            FileType = MeshFileType.NbssAppointmentEvents,
+            MailboxId = "test-mailbox",
+            FileId = "file-1",
+            Status = MeshFileStatus.FailedExtract, // Not eligible
+            LastUpdatedUtc = DateTime.UtcNow
+        };
+        _dbContext.MeshFiles.Add(file);
+        await _dbContext.SaveChangesAsync();
+
+        var message = new FileTransformQueueMessage { FileId = "file-1" };
+
+        // Act
+        await _function.Run(message);
+
+        // Assert
+        _loggerMock.Verify(
+            x => x.Log(
+                LogLevel.Warning,
+                It.IsAny<EventId>(),
+                It.Is<It.IsAnyType>((v, t) => v.ToString() == $"File with id: {message.FileId} found in MeshFiles table but is not suitable for transformation. Status: {file.Status}"),
+                null,
+                It.IsAny<Func<It.IsAnyType, Exception?, string>>()
+            ), Times.Once);
+        var fileFromDb = await _dbContext.MeshFiles.SingleOrDefaultAsync(x => x.FileId == file.FileId);
+        Assert.Equal(MeshFileStatus.FailedExtract, fileFromDb?.Status);
+        _blobStoreMock.Verify(x => x.DownloadAsync(It.IsAny<MeshFile>()), Times.Never);
+    }
+
+    [Fact]
+    public async Task Run_FileValid_DownloadsBlob()
+    {
+        // Arrange
+        var file = new MeshFile
+        {
+            FileType = MeshFileType.NbssAppointmentEvents,
+            MailboxId = "test-mailbox",
+            FileId = "file-1",
+            Status = MeshFileStatus.Extracted,
+            LastUpdatedUtc = DateTime.UtcNow
+        };
+        _dbContext.MeshFiles.Add(file);
+        await _dbContext.SaveChangesAsync();
+
+        var message = new FileTransformQueueMessage { FileId = "file-1" };
+
+        // Act
+        await _function.Run(message);
+
+        // Assert
+        _loggerMock.Verify(
+            x => x.Log(
+                LogLevel.Warning,
+                It.IsAny<EventId>(),
+                It.IsAny<It.IsAnyType>(),
+                null,
+                It.IsAny<Func<It.IsAnyType, Exception?, string>>()
+            ), Times.Never);
+        _blobStoreMock.Verify(x => x.DownloadAsync(file), Times.Once);
+    }
+}