feat: Added AppConfiguration to FileTransformFunction

Author: alex-clayton-1

src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs

@@ -4,14 +4,17 @@ public class AppConfiguration :
     IFileDiscoveryFunctionConfiguration,
     IFileExtractFunctionConfiguration,
     IFileExtractQueueClientConfiguration,
-    IFileTransformQueueClientConfiguration
+    IFileTransformQueueClientConfiguration,
+    IFileTransformFunctionConfiguration
 {
     public string NbssMeshMailboxId => GetRequired("NbssMailboxId");
 
     public string FileExtractQueueName => GetRequired("FileExtractQueueName");
 
     public string FileTransformQueueName => GetRequired("FileTransformQueueName");
 
+    public int StaleHours => throw new NotImplementedException();
+
     private static string GetRequired(string key)
     {
         var value = Environment.GetEnvironmentVariable(key);

src/ServiceLayer.Mesh/Configuration/IFileTransformFunctionConfiguration.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Configuration;
+
+public interface IFileTransformFunctionConfiguration
+{
+    int StaleHours { get; }
+}

src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs

@@ -1,6 +1,8 @@
+using Google.Protobuf.WellKnownTypes;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
+using ServiceLayer.Mesh.Configuration;
 using ServiceLayer.Mesh.Data;
 using ServiceLayer.Mesh.Messaging;
 using ServiceLayer.Mesh.Models;
@@ -11,7 +13,8 @@ namespace ServiceLayer.Mesh.Functions;
 public class FileTransformFunction(
     ILogger<FileTransformFunction> logger,
     ServiceLayerDbContext serviceLayerDbContext,
-    IMeshFilesBlobStore meshFileBlobStore)
+    IMeshFilesBlobStore meshFileBlobStore,
+    IFileTransformFunctionConfiguration configuration)
 {
     [Function("FileTransformFunction")]
     public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQueueMessage message)
@@ -26,9 +29,8 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
             return;
         }
 
-        if (file.Status != MeshFileStatus.Extracted)
+        if (!IsFileSuitableForTransformation(file))
         {
-            logger.LogWarning("File with id: {fileId} found in MeshFiles table but is not suitable for transformation. Status: {status}", message.FileId, file.Status);
             return;
         }
 
@@ -47,4 +49,22 @@ private async Task UpdateFileStatusForTransformation(MeshFile file)
         file.LastUpdatedUtc = DateTime.UtcNow;
         await serviceLayerDbContext.SaveChangesAsync();
     }
+
+    private bool IsFileSuitableForTransformation(MeshFile file)
+    {
+        // We only want to transform files if they are in a Extracted state,
+        // or are in a Transforming state and were last touched over 12 hours ago.
+        var expectedStatuses = new[] { MeshFileStatus.Extracted, MeshFileStatus.Transforming };
+        if (!expectedStatuses.Contains(file.Status) ||
+            (file.Status == MeshFileStatus.Transforming && file.LastUpdatedUtc > DateTime.UtcNow.AddHours(-configuration.StaleHours)))
+        {
+            logger.LogWarning(
+                "File with id: {fileId} found in MeshFiles table but is not suitable for transformation. Status: {status}, LastUpdatedUtc: {lastUpdatedUtc}.",
+                file.FileId,
+                file.Status,
+                file.LastUpdatedUtc.ToTimestamp());
+            return false;
+        }
+        return true;
+    }
 }

src/ServiceLayer.Mesh/Program.cs

@@ -63,6 +63,7 @@
         services.AddTransient<IFileExtractFunctionConfiguration, AppConfiguration>();
         services.AddTransient<IFileExtractQueueClientConfiguration, AppConfiguration>();
         services.AddTransient<IFileTransformQueueClientConfiguration, AppConfiguration>();
+        services.AddTransient<IFileTransformFunctionConfiguration, AppConfiguration>();
     });
 
 

src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs

@@ -8,14 +8,7 @@ public class MeshFilesBlobStore(BlobContainerClient blobContainerClient) : IMesh
     public async Task<Stream> DownloadAsync(MeshFile file)
     {
         var blobClient = blobContainerClient.GetBlobClient(file.BlobPath);
-
-        var dataStream = new MemoryStream();
-
-        await blobClient.DownloadToAsync(dataStream);
-
-        dataStream.Close();
-
-        return dataStream;
+        return (await blobClient.DownloadAsync()).Value.Content;
     }
 
     public async Task<string> UploadAsync(MeshFile file, byte[] data)

tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs

@@ -1,6 +1,8 @@
+using Google.Protobuf.WellKnownTypes;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 using Moq;
+using ServiceLayer.Mesh.Configuration;
 using ServiceLayer.Mesh.Data;
 using ServiceLayer.Mesh.Functions;
 using ServiceLayer.Mesh.Messaging;
@@ -11,27 +13,28 @@ namespace ServiceLayer.Mesh.Tests.Functions;
 
 public class FileTransformFunctionTests
 {
-    private readonly Mock<ILogger<FileTransformFunction>> _loggerMock;
-    private readonly Mock<IMeshFilesBlobStore> _blobStoreMock;
+    private readonly Mock<ILogger<FileTransformFunction>> _loggerMock = new();
+    private readonly Mock<IMeshFilesBlobStore> _blobStoreMock = new();
+    private readonly Mock<IFileTransformFunctionConfiguration> _configuration = new();
     private readonly ServiceLayerDbContext _dbContext;
     private readonly FileTransformFunction _function;
 
     public FileTransformFunctionTests()
     {
-        _loggerMock = new Mock<ILogger<FileTransformFunction>>();
-        _blobStoreMock = new Mock<IMeshFilesBlobStore>();
-
         var options = new DbContextOptionsBuilder<ServiceLayerDbContext>()
             .UseInMemoryDatabase(Guid.NewGuid().ToString())
             .ConfigureWarnings(warnings =>
                 warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning))
             .Options;
         _dbContext = new ServiceLayerDbContext(options);
 
+        _configuration.Setup(c => c.StaleHours).Returns(12);
+
         _function = new FileTransformFunction(
             _loggerMock.Object,
             _dbContext,
-            _blobStoreMock.Object
+            _blobStoreMock.Object,
+            _configuration.Object
         );
     }
 
@@ -83,7 +86,7 @@ public async Task Run_FileStatusInvalid_ExitsSilently()
             x => x.Log(
                 LogLevel.Warning,
                 It.IsAny<EventId>(),
-                It.Is<It.IsAnyType>((v, t) => v.ToString() == $"File with id: {message.FileId} found in MeshFiles table but is not suitable for transformation. Status: {file.Status}"),
+                It.Is<It.IsAnyType>((v, t) => v.ToString() == $"File with id: {message.FileId} found in MeshFiles table but is not suitable for transformation. Status: {file.Status}, LastUpdatedUtc: {file.LastUpdatedUtc.ToTimestamp()}."),
                 null,
                 It.IsAny<Func<It.IsAnyType, Exception?, string>>()
             ), Times.Once);
@@ -92,6 +95,40 @@ public async Task Run_FileStatusInvalid_ExitsSilently()
         _blobStoreMock.Verify(x => x.DownloadAsync(It.IsAny<MeshFile>()), Times.Never);
     }
 
+    [Fact]
+    public async Task Run_FileStatusTransformingButNotTimedOut_ExitsSilently()
+    {
+        // Arrange
+        var file = new MeshFile
+        {
+            FileType = MeshFileType.NbssAppointmentEvents,
+            MailboxId = "test-mailbox",
+            FileId = "file-1",
+            Status = MeshFileStatus.Transforming,
+            LastUpdatedUtc = DateTime.UtcNow.AddHours(-11)  // Not timed out
+        };
+        _dbContext.MeshFiles.Add(file);
+        await _dbContext.SaveChangesAsync();
+
+        var message = new FileTransformQueueMessage { FileId = "file-1" };
+
+        // Act
+        await _function.Run(message);
+
+        // Assert
+        _loggerMock.Verify(
+            x => x.Log(
+                LogLevel.Warning,
+                It.IsAny<EventId>(),
+                It.Is<It.IsAnyType>((v, t) => v.ToString() == $"File with id: {message.FileId} found in MeshFiles table but is not suitable for transformation. Status: {file.Status}, LastUpdatedUtc: {file.LastUpdatedUtc.ToTimestamp()}."),
+                null,
+                It.IsAny<Func<It.IsAnyType, Exception?, string>>()
+            ), Times.Once);
+        var fileFromDb = await _dbContext.MeshFiles.SingleOrDefaultAsync(x => x.FileId == file.FileId);
+        Assert.Equal(MeshFileStatus.Transforming, fileFromDb?.Status);
+        _blobStoreMock.Verify(x => x.DownloadAsync(It.IsAny<MeshFile>()), Times.Never);
+    }
+
     [Fact]
     public async Task Run_FileValid_DownloadsBlob()
     {