feat: responding to comments

Author: SamTyrrellNHS

.env.example

@@ -19,11 +19,12 @@ QueueUrl=http://127.0.0.1:10001
 FileExtractQueueName=file-extract
 FileTransformQueueName=file-transform
 StaleHours=12
-BlobContainerName=blob-container
+BlobContainerName=incoming-mesh-files
+
 
 # API Configuration
 API_PORT=7071
-MESH_PORT=7072
+MESH_INGEST_PORT=7072
 
 # Event Grid Configuration
 EVENT_GRID_TOPIC_URL=https://localhost:60101/api/events

compose.yaml

@@ -30,8 +30,8 @@ services:
     networks:
       - backend
 
-  service-layer-mesh:
-    container_name: "service-layer-mesh"
+  mesh-ingest:
+    container_name: "mesh-ingest"
     build:
       context: ./Src
       dockerfile: ServiceLayer.MESH/Dockerfile
@@ -58,7 +58,7 @@ services:
       ASPNETCORE_URLS: "http://0.0.0.0:8080"
       BlobContainerName: "${BlobContainerName}"
     ports:
-      - "${MESH_PORT}:8080"
+      - "${MESH_INGEST_PORT}:8080"
     healthcheck:
       test: ["CMD-SHELL", "curl -f http://localhost:8080/api/health || exit 1"]
       interval: 30s
@@ -93,14 +93,6 @@ services:
     networks:
       - backend
 
-  tester:
-    image: curlimages/curl
-    depends_on:
-      - azurite
-    command: ["sh", "-c", "sleep 10 && curl http://azurite:10000/devstoreaccount1"]
-    networks:
-      - backend
-
   db:
     container_name: "db"
     image: mcr.microsoft.com/mssql/server:2022-latest

src/ServiceLayer.Common/EnvironmentVariables.cs

@@ -0,0 +1,22 @@
+namespace ServiceLayer.Common;
+
+public static class EnvironmentVariables
+{
+    /// <summary>
+    /// Gets an environment variable by name. Throws if not found.
+    /// </summary>
+    /// <param name="key">The name of the environment variable.</param>
+    /// <returns>The value of the environment variable.</returns>
+    /// <exception cref="InvalidOperationException">Thrown when the variable is not found or is empty.</exception>
+    public static string GetRequired(string key)
+    {
+        var value = Environment.GetEnvironmentVariable(key);
+
+        if (string.IsNullOrEmpty(value))
+        {
+            throw new InvalidOperationException($"Environment variable '{key}' is not set or is empty.");
+        }
+
+        return value;
+    }
+}

src/ServiceLayer.Mesh/Configuration/AppConfiguration.cs

@@ -1,3 +1,5 @@
+using ServiceLayer.Common;
+
 namespace ServiceLayer.Mesh.Configuration;
 
 public class AppConfiguration :
@@ -19,12 +21,7 @@ public class AppConfiguration :
 
     private static string GetRequired(string key)
     {
-        var value = Environment.GetEnvironmentVariable(key);
-
-        if (string.IsNullOrEmpty(value))
-        {
-            throw new InvalidOperationException($"Environment variable '{key}' is not set or is empty.");
-        }
+        var value = EnvironmentVariables.GetRequired(key);
 
         return value;
     }

src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs

@@ -2,7 +2,6 @@
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
-using Microsoft.VisualBasic;
 using NHS.MESH.Client.Contracts.Services;
 using ServiceLayer.Data;
 using ServiceLayer.Data.Models;
@@ -24,30 +23,30 @@ public class FileExtractFunction(
     [Function("FileExtractFunction")]
     public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueMessage message)
     {
-        logger.LogInformation("{functionName} started at: {time}", nameof(FileExtractFunction), DateTime.UtcNow);
+        logger.LogInformation("{functionName} started.", nameof(FileExtractFunction));
 
         await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync();
 
-        var file = await GetFileAsync(serviceLayerDbContext, message.FileId);
+        var file = await GetFileAsync(message.FileId);
         if (file == null || !IsFileSuitableForExtraction(file))
         {
             return;
         }
 
-        await UpdateFileStatusForExtraction(serviceLayerDbContext, file);
+        await UpdateFileStatusForExtraction(file);
         await transaction.CommitAsync();
 
         try
         {
-            await ProcessFileExtraction(serviceLayerDbContext, file, message);
+            await ProcessFileExtraction(file, message);
         }
         catch (Exception ex)
         {
-            await HandleExtractionError(serviceLayerDbContext, file, message, ex);
+            await HandleExtractionError(file, message, ex);
         }
     }
 
-    private async Task<MeshFile?> GetFileAsync(ServiceLayerDbContext serviceLayerDbContext, string fileId)
+    private async Task<MeshFile?> GetFileAsync(string fileId)
     {
         var file = await serviceLayerDbContext.MeshFiles
             .FirstOrDefaultAsync(f => f.FileId == fileId);
@@ -78,14 +77,14 @@ private bool IsFileSuitableForExtraction(MeshFile file)
         return true;
     }
 
-    private async Task UpdateFileStatusForExtraction(ServiceLayerDbContext serviceLayerDbContext, MeshFile file)
+    private async Task UpdateFileStatusForExtraction(MeshFile file)
     {
         file.Status = MeshFileStatus.Extracting;
         file.LastUpdatedUtc = DateTime.UtcNow;
         await serviceLayerDbContext.SaveChangesAsync();
     }
 
-    private async Task ProcessFileExtraction(ServiceLayerDbContext serviceLayerDbContext, MeshFile file, FileExtractQueueMessage message)
+    private async Task ProcessFileExtraction(MeshFile file, FileExtractQueueMessage message)
     {
         var meshResponse = await meshInboxService.GetMessageByIdAsync(configuration.NbssMeshMailboxId, file.FileId);
         if (!meshResponse.IsSuccessful)
@@ -109,7 +108,7 @@ private async Task ProcessFileExtraction(ServiceLayerDbContext serviceLayerDbCon
         await fileTransformQueueClient.EnqueueFileTransformAsync(file);
     }
 
-    private async Task HandleExtractionError(ServiceLayerDbContext serviceLayerDbContext, MeshFile file, FileExtractQueueMessage message, Exception ex)
+    private async Task HandleExtractionError(MeshFile file, FileExtractQueueMessage message, Exception ex)
     {
         logger.LogError(ex, "An exception occurred during file extraction for fileId: {fileId}", message.FileId);
         file.Status = MeshFileStatus.FailedExtract;

src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs

@@ -22,11 +22,11 @@ public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myT
 
         var staleDateTimeUtc = DateTime.UtcNow.AddHours(-configuration.StaleHours);
 
-        await RetryStaleExtractions(serviceLayerDbContext, staleDateTimeUtc);
-        await RetryStaleTransformations(serviceLayerDbContext, staleDateTimeUtc);
+        await RetryStaleExtractions(staleDateTimeUtc);
+        await RetryStaleTransformations(staleDateTimeUtc);
     }
 
-    private async Task RetryStaleExtractions(ServiceLayerDbContext serviceLayerDbContext, DateTime staleDateTimeUtc)
+    private async Task RetryStaleExtractions(DateTime staleDateTimeUtc)
     {
         var staleFiles = await serviceLayerDbContext.MeshFiles
             .Where(f =>
@@ -45,7 +45,7 @@ private async Task RetryStaleExtractions(ServiceLayerDbContext serviceLayerDbCon
         }
     }
 
-    private async Task RetryStaleTransformations(ServiceLayerDbContext serviceLayerDbContext, DateTime staleDateTimeUtc)
+    private async Task RetryStaleTransformations(DateTime staleDateTimeUtc)
     {
         var staleFiles = await serviceLayerDbContext.MeshFiles
             .Where(f =>

src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs

@@ -34,7 +34,7 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
             return;
         }
 
-        await UpdateFileStatusForTransformation(serviceLayerDbContext, file);
+        await UpdateFileStatusForTransformation(file);
         await transaction.CommitAsync();
 
         var fileContent = await meshFileBlobStore.DownloadAsync(file);
@@ -44,7 +44,7 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu
         // to handle the functionality that differs between file types.
     }
 
-    private async Task UpdateFileStatusForTransformation(ServiceLayerDbContext serviceLayerDbContext, MeshFile file)
+    private async Task UpdateFileStatusForTransformation(MeshFile file)
     {
         file.Status = MeshFileStatus.Transforming;
         file.LastUpdatedUtc = DateTime.UtcNow;

src/ServiceLayer.Mesh/Program.cs

@@ -9,63 +9,60 @@
 using ServiceLayer.Mesh.Messaging;
 using ServiceLayer.Data;
 using ServiceLayer.Mesh.Storage;
+using ServiceLayer.Common;
 
 var host = new HostBuilder()
     .ConfigureFunctionsWebApplication()
     .ConfigureServices(services =>
     {
-        var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
+        var environment = EnvironmentVariables.GetRequired("ASPNETCORE_ENVIRONMENT");
         var isLocalEnvironment = environment == "Development";
 
         // MESH Client config
         services
-            .AddMeshClient(_ => _.MeshApiBaseUrl = Environment.GetEnvironmentVariable("MeshApiBaseUrl"))
-            .AddMailbox(Environment.GetEnvironmentVariable("NbssMailboxId"), new NHS.MESH.Client.Configuration.MailboxConfiguration
+            .AddMeshClient(_ => _.MeshApiBaseUrl = EnvironmentVariables.GetRequired("MeshApiBaseUrl"))
+            .AddMailbox(EnvironmentVariables.GetRequired("NbssMailboxId"), new NHS.MESH.Client.Configuration.MailboxConfiguration
             {
-                Password = Environment.GetEnvironmentVariable("MeshPassword"),
-                SharedKey = Environment.GetEnvironmentVariable("MeshSharedKey"),
+                Password = EnvironmentVariables.GetRequired("MeshPassword"),
+                SharedKey = EnvironmentVariables.GetRequired("MeshSharedKey"),
             }).Build();
 
         // EF Core DbContext
         services.AddDbContext<ServiceLayerDbContext>(options =>
         {
-            var connectionString = Environment.GetEnvironmentVariable("DatabaseConnectionString");
+            var connectionString = EnvironmentVariables.GetRequired("DatabaseConnectionString");
             if (string.IsNullOrEmpty(connectionString))
                 throw new InvalidOperationException("The connection string has not been initialized.");
 
             options.UseSqlServer(connectionString);
         });
 
+        var queueClientOptions = new QueueClientOptions
+        {
+            MessageEncoding = QueueMessageEncoding.Base64
+        };
+
         // Register QueueClients as singletons
         services.AddSingleton(provider =>
         {
             if (isLocalEnvironment)
             {
-                var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
-                return new QueueServiceClient(connectionString, new QueueClientOptions
-                {
-                    MessageEncoding = QueueMessageEncoding.Base64
-                });
+                var connectionString = EnvironmentVariables.GetRequired("AzureWebJobsStorage");
+                return new QueueServiceClient(connectionString, queueClientOptions);
             }
 
-            var meshStorageAccountUrl = Environment.GetEnvironmentVariable("MeshStorageAccountUrl");
-            return new QueueServiceClient(new Uri(meshStorageAccountUrl), new DefaultAzureCredential(), new QueueClientOptions
-            {
-                MessageEncoding = QueueMessageEncoding.Base64
-            });
+            var meshStorageAccountUrl = EnvironmentVariables.GetRequired("MeshStorageAccountUrl");
+            return new QueueServiceClient(new Uri(meshStorageAccountUrl), new DefaultAzureCredential(), queueClientOptions);
         });
 
         services.AddSingleton<IFileExtractQueueClient, FileExtractQueueClient>();
         services.AddSingleton<IFileTransformQueueClient, FileTransformQueueClient>();
 
         services.AddSingleton(provider =>
         {
-            var client = new BlobContainerClient(
-                Environment.GetEnvironmentVariable("AzureWebJobsStorage"),
-                Environment.GetEnvironmentVariable("BlobContainerName"));
-            client.CreateIfNotExistsAsync();
-            // might wanna make this async
-            return client;
+            return new BlobContainerClient(
+                EnvironmentVariables.GetRequired("AzureWebJobsStorage"),
+                EnvironmentVariables.GetRequired("BlobContainerName"));
         });
 
         services.AddSingleton<IMeshFilesBlobStore, MeshFilesBlobStore>();

src/ServiceLayer.Mesh/Storage/MeshFilesBlobStore.cs

@@ -3,23 +3,36 @@
 
 namespace ServiceLayer.Mesh.Storage;
 
-public class MeshFilesBlobStore(BlobContainerClient blobContainerClient) : IMeshFilesBlobStore
+public class MeshFilesBlobStore : IMeshFilesBlobStore
 {
+    private BlobContainerClient _blobContainerClient;
+
+    public MeshFilesBlobStore(BlobContainerClient blobContainerClient)
+    {
+        _blobContainerClient = blobContainerClient;
+        EnsureContainerExists();
+    }
+
     public async Task<Stream> DownloadAsync(MeshFile file)
     {
-        var blobClient = blobContainerClient.GetBlobClient(file.BlobPath);
+        var blobClient = _blobContainerClient.GetBlobClient(file.BlobPath);
         return (await blobClient.DownloadAsync()).Value.Content;
     }
 
     public async Task<string> UploadAsync(MeshFile file, byte[] data)
     {
         var blobPath = $"{file.FileType}/{file.FileId}";
-        var blobClient = blobContainerClient.GetBlobClient(blobPath);
+        var blobClient = _blobContainerClient.GetBlobClient(blobPath);
 
         var dataStream = new MemoryStream(data);
 
         await blobClient.UploadAsync(dataStream, overwrite: true);
 
         return blobPath;
     }
+
+    private void EnsureContainerExists()
+    {
+        _blobContainerClient.CreateIfNotExists();
+    }
 }