APPT-2108: Lease factory introduce with async method

src/api/Nhs.Appointments.Api/FunctionConfigurationExtensions.cs

@@ -123,15 +123,8 @@
             .AddScoped<ILastUpdatedByResolver, LastUpdatedByResolver>()
             .AddTransient<IUserCsvWriter, UserCsvWriter>();
 
         var leaseManagerConnection = Environment.GetEnvironmentVariable("LEASE_MANAGER_CONNECTION");
-        if (leaseManagerConnection == "local")
-        {
-            builder.Services.AddInMemoryLeasing();
-        }
-        else
-        {
-            builder.Services.AddAzureBlobStoreLeasing(leaseManagerConnection, "leases");
-        }
+        builder.Services.AddConcurrency(configuration);
 
         builder.Services.AddHttpClient();
 

src/api/Nhs.Appointments.Core/Blob/AzureBlobStorage.cs

@@ -6,7 +6,7 @@ public class AzureBlobStorage(BlobServiceClient blobServiceClient) : IAzureBlobS
 {
     public async Task<Stream> GetBlobUploadStream(string containerName, string blobName)
     {
-        var blobClient = GetBlobClientFromContainerAndBlobName(containerName, blobName);
+        var blobClient = await GetBlobClientFromContainerAndBlobNameAsync(containerName, blobName);
 
         return await blobClient.OpenWriteAsync(true);
     }
@@ -18,6 +18,13 @@ public BlobClient GetBlobClientFromContainerAndBlobName(string containerName, st
         return containerClient.GetBlobClient(blobName);
     }
 
+    public async Task<BlobClient> GetBlobClientFromContainerAndBlobNameAsync(string containerName, string blobName)
+    {
+        var containerClient = await ResolveContainerClientAsync(containerName);
+
+        return containerClient.GetBlobClient(blobName);
+    }
+
     private BlobContainerClient ResolveContainerClient(string containerName)
     {
         var containers = blobServiceClient.GetBlobContainers();
@@ -26,4 +33,13 @@ private BlobContainerClient ResolveContainerClient(string containerName)
             ? blobServiceClient.GetBlobContainerClient(containerName) 
             : blobServiceClient.CreateBlobContainer(containerName);
     }
+
+    private async Task<BlobContainerClient> ResolveContainerClientAsync(string containerName)
+    {
+        var exists = await blobServiceClient.GetBlobContainersAsync().AnyAsync(x => x.Name.Equals(containerName));
+
+        return exists
+            ? blobServiceClient.GetBlobContainerClient(containerName) 
+            : await blobServiceClient.CreateBlobContainerAsync(containerName);
+    }
 }

src/api/Nhs.Appointments.Core/Blob/IAzureBlobStorage.cs

@@ -7,4 +7,5 @@ public interface IAzureBlobStorage
     Task<Stream> GetBlobUploadStream(string containerName, string blobName);
 
     BlobClient GetBlobClientFromContainerAndBlobName(string containerName, string blobName);
+    Task<BlobClient> GetBlobClientFromContainerAndBlobNameAsync(string containerName, string blobName);
 }

src/api/Nhs.Appointments.Core/Bookings/BookingWriteService.cs

@@ -48,7 +48,7 @@
     IBookingsDocumentStore bookingDocumentStore,
     IBookingQueryService bookingQueryService,
     IReferenceNumberProvider referenceNumberProvider,
-    ILeaseManager leaseManager,
+    ILeaseManagerFactory leaseManagerFactory,
     IBookingAvailabilityStateService bookingAvailabilityStateService,
     IBookingEventFactory eventFactory,
     IMessageBus bus,
@@ -64,7 +64,7 @@
 
         var leaseKey = LeaseKeys.SiteKeyFactory.Create(booking.Site, booking.Date);
 
-        using var leaseContent = leaseManager.Acquire(leaseKey);
+        using var leaseContent = leaseManagerFactory.Create().Acquire(leaseKey);
 
         var availableSlots = await bookingAvailabilityStateService.GetAvailableSlots(booking.Site, from, to);
 
@@ -358,7 +358,7 @@
 
         var leaseKey = LeaseKeys.SiteKeyFactory.Create(site, day);
 
-        using var leaseContent = leaseManager.Acquire(leaseKey);
+        using var leaseContent = leaseManagerFactory.Create().Acquire(leaseKey);
 
         var recalculations =
             (await bookingAvailabilityStateService.BuildRecalculations(site, bookingDayRange.Start, bookingDayRange.End,

src/api/Nhs.Appointments.Core/Concurrency/AzureStorageLeaseManager.cs

@@ -9,45 +9,64 @@
 internal class AzureStorageLeaseManager : ILeaseManager
 {
     private readonly IAzureBlobStorage _azureBlobStorage;
-    private readonly LeaseManagerOptions _options;
-    private readonly int _acquireTimeInSeconds;
+    private readonly LeaseManagerOptions _defaultOptions;
     private readonly int _delayRetryTimeInMilliseconds;
 
     public AzureStorageLeaseManager(
         IOptions<LeaseManagerOptions> options, 
         IAzureBlobStorage azureBlobStorage, 
-        int acquireTimeInSeconds = 20, 
         int delayRetryTimeInMilliseconds = 100
         )
     {
-        ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(acquireTimeInSeconds, 0, nameof(acquireTimeInSeconds));
         ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(delayRetryTimeInMilliseconds, 0, nameof(delayRetryTimeInMilliseconds));
 
         _azureBlobStorage = azureBlobStorage ?? throw new ArgumentNullException(nameof(azureBlobStorage));
-        _options = options.Value;
-        _acquireTimeInSeconds = acquireTimeInSeconds;
         _delayRetryTimeInMilliseconds = delayRetryTimeInMilliseconds;
+        _defaultOptions = options.Value;
     }
 
-    public ILeaseContext Acquire(string leaseKey)
+    public string Mode => LeaseManagerMode.DistributedAzureBlob;
+
+    public ILeaseContext Acquire(string leaseKey, LeaseManagerOptions options = null)
     {
-        var leaseClient = GetLeaseClient(leaseKey);
-        var leasePipeline = CreateResiliencePipeline();
-        leasePipeline.Execute(() => leaseClient.Acquire(TimeSpan.FromSeconds(_acquireTimeInSeconds)));
+        var leaseClient = GetLeaseClient(ResolveContainerName(options),leaseKey);
+        CreateResiliencePipeline().Execute(() => leaseClient.Acquire(ResolveTimeout(options)));
 
         return new LeaseContext(leaseKey, () => leaseClient.Release());
     }
+
+    public async Task<ILeaseContext> AcquireAsync(string leaseKey, LeaseManagerOptions options = null)
+    {
+        var leaseClient = await GetLeaseClientAsync(ResolveContainerName(options),leaseKey);
+        await CreateResiliencePipeline().ExecuteAsync(
+            async (cancellationToken) => await leaseClient.AcquireAsync(
+                ResolveTimeout(options), 
+                cancellationToken: cancellationToken));
 
-    private BlobLeaseClient GetLeaseClient(string blobName)
+        return new LeaseContext(leaseKey, () => leaseClient.Release());
+    }
+
+    private BlobLeaseClient GetLeaseClient(string containerName, string blobName)
     {
-        var blobClient = _azureBlobStorage.GetBlobClientFromContainerAndBlobName(_options.ContainerName, blobName);
+        var blobClient = _azureBlobStorage.GetBlobClientFromContainerAndBlobName(containerName, blobName);
         if (blobClient.Exists() == false)
         {
             blobClient.Upload(BinaryData.FromString(""));
         }
 
         return blobClient.GetBlobLeaseClient();
     }
+
+    private async Task<BlobLeaseClient> GetLeaseClientAsync(string containerName, string blobName)
+    {
+        var blobClient = await _azureBlobStorage.GetBlobClientFromContainerAndBlobNameAsync(containerName, blobName);
+        if (await blobClient.ExistsAsync() == false)
+        {
+            await blobClient.UploadAsync(BinaryData.FromString(""));
+        }
+
+        return blobClient.GetBlobLeaseClient();
+    }
 
     private ResiliencePipeline<Azure.Response<BlobLease>> CreateResiliencePipeline()
     {
@@ -63,5 +82,9 @@
                 Delay = TimeSpan.FromMilliseconds(_delayRetryTimeInMilliseconds)
             })
             .Build();
-    }                
+    }
+
+    private string ResolveContainerName(LeaseManagerOptions options = null) => options?.Realm ?? _defaultOptions.Realm;
+    private TimeSpan ResolveTimeout(LeaseManagerOptions options = null) =>
+        options?.Timeout ?? _defaultOptions.Timeout;
 }

src/api/Nhs.Appointments.Core/Concurrency/ILeaseManager.cs

@@ -3,5 +3,7 @@ namespace Nhs.Appointments.Core.Concurrency;
 
 public interface ILeaseManager
 {
-    ILeaseContext Acquire(string leaseKey);
+    string Mode { get; }
+    ILeaseContext Acquire(string leaseKey, LeaseManagerOptions options = null);
+    Task<ILeaseContext> AcquireAsync(string leaseKey, LeaseManagerOptions options = null);
 }

src/api/Nhs.Appointments.Core/Concurrency/ILeaseManagerFactory.cs

@@ -0,0 +1,6 @@
+namespace Nhs.Appointments.Core.Concurrency;
+
+public interface ILeaseManagerFactory
+{
+    ILeaseManager Create(string mode = null);
+}

src/api/Nhs.Appointments.Core/Concurrency/InMemoryLeaseManager.cs

@@ -5,18 +5,45 @@
 internal class InMemoryLeaseManager : ILeaseManager
 {
     private readonly Dictionary<string, SemaphoreSlim> _locks;
-    private readonly LeaseManagerOptions _options;
+    private readonly LeaseManagerOptions _defaultOptions;
 
     public InMemoryLeaseManager(IOptions<LeaseManagerOptions> options)
     {
         _locks = new Dictionary<string, SemaphoreSlim>();
-        _options = options.Value;
+        _defaultOptions = options.Value;
     }
 
-    public ILeaseContext Acquire(string leaseKey)
+    public string Mode => LeaseManagerMode.InMemory;
+
+    public ILeaseContext Acquire(string leaseKey, LeaseManagerOptions options = null)
     {
-        SemaphoreSlim mutex;
+        var inMemoryLeaseKey = BuildInMemoryKey(leaseKey, options);
+        var mutex = ResolveMutex(inMemoryLeaseKey, options);
+        if (!mutex.Wait(ResolveTimeout(options)))
+        {
+            throw new AbandonedMutexException($"Abandoned attempt to acquire lock for lease key {inMemoryLeaseKey}");
+        }
+
+        return new LeaseContext(inMemoryLeaseKey, () => mutex.Release());
+    }
+
+    public async Task<ILeaseContext> AcquireAsync(string leaseKey, LeaseManagerOptions options = null)
+    {
+        var inMemoryLeaseKey = BuildInMemoryKey(leaseKey, options);
+        var mutex = ResolveMutex(inMemoryLeaseKey, options);
+
+        if (!(await mutex.WaitAsync(ResolveTimeout(options))))
+        {
+            throw new AbandonedMutexException($"Abandoned attempt to acquire lock for lease key {inMemoryLeaseKey}");
+        }
+
+        return new LeaseContext(inMemoryLeaseKey, () => mutex.Release());
+    }
 
+    private SemaphoreSlim ResolveMutex(string leaseKey, LeaseManagerOptions options = null)
+    {
+        SemaphoreSlim mutex;
+
         lock (_locks)
         {
             if (!_locks.ContainsKey(leaseKey))
@@ -25,12 +52,12 @@
             }
             mutex = _locks[leaseKey];
         }
-
-        if (!mutex.Wait(_options.Timeout))
-        {
-            throw new AbandonedMutexException($"Abandoned attempt to acquire lock for lease key {leaseKey}");
-        }
-
-        return new LeaseContext(leaseKey, () => mutex.Release());
+
+        return mutex;
     }
+
+    private string BuildInMemoryKey(string leaseKey, LeaseManagerOptions options = null) =>
+        $"{options?.Realm ?? _defaultOptions.Realm}_{leaseKey}";
+    private TimeSpan ResolveTimeout(LeaseManagerOptions options = null) =>
+        options?.Timeout ?? _defaultOptions.Timeout;
 }

src/api/Nhs.Appointments.Core/Concurrency/LeaseManagerFactory.cs

@@ -0,0 +1,26 @@
+namespace Nhs.Appointments.Core.Concurrency;
+
+public class LeaseManagerFactory : ILeaseManagerFactory
+{
+    private readonly IEnumerable<ILeaseManager> _leaseManagers;
+
+    public LeaseManagerFactory(IEnumerable<ILeaseManager> leaseManagers)
+    {
+        _leaseManagers = leaseManagers;
+    }
+
+    public ILeaseManager Create(string mode = null)
+    {
+        if (string.IsNullOrEmpty(mode))
+        {
+            return ResolveDefault();
+        }
+
+        return _leaseManagers.SingleOrDefault(x => x.Mode.Equals(mode)) ?? throw new ArgumentException($"No lease manager found for mode: {mode}");
+    }
+
+    private ILeaseManager ResolveDefault() => 
+        _leaseManagers.SingleOrDefault(x => x.Mode.Equals(LeaseManagerMode.DistributedAzureBlob)) 
+        ?? _leaseManagers.SingleOrDefault(x => x.Mode.Equals(LeaseManagerMode.InMemory)) 
+        ?? throw new ArgumentException("No default lease manager found");
+}

src/api/Nhs.Appointments.Core/Concurrency/LeaseManagerMode.cs

@@ -0,0 +1,7 @@
+namespace Nhs.Appointments.Core.Concurrency;
+
+public static class LeaseManagerMode
+{
+    public const string InMemory = "InMemory";   
+    public const string DistributedAzureBlob = "DistributedAzureBlob";   
+}

src/api/Nhs.Appointments.Core/Concurrency/LeaseManagerOptions.cs

@@ -3,5 +3,5 @@ namespace Nhs.Appointments.Core.Concurrency;
 public class LeaseManagerOptions
 {
     public TimeSpan Timeout { get; set; }
-    public string ContainerName { get; set; }
+    public string Realm { get; set; }
 }

src/api/Nhs.Appointments.Core/Concurrency/ServiceRegistration.cs

@@ -1,31 +1,43 @@
 using Microsoft.Extensions.Azure;
+using Microsoft.Extensions.Configuration;
 using Nhs.Appointments.Core.Blob;
 using Nhs.Appointments.Core.Concurrency;
 
 namespace Microsoft.Extensions.DependencyInjection;
 
 public static class ServiceRegistration
 {
-    public static IServiceCollection AddInMemoryLeasing(this IServiceCollection services)
+    public static IServiceCollection AddConcurrency(this IServiceCollection services, IConfiguration configuration)
     {
-        services.Configure<LeaseManagerOptions>(opts => opts.Timeout = TimeSpan.FromSeconds(15));
-        return services.AddSingleton<ILeaseManager, InMemoryLeaseManager>();
+        var leaseManagerConnection = configuration.GetValue<string>("LEASE_MANAGER_CONNECTION");
+
+        services
+            .AddTransient<ILeaseManagerFactory, LeaseManagerFactory>()
+            .Configure<LeaseManagerOptions>(opts =>
+            {
+                opts.Timeout =
+                    TimeSpan.FromSeconds(configuration.GetValue("LEASE_MANAGER_DEFAULT_TIME_OUT_SECONDS", 15));
+                opts.Realm = configuration.GetValue("LEASE_MANAGER_DEFAULT_REALM", "leases");
+            })
+            .AddSingleton<ILeaseManager, InMemoryLeaseManager>();
+
+        if (!string.IsNullOrEmpty(leaseManagerConnection))
+        {
+            services.AddAzureBlobStoreLeasing(leaseManagerConnection);
+        }
+
+        return services;
     }
 
-    public static IServiceCollection AddAzureBlobStoreLeasing(this IServiceCollection services, string connectionString, string containerName)
+    private static IServiceCollection AddAzureBlobStoreLeasing(this IServiceCollection services, string connectionString)
     {
-        services.Configure<LeaseManagerOptions>(opts => { 
-            opts.Timeout = TimeSpan.FromSeconds(30);
-            opts.ContainerName = containerName;
-        });
-
         services.AddAzureClients(x =>
         {
             x.AddBlobServiceClient(connectionString);
         });
 
         return services
             .AddSingleton<IAzureBlobStorage, AzureBlobStorage>()
-            .AddSingleton<ILeaseManager, AzureStorageLeaseManager>();
+            .AddTransient<ILeaseManager, AzureStorageLeaseManager>();
     }
 }