fix(worker): debounce outages behind a consecutive-failure threshold

A single failed probe opened an outage immediately, so a transient CDN
5xx or a one-off timeout - routine on GitHub Pages - created and mailed
incidents several times a day. Require FailureThreshold (default 3)
consecutive down probes before opening; any reachable probe, operational
or degraded, resets the run. At the 60s interval this confirms a real
outage within about two extra minutes. Set the threshold to 1 to restore
opening on the first failed probe.

Author: daniel-miller

config/appsettings.json

@@ -28,6 +28,7 @@
       "IntervalSeconds": 60,
       "TimeoutSeconds": 5,
       "DegradedLatencyMs": 1000,
+      "FailureThreshold": 3,
       "HistoryBars": 60,
       "AbuseContact": "bump@example.com",
       "MaintenanceWindows": [

src/Bump.Api/BumpSettings.cs

@@ -22,6 +22,15 @@ public sealed class ServicesSettings
     /// <summary>Latency in milliseconds above which a healthy service is reported degraded.</summary>
     public int DegradedLatencyMs { get; set; } = 1000;
 
+    /// <summary>
+    /// Consecutive down probes required before an outage is opened. Debounces
+    /// transient single-probe failures (a one-off CDN 5xx or timeout) that would
+    /// otherwise open and close incidents several times a day. At the default 60s
+    /// interval, 3 confirms a real outage within about two extra minutes. Set to 1
+    /// to open on the first failed probe.
+    /// </summary>
+    public int FailureThreshold { get; set; } = 3;
+
     /// <summary>Number of history bars retained per service on the status page.</summary>
     public int HistoryBars { get; set; } = 60;
 

src/Bump.Api/Services/ServiceClassifier.cs

@@ -23,3 +23,33 @@ public static string Classify(int? statusCode, long? latencyMs, bool networkErro
         return ServiceStatuses.Operational;
     }
 }
+
+/// <summary>
+/// Decides when a run of failed probes has been confirmed long enough to open an
+/// outage. A single failed probe is not enough: transient blips (a CDN edge
+/// returning one 5xx, a one-off timeout — routine on GitHub Pages) would open and
+/// close incidents several times a day. Only a run of consecutive down probes that
+/// reaches the configured threshold counts as a real outage.
+/// </summary>
+public static class OutagePolicy
+{
+    /// <summary>
+    /// Count of trailing <c>down</c> probes at the end of the history. Any reachable
+    /// probe — <c>operational</c> or <c>degraded</c> — breaks the run and resets it to
+    /// zero, because a slow response is still a response.
+    /// </summary>
+    public static int TrailingDownStreak(IReadOnlyList<string> history)
+    {
+        int n = 0;
+        for (int i = history.Count - 1; i >= 0 && history[i] == ServiceStatuses.Down; i--) n++;
+        return n;
+    }
+
+    /// <summary>
+    /// True once the trailing down streak reaches <paramref name="failureThreshold"/>
+    /// consecutive probes. A threshold of 1 or less preserves the old behavior of
+    /// opening on the first down probe.
+    /// </summary>
+    public static bool OutageConfirmed(IReadOnlyList<string> history, int failureThreshold)
+        => TrailingDownStreak(history) >= Math.Max(1, failureThreshold);
+}

src/Bump.Worker/Services/ServiceProber.cs

@@ -25,6 +25,7 @@ public sealed class ServiceProber : BackgroundService
     private readonly TimeSpan _interval;
     private readonly TimeSpan _timeout;
     private readonly int _degradedLatencyMs;
+    private readonly int _failureThreshold;
     private readonly int _historyBars;
     private readonly string _publicBaseUrl;
     private readonly string? _alertRecipient;
@@ -54,6 +55,7 @@ public ServiceProber(
         _interval = settings.Interval;
         _timeout = settings.Timeout;
         _degradedLatencyMs = settings.DegradedLatencyMs;
+        _failureThreshold = settings.FailureThreshold;
         _historyBars = settings.HistoryBars;
         _publicBaseUrl = (config["Bump:Web:BaseUrl"] ?? "").TrimEnd('/');
         _alertRecipient = alerts.Contact;
@@ -151,10 +153,15 @@ private async Task ProbeOneAsync(Service service, HttpClient http, CancellationT
         int badCount = history.Count(s => s != ServiceStatuses.Operational);
         decimal uptimePct = Math.Max(95.00m, 100.00m - (decimal)badCount * 5m / _historyBars);
 
+        // Debounce: one failed probe does not open an outage. A transient blip
+        // (a single CDN 5xx or a one-off timeout, routine on GitHub Pages) would
+        // otherwise open and close incidents several times a day. Only a run of
+        // consecutive down probes reaching FailureThreshold counts. Any reachable
+        // probe in between resets the run.
         DateTimeOffset? lastOutageAt = state?.LastOutageAt;
-        bool needsOutage = status == ServiceStatuses.Down;
-        Outage? existingOutage = needsOutage ? await _outages.GetOpenForServiceAsync(service.ServiceId, ct) : null;
-        bool freshOutage = needsOutage && existingOutage is null;
+        bool outageConfirmed = OutagePolicy.OutageConfirmed(history, _failureThreshold);
+        Outage? existingOutage = outageConfirmed ? await _outages.GetOpenForServiceAsync(service.ServiceId, ct) : null;
+        bool freshOutage = outageConfirmed && existingOutage is null;
         if (freshOutage) lastOutageAt = DateTimeOffset.UtcNow;
 
         await _services.UpsertStateAsync(

tests/Bump.Api.Tests/OutagePolicyTests.cs

@@ -0,0 +1,75 @@
+using Bump.Api.Services;
+using Xunit;
+
+namespace Bump.Api.Tests;
+
+public sealed class OutagePolicyTests
+{
+    private static List<string> History(params string[] statuses) => statuses.ToList();
+
+    [Fact]
+    public void TrailingDownStreak_counts_only_the_trailing_run()
+    {
+        var history = History(
+            ServiceStatuses.Down,        // earlier run, does not count
+            ServiceStatuses.Operational, // breaks it
+            ServiceStatuses.Down,
+            ServiceStatuses.Down);
+        Assert.Equal(2, OutagePolicy.TrailingDownStreak(history));
+    }
+
+    [Fact]
+    public void Degraded_breaks_the_run()
+    {
+        // A slow-but-reachable probe is not down and must reset the streak.
+        var history = History(
+            ServiceStatuses.Down,
+            ServiceStatuses.Degraded,
+            ServiceStatuses.Down);
+        Assert.Equal(1, OutagePolicy.TrailingDownStreak(history));
+    }
+
+    [Fact]
+    public void Empty_history_has_no_streak()
+    {
+        Assert.Equal(0, OutagePolicy.TrailingDownStreak(new List<string>()));
+    }
+
+    [Theory]
+    [InlineData(1, 3, false)] // one failure, threshold 3 -> not yet
+    [InlineData(2, 3, false)] // two failures, threshold 3 -> not yet
+    [InlineData(3, 3, true)]  // threshold reached
+    [InlineData(4, 3, true)]  // still open past the threshold
+    public void OutageConfirmed_requires_threshold_consecutive_downs(int downCount, int threshold, bool expected)
+    {
+        var history = Enumerable.Repeat(ServiceStatuses.Down, downCount).ToList();
+        Assert.Equal(expected, OutagePolicy.OutageConfirmed(history, threshold));
+    }
+
+    [Fact]
+    public void A_single_operational_probe_before_the_run_still_confirms()
+    {
+        var history = History(
+            ServiceStatuses.Operational,
+            ServiceStatuses.Down,
+            ServiceStatuses.Down,
+            ServiceStatuses.Down);
+        Assert.True(OutagePolicy.OutageConfirmed(history, 3));
+    }
+
+    [Theory]
+    [InlineData(0)]
+    [InlineData(-5)]
+    public void Threshold_of_one_or_less_opens_on_the_first_down(int threshold)
+    {
+        var history = History(ServiceStatuses.Down);
+        Assert.True(OutagePolicy.OutageConfirmed(history, threshold));
+    }
+
+    [Fact]
+    public void Threshold_of_one_does_not_open_on_a_reachable_probe()
+    {
+        Assert.False(OutagePolicy.OutageConfirmed(History(ServiceStatuses.Degraded), 1));
+        Assert.False(OutagePolicy.OutageConfirmed(History(ServiceStatuses.Operational), 1));
+    }
+}