Settable blocking bands, so the colours agree with the alerts (#6927)

Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Linq;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using Npgsql;
+using PerformanceMonitor.Common;
+using PerformanceMonitor.Darling.Service;
+using PerformanceMonitor.Darling.Service.Mcp;
+using PerformanceMonitor.Darling.Storage;
+using Xunit;
+
+namespace Darling.Tests;
+
+/// <summary>
+/// The blocking band tiers (#6927, V228) on a real PostgreSQL store: an upgraded store whose settings row already exists keeps
+/// today's defaults and its other values, and the tiers round trip through <c>update_alert_settings</c> and <c>get_alert_settings</c>
+/// with a critical bar below its warn bar refused and nothing written. Each fact mints its own scratch database.
+/// </summary>
+/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Each fact mints its own scratch database through
+   ScratchPostgres and never touches the shared store's tables. */
+public sealed class BlockingBandSettingsLiveTests
+{
+    private const string TiersSql = "SELECT blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1";
+
+    private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+
+    [Fact]
+    public async Task AnUpgradedStoreWithASettingsRow_KeepsTodaysDefaults_AndItsOtherValues()
+    {
+        var baseConnectionString = ConnectionString;
+        Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the V228 upgrade test.");
+        var ct = TestContext.Current.CancellationToken;
+
+        await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct);
+        await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+        await connection.OpenAsync(ct);
+
+        await PgMigrations.MigrateAsync(connection, ct);
+        await SeedAsync(scratch.ConnectionString, ct);
+
+        /* Roll the store back to look like V227: the settings row exists and carries an operator's own deadlock tier, the four
+           columns do not. */
+        await ExecAsync(connection, ct, "UPDATE config.config_alert_settings SET deadlock_warn_per_hour = 7.0, deadlock_critical_per_hour = 30.0 WHERE id = 1");
+        foreach (var column in new[] { "blocking_warn_per_hour", "blocking_critical_per_hour", "blocking_warn_wait_seconds", "blocking_critical_wait_seconds" })
+        {
+            await ExecAsync(connection, ct, "ALTER TABLE config.config_alert_settings DROP COLUMN " + column);
+        }
+
+        await ExecAsync(connection, ct, "DELETE FROM darling_schema_version WHERE version >= 228");
+
+        var applied = await PgMigrations.MigrateAsync(connection, ct);
+        Assert.Equal(PgMigrations.Scripts.Count(m => m.Version >= 228), applied);
+
+        Assert.Equal((5.0, 20.0, 10.0, 60.0), await ReadTiersAsync(connection, ct));
+
+        /* The band reads today's constants from the upgraded row, and the operator's deadlock tier did not move. */
+        await using var dataSource = NpgsqlDataSource.Create(scratch.ConnectionString);
+        Assert.Equal(BlockingBandThresholds.Default, await BlockingBandStore.ReadAsync(dataSource, 10, ct));
+        using var deadlock = new NpgsqlCommand("SELECT deadlock_warn_per_hour, deadlock_critical_per_hour FROM config_alert_settings WHERE id = 1", connection);
+        using var reader = await deadlock.ExecuteReaderAsync(ct);
+        Assert.True(await reader.ReadAsync(ct));
+        Assert.Equal(7.0, reader.GetDouble(0));
+        Assert.Equal(30.0, reader.GetDouble(1));
+    }
+
+    [Fact]
+    public async Task UpdateAlertSettings_SavesTheFourTiers_GetAlertSettingsReadsThemBack_AndACriticalBelowWarnIsRefused()
+    {
+        var baseConnectionString = ConnectionString;
+        Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the blocking band settings test.");
+        var ct = TestContext.Current.CancellationToken;
+
+        await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct);
+        await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+        await connection.OpenAsync(ct);
+        await PgMigrations.MigrateAsync(connection, ct);
+        await SeedAsync(scratch.ConnectionString, ct);
+
+        await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString);
+
+        /* The seeded row reads today's constants. */
+        AssertBands(await DarlingMcpAlertTools.GetAlertSettings(postgres), 5, 20, 10, 60);
+
+        /* A valid body saves, and a fresh read carries it. */
+        var saved = await DarlingMcpAlertTools.UpdateAlertSettings(postgres,
+            "{\"health_bands\":{\"blocking_warn_per_hour\":50,\"blocking_critical_per_hour\":100,\"blocking_warn_wait_seconds\":60,\"blocking_critical_wait_seconds\":180}}");
+        Assert.Equal("updated", DarlingMcpTestData.StatusOf(saved));
+        Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct));
+        AssertBands(await DarlingMcpAlertTools.GetAlertSettings(postgres), 50, 100, 60, 180);
+        Assert.Equal(new BlockingBandThresholds(50, 100, 60, 180), await BlockingBandStore.ReadAsync(postgres, 10, ct));
+
+        /* A critical bar below its warn bar is refused with nothing written: in one body, and as a lone value against the stored other one. */
+        foreach (var body in new[]
+        {
+            "{\"health_bands\":{\"blocking_warn_per_hour\":80,\"blocking_critical_per_hour\":30}}",
+            "{\"health_bands\":{\"blocking_critical_per_hour\":40}}",
+            "{\"health_bands\":{\"blocking_warn_wait_seconds\":200}}",
+            "{\"health_bands\":{\"blocking_warn_wait_seconds\":90,\"blocking_critical_wait_seconds\":30}}",
+        })
+        {
+            var refused = await DarlingMcpAlertTools.UpdateAlertSettings(postgres, body);
+            Assert.Equal("invalid", DarlingMcpTestData.StatusOf(refused));
+            Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct));
+        }
+
+        /* Out of range is refused too. */
+        Assert.Equal("invalid", DarlingMcpTestData.StatusOf(await DarlingMcpAlertTools.UpdateAlertSettings(postgres,
+            "{\"health_bands\":{\"blocking_warn_per_hour\":0}}")));
+        Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct));
+    }
+
+    private static void AssertBands(string settings, double warnPerHour, double criticalPerHour, double warnWait, double criticalWait)
+    {
+        Assert.False(McpHelpers.IsErrorEnvelope(settings), settings);
+        using var doc = JsonDocument.Parse(settings);
+        var bands = doc.RootElement.GetProperty("health_bands");
+        Assert.Equal(warnPerHour, bands.GetProperty("blocking_warn_per_hour").GetDouble());
+        Assert.Equal(criticalPerHour, bands.GetProperty("blocking_critical_per_hour").GetDouble());
+        Assert.Equal(warnWait, bands.GetProperty("blocking_warn_wait_seconds").GetDouble());
+        Assert.Equal(criticalWait, bands.GetProperty("blocking_critical_wait_seconds").GetDouble());
+    }
+
+    private static async Task<(double WarnPerHour, double CriticalPerHour, double WarnWait, double CriticalWait)> ReadTiersAsync(NpgsqlConnection connection, CancellationToken ct)
+    {
+        using var command = new NpgsqlCommand(TiersSql, connection);
+        using var reader = await command.ExecuteReaderAsync(ct);
+        Assert.True(await reader.ReadAsync(ct), "config_alert_settings has no singleton row");
+        return (reader.GetDouble(0), reader.GetDouble(1), reader.GetDouble(2), reader.GetDouble(3));
+    }
+
+    private static async Task ExecAsync(NpgsqlConnection connection, CancellationToken ct, string sql)
+    {
+        using var command = new NpgsqlCommand(sql, connection);
+        await command.ExecuteNonQueryAsync(ct);
+    }
+
+    /// <summary>Seeds the singleton rows through the product's own path, so this test need not track their other NOT NULL columns.</summary>
+    private static async Task SeedAsync(string connectionString, CancellationToken ct)
+    {
+        await using var dataSource = NpgsqlDataSource.Create(connectionString);
+        await new StoreConfigProvider(dataSource).SeedIfEmptyAsync(new DarlingConfig(), ct);
+    }
+}

Darling/Darling.Tests/BlockingBandTiersTests.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Collections.Generic;
+using PerformanceMonitor.Common;
+using PerformanceMonitor.Darling.Service.Mcp;
+using Xunit;
+
+namespace PerformanceMonitor.Darling.Tests;
+
+/// <summary>#6927: the blocking band reads settable tiers whose defaults are the constants every earlier build used.</summary>
+public sealed class BlockingBandTiersTests
+{
+    private static readonly TimeSpan Day = TimeSpan.FromHours(24);
+
+    [Fact]
+    public void Defaults_EqualTheCompileTimeConstants()
+    {
+        var d = BlockingBandThresholds.Default;
+        Assert.Equal(ServerHealthThresholds.BlockingWarnPerHour, d.WarnPerHour);
+        Assert.Equal(ServerHealthThresholds.BlockingCriticalPerHour, d.CriticalPerHour);
+        Assert.Equal(ServerHealthThresholds.BlockingWarnWaitSeconds, d.WarnWaitSeconds);
+        Assert.Equal(ServerHealthThresholds.BlockingCriticalWaitSeconds, d.CriticalWaitSeconds);
+        Assert.Equal(
+            ServerHealthClassifier.BlockingSeverity(1434, 162.0, Day),
+            ServerHealthClassifier.BlockingSeverity(1434, 162.0, Day, d));
+    }
+
+    [Fact]
+    public void ADayTunedTo100EventsAnd180Seconds_IsNotCritical_ButIsUnderTheDefaults()
+    {
+        /* 2026-10-23 in the issue: 1,434 reports over a day is 59.75 an hour here (73.5 on the day's own window), peak block 2.7 minutes. */
+        const long reports = 1764; // 73.5 an hour over 24 hours
+        const double peakSeconds = 162.0; // 2.7 minutes
+
+        Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(reports, peakSeconds, Day));
+
+        var tuned = new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0);
+        Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(reports, peakSeconds, Day, tuned));
+        Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(2400, 100.0, Day, tuned)); // 100 an hour
+        Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(1, 180.0, Day, tuned));
+    }
+
+    [Fact]
+    public void TheDailyClassifier_UsesTheTunedTiers()
+    {
+        var signals = new DailyHealthSignals
+        {
+            HasData = true,
+            Window = Day,
+            BlockingReports = 1764,
+            BlockingEvents = 1764,
+            PeakBlockWaitMs = 162_000,
+        };
+
+        Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(signals));
+        var tuned = new DailyHealthThresholds { BlockingBands = new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0) };
+        Assert.NotEqual(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(signals, tuned));
+    }
+
+    [Fact]
+    public void RawValuesClampToTheFloorsAndCeilings_AndADefaultStructClampsToTheFloorNotZero()
+    {
+        var wild = new BlockingBandThresholds(-5, double.NaN, 0, 1e12);
+        Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, wild.WarnPerHour);
+        Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, wild.CriticalPerHour);
+        Assert.Equal(ServerHealthThresholds.BlockingWaitSecondsFloor, wild.WarnWaitSeconds);
+        Assert.Equal(ServerHealthThresholds.BlockingWaitSecondsCeiling, wild.CriticalWaitSeconds);
+        Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, default(BlockingBandThresholds).WarnPerHour);
+    }
+
+    private static FleetServerCard CardAt(int id, HealthSeverity blocking, HealthSeverity deadlock) => new()
+    {
+        ServerId = id,
+        DisplayName = "s" + id,
+        ServerName = "s" + id,
+        BlockingCount = 3,
+        BlockingSeverity = blocking,
+        DeadlockSeverity = deadlock,
+    };
+
+    [Fact]
+    public void TheFleetTotalsTiles_TakeTheWorstCardBand_SoTenServersAtThreeAnHourAreNotCritical()
+    {
+        var now = new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc);
+        var empty = DarlingFleetReader.BuildRollup(Array.Empty<FleetServerCard>(), now, now.AddHours(-1), now);
+        Assert.Equal(HealthSeverity.Healthy, empty.TotalBlockingSeverity);
+        Assert.Equal(HealthSeverity.Healthy, empty.TotalDeadlockSeverity);
+
+        /* Ten servers each at 3 reports an hour: no card is even Warning at the default 5 an hour, and the fleet sum of 30 an hour
+           must not band as one server's rate. */
+        var window = TimeSpan.FromHours(1);
+        var cards = new List<FleetServerCard>();
+        for (var i = 1; i <= 10; i++)
+        {
+            var sev = ServerHealthClassifier.BlockingSeverity(3, 0.0, window, BlockingBandThresholds.Default);
+            Assert.Equal(HealthSeverity.Healthy, sev);
+            cards.Add(CardAt(i, sev, HealthSeverity.Healthy));
+        }
+        Assert.Equal(30, DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now).TotalBlockingEvents);
+        Assert.Equal(HealthSeverity.Healthy, DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now).TotalBlockingSeverity);
+
+        cards[4] = CardAt(5, HealthSeverity.Critical, HealthSeverity.Warning);
+        var one = DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now);
+        Assert.Equal(HealthSeverity.Critical, one.TotalBlockingSeverity);
+        Assert.Equal(HealthSeverity.Warning, one.TotalDeadlockSeverity);
+    }
+}

Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs

@@ -23,7 +23,7 @@ namespace Darling.Tests;
 /// #5640 (Lane E1): the Darling Viewer's Client Timeouts settings, the V179 twin of
 /// <c>RunnableQueueSettingsSurfaceTests</c>. The row carries the engine's defaults (ON, 10, 60), compares and saves
 /// all three columns through the engine's clamp, and the Settings window names the alert. V179 is the newest append, so
-/// this class owns the settings row's total width (86). The viewer needs no per-version gate for the columns: it refuses a
+/// this class owns the settings row's total width (90). The viewer needs no per-version gate for the columns: it refuses a
 /// store below <c>StorageVersion.SchemaVersion</c> at connect, as it did for V177's.
 /// </summary>
 public sealed class ClientTimeoutsSettingsSurfaceTests
@@ -72,7 +72,7 @@ public void TheUpsertAndTheSelect_CarryTheThreeColumns_AtTheEnd()
 
     /// <summary>
     /// V179's three columns sit at the FIXED ordinals 73, 74 and 75 (parameters $74 to $76) on every positional
-    /// settings-row surface, and the row is 86 columns wide (V200 appended the excluded volumes at 76, V211 the two multiples at 77 and 78, V222 the Agent alert gaps switch at 79, V223 the three database protection switches at 80 to 82, V224 the three identity capacity columns at 83 to 85): the viewer's select and upsert, the service's own read and
+    /// settings-row surface, and the row is 90 columns wide (V228 appended the four blocking band tiers at 86 to 89, V200 appended the excluded volumes at 76, V211 the two multiples at 77 and 78, V222 the Agent alert gaps switch at 79, V223 the three database protection switches at 80 to 82, V224 the three identity capacity columns at 83 to 85): the viewer's select and upsert, the service's own read and
     /// its seed, and the MCP read with its positional record. This is the test that was red while the seed had 76 columns
     /// and the viewer 73.
     /// </summary>
@@ -82,7 +82,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co
         var viewerColumns = ViewerDataService.AlertSettingsSelectSql
             .Replace("SELECT ", "", StringComparison.Ordinal).Split(" FROM ")[0]
             .Split(',').Select(c => c.Trim()).ToList();
-        Assert.Equal(86, viewerColumns.Count);
+        Assert.Equal(90, viewerColumns.Count);
         Assert.Equal(Expected, viewerColumns.Skip(73).Take(3).ToArray());
 
         var service = File.ReadAllText(Path.Combine(RepoRoot(), "Darling", "PerformanceMonitor.Darling.Service", "StoreConfigProvider.cs"));
@@ -91,7 +91,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co
         var select = service[start..];
         select = select[(select.IndexOf("SELECT", StringComparison.Ordinal) + 6)..select.IndexOf("FROM config_alert_settings WHERE id = 1", StringComparison.Ordinal)];
         var serviceColumns = select.Split(',').Select(c => c.Trim()).Where(c => c.Length > 0).ToList();
-        Assert.Equal(86, serviceColumns.Count);
+        Assert.Equal(90, serviceColumns.Count);
         Assert.Equal(Expected, serviceColumns.Skip(73).Take(3).ToArray());
         Assert.Contains("ClientTimeoutsEnabled = reader.GetBoolean(73),", service, StringComparison.Ordinal);
         Assert.Contains("ClientTimeoutsWarningPerMinute = reader.GetInt32(74),", service, StringComparison.Ordinal);
@@ -107,10 +107,10 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co
         var mcpColumns = DarlingAlertReader.AlertSettingsSelectSql
             .Split("FROM config_alert_settings")[0].Replace("SELECT", "", StringComparison.Ordinal)
             .Split(',').Select(c => c.Trim()).Where(c => c.Length > 0).ToList();
-        Assert.Equal(86, mcpColumns.Count);
+        Assert.Equal(90, mcpColumns.Count);
         Assert.Equal(Expected, mcpColumns.Skip(73).Take(3).ToArray());
         var parameters = typeof(DarlingAlertReader.AlertSettingsReadRow).GetConstructors().Single().GetParameters();
-        Assert.Equal(86, parameters.Length);
+        Assert.Equal(90, parameters.Length);
         Assert.Equal(new[] { "ClientTimeoutsEnabled", "ClientTimeoutsWarningPerMinute", "ClientTimeoutsCriticalPerMinute" },
             parameters.Skip(73).Take(3).Select(p => p.Name!).ToArray());
         Assert.Equal(typeof(bool), parameters[73].ParameterType);
@@ -120,7 +120,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co
         /* The viewer's bind supplies $74 to $76 at indexes 73 to 75, in column order (V200's excluded_volumes follows at 76). */
         var row = new AlertSettingsRow { ClientTimeoutsEnabled = true, ClientTimeoutsWarningPerMinute = 5, ClientTimeoutsCriticalPerMinute = 11 };
         var command = Bind(row);
-        Assert.Equal(86, command.Parameters.Count);
+        Assert.Equal(90, command.Parameters.Count);
         Assert.Equal(true, command.Parameters[73].Value);
         Assert.Equal(5, command.Parameters[74].Value);
         Assert.Equal(11, command.Parameters[75].Value);

Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs

@@ -1333,7 +1333,8 @@ dropped the columns and fell back to the engine defaults still match. Warning is
         ClientTimeoutsWarningMultiple: 2, ClientTimeoutsCriticalMultiple: 7,
         AgentAlertGapsEnabled: true,
         BackupOverdueEnabled: true, LogBackupOverdueEnabled: true, CheckDbOverdueEnabled: true,
-        IdentityCapacityEnabled: true, IdentityCapacityWarningPercent: 72, IdentityCapacityCriticalPercent: 92);
+        IdentityCapacityEnabled: true, IdentityCapacityWarningPercent: 72, IdentityCapacityCriticalPercent: 92,
+        BlockingWarnPerHour: 6.5, BlockingCriticalPerHour: 22.5, BlockingWarnWaitSeconds: 11.5, BlockingCriticalWaitSeconds: 62.5);
 
     [Fact]
     public void AlertSettingsSql_ReadsSingleGlobalRow()