Notification routes: match on alert severity (#6928)

Darling/Darling.Tests/AdminPageTests.cs

@@ -43,7 +43,7 @@ public void TheServersTab_ReadsTheAdminRoute_NotListServers_AndDrawsDisabledRows
 
     [Theory]
     [InlineData("SERVER_COLUMNS", new[] { "display_name", "server_name", "auth", "engine", "version", "status", "freshness", "monthly_cost", "read_only", "added", "last_collected", "same_machine_as" })]
-    [InlineData("ROUTE_COLUMNS", new[] { "route_id", "enabled", "metric_match", "match_kind", "family", "applies_to", "status", "channels", "agent", "resolutions", "smtp_recipients", "modified_at_utc" })]
+    [InlineData("ROUTE_COLUMNS", new[] { "route_id", "enabled", "metric_match", "match_kind", "family", "applies_to", "severities", "status", "channels", "agent", "resolutions", "smtp_recipients", "modified_at_utc" })]
     [InlineData("SETTING_COLUMNS", new[] { "setting", "value" })]
     public void EachTab_DeclaresItsColumnKeys(string name, string[] keys)
     {

Darling/Darling.Tests/AdminPendingNotesTests.cs

@@ -120,6 +120,54 @@ public void AddingARouteWithNoMatch_IsRefusedByTheFormBeforeARequest_Pure()
         Assert.Equal("blocking", named.GetProperty("payload").GetProperty("metric_match").GetString());
     }
 
+    /* #6928: the Routes list names the severity filter in the desktop window's words (Both when the field is missing or unknown), the editor
+       reads it back from a route and the save sends it, and a word the page does not know is sent as both. */
+    [Fact]
+    public void TheRoutesList_ShowsTheSeverityBesideTheScope_AndTheEditorRoundTripsIt_Pure()
+    {
+        var data = JsonNode.Parse("""
+            {"routes":[
+              {"route_id":1,"metric_match":"blocking","severities":"critical","enabled":true},
+              {"route_id":2,"metric_match":"performance","severities":"warning","enabled":true},
+              {"route_id":3,"metric_match":"reports","severities":"both","enabled":true},
+              {"route_id":4,"metric_match":"agent-jobs","enabled":true},
+              {"route_id":5,"metric_match":"waits","severities":"someday","enabled":true}
+            ]}
+            """);
+        var rows = Pure("routeRows", data).EnumerateArray().ToArray();
+        Assert.Equal(new[] { "Critical only", "Warning only", "Both", "Both", "Both" }, rows.Select(r => r.GetProperty("severities").GetString()));
+
+        var columns = Pure("ROUTE_COLUMNS").EnumerateArray().Select(c => c.GetProperty("key").GetString()).ToList();
+        Assert.Equal(columns.IndexOf("applies_to") + 1, columns.IndexOf("severities"));
+
+        Assert.Equal("critical", Pure("entriesFromRoute", new { route_id = 1, metric_match = "blocking", severities = "critical" }).GetProperty("severities").GetString());
+        Assert.Equal("both", Pure("entriesFromRoute", new { route_id = 1, metric_match = "blocking" }).GetProperty("severities").GetString());
+        Assert.Equal("both", Pure("entriesFromRoute", (object?)null).GetProperty("severities").GetString());
+
+        string Sent(string severities) => Pure("routePayload", new { id = (int?)null, match = "blocking", perHour = "10", perDay = "50", lease = "30", expiry = "120", fallback = "", scope = "", level = "", resolutions = "", enabled = true, severities })
+            .GetProperty("payload").GetProperty("severities").GetString()!;
+        Assert.Equal("warning", Sent("warning"));
+        Assert.Equal("critical", Sent("critical"));
+        Assert.Equal("both", Sent("both"));
+        Assert.Equal("both", Sent("someday"));
+    }
+
+    /* #6936: Edit then Save on the Routes tab sends the version the form read (expected_modified_at), because the service refuses an edit
+       without it with a 400. The test drives the page itself: it opens the editor on a stored Critical only route, picks Warning only,
+       presses Save and reads the write the page made. */
+    [Fact]
+    public void SavingAnEditedRoute_SendsTheVersionTheFormRead_AndTheChosenSeverity()
+    {
+        var writes = AdminServerEditBehaviourTests.Run("routeEdit:warning").GetProperty("writes").EnumerateArray().ToArray();
+
+        var write = Assert.Single(writes);
+        Assert.Equal("/api/notification-routes/7", write.GetProperty("url").GetString());
+        var body = write.GetProperty("body");
+        Assert.Equal("2026-01-01T00:00:00.1234567Z", body.GetProperty("expected_modified_at").GetString());
+        Assert.Equal("warning", body.GetProperty("severities").GetString());
+        Assert.Equal("blocking", body.GetProperty("metric_match").GetString());
+    }
+
     [Fact]
     public void OnlyASeatThatCanEdit_GetsTheDecideColumn_Pure()
     {

Darling/Darling.Tests/AgentActionGateSeverityLiveTests.cs

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging.Abstractions;
+using Npgsql;
+using PerformanceMonitor.Common;
+using PerformanceMonitor.Darling.Service.Mcp;
+using PerformanceMonitor.Darling.Storage;
+using Xunit;
+
+namespace Darling.Tests;
+
+/* #1776 own-store: the fact mints its own scratch database through ScratchPostgres, so the class is deliberately NOT [Collection("live-postgres")]. */
+
+/// <summary>
+/// #6928: the agent-action gate re-resolves a request's routes at the tier each alert arrived at, so a Critical alert dispatched under a
+/// fleet route is not refused because a server route that takes only Warning (agent Off) would have won the lookup with no tier.
+/// </summary>
+public sealed class AgentActionGateSeverityLiveTests
+{
+    private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+
+    [Fact]
+    public async Task ACriticalAlertDispatchedUnderTheFleetRoute_IsNotRefusedByAWarningOnlyServerRouteThatIsOff()
+    {
+        Assert.SkipWhen(string.IsNullOrEmpty(ConnectionString), "Set DARLING_TEST_PG to run the agent gate severity pin (it mints its own scratch database).");
+        var ct = TestContext.Current.CancellationToken;
+
+        await using var scratch = await ScratchPostgres.CreateAsync(ConnectionString!, ct);
+        await using (var migrate = new NpgsqlConnection(scratch.ConnectionString))
+        {
+            await migrate.OpenAsync(ct);
+            await PgMigrations.MigrateAsync(migrate, ct);
+        }
+
+        await using var owner = NpgsqlDataSource.Create(scratch.ConnectionString);
+        var bodySucceeded = false;
+        try
+        {
+            var proxies = new WebhookProxies(null, null, null, null);
+            /* A server-scoped Blocking route that takes Warning only, with the agent Off, and a fleet-wide Blocking route with the agent at Act. */
+            await using (var seed = owner.CreateCommand("INSERT INTO config.config_monitored_servers (server_id, name, host) VALUES (7, 'srv-7', 'host-a') ON CONFLICT DO NOTHING"))
+            {
+                await seed.ExecuteNonQueryAsync(ct);
+            }
+
+            await NotificationRouteWriter.SaveAsync(
+                owner, new NotificationRouteDraft { MetricMatch = "Blocking", ScopeServerId = 7, Severities = "warning", AgentLevel = 0 },
+                null, proxies, null, null, 30, ct);
+            await NotificationRouteWriter.SaveAsync(
+                owner, new NotificationRouteDraft { MetricMatch = "Blocking", AgentLevel = 2 }, null, proxies, null, null, 30, ct);
+
+            var store = new InvestigationRequestStore(owner, 30);
+            /* A Critical alert dispatches under the fleet route: request level 2. */
+            var opened = await store.OpenOrJoinAsync(
+                new InvestigationAlert(1, 7, "srv-7", "blocking", "Blocking", DateTime.UtcNow, Array.Empty<string>(), null, 20, 100, LeaseMinutes: 15, ExpiryMinutes: 240)
+                { AutonomyLevel = 2, Severity = "critical" }, ct);
+            var request = opened.RequestId!.Value;
+            var claim = (await store.ClaimAsync(request, "agent-a", ct)).ClaimId!.Value;
+
+            var writes = 0;
+            var hooks = new AgentActionHooks
+            {
+                LoadLevels = (connection, _, token) => DarlingAgentActionGate.LoadLevelsAsync(connection, NullLogger.Instance, token),
+                ReadBefore = (_, _, _) => Task.FromResult<string?>(null),
+                Write = (_, _, _) => { writes++; return Task.FromResult<string?>("{}"); },
+            };
+            var call = new AgentActionCall("create_mute_rule", "mute", new AgentAction(request, claim, "agent-a", "known batch",
+                "[{\"tool\":\"get_blocking\",\"arguments\":{},\"key_numbers\":\"12\"}]"), new AgentActionTarget(7, "srv-7", null, null));
+
+            var done = await store.RunAgentActionAsync(call, hooks, ct);
+
+            Assert.True(done.Done, done.Refusal?.Message);
+            Assert.Equal(1, writes);
+            bodySucceeded = true;
+        }
+        finally
+        {
+            await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, static (_, _) => Task.CompletedTask);
+        }
+    }
+}

Darling/Darling.Tests/AgentActionGateTests.cs

@@ -218,7 +218,7 @@ private static InvestigationAlert Alert(DateTime at, int level = 2, string metri
 
     private AgentActionHooks Hooks(int? resolved = 2, AgentActionRefusal? kind = null) => new()
     {
-        LoadLevels = (_, _, _) => Task.FromResult<Func<string, int, int?>>((_, _) => resolved),
+        LoadLevels = (_, _, _) => Task.FromResult<Func<string, int, PerformanceMonitor.Notifications.AlertSeverityLevel?, int?>>((_, _, _) => resolved),
         KindRule = kind is null ? null : (_, _, _, _) => Task.FromResult<AgentActionRefusal?>(kind),
         ReadBefore = (_, _, _) => Task.FromResult<string?>("{\"reason\":\"old\"}"),
         Write = (_, _, _) => { _writes++; return Task.FromResult<string?>("{\"reason\":\"new\"}"); },
@@ -363,7 +363,7 @@ await RunAsync(async (store, connection, ct) =>
                 LoadLevels = (_, _, _) =>
                 {
                     var level = ++calls == 1 ? 2 : 1;
-                    return Task.FromResult<Func<string, int, int?>>((_, _) => level);
+                    return Task.FromResult<Func<string, int, PerformanceMonitor.Notifications.AlertSeverityLevel?, int?>>((_, _, _) => level);
                 },
                 ReadBefore = (_, _, _) => Task.FromResult<string?>(null),
                 Write = (_, _, _) => { _writes++; return Task.FromResult<string?>(null); },
@@ -387,7 +387,7 @@ await RunAsync(async (store, connection, ct) =>
             await ExecAsync(connection, "UPDATE config.config_service SET agent_actions_enabled = true", ct);
             var hooks = new AgentActionHooks
             {
-                LoadLevels = (_, _, _) => Task.FromResult<Func<string, int, int?>>((_, _) => 2),
+                LoadLevels = (_, _, _) => Task.FromResult<Func<string, int, PerformanceMonitor.Notifications.AlertSeverityLevel?, int?>>((_, _, _) => 2),
                 /* Runs in the gate's transaction after the switch check passed and before action_called commits: a person turns the switch
                    off now, so only the write's re-check can see it. */
                 ReadBefore = async (_, _, token) =>
@@ -486,7 +486,7 @@ await RunAsync(async (store, connection, ct) =>
                 LoadLevels = (_, _, _) =>
                 {
                     loads++;
-                    return Task.FromResult<Func<string, int, int?>>((metric, _) => { seen.Add(metric); return 2; });
+                    return Task.FromResult<Func<string, int, PerformanceMonitor.Notifications.AlertSeverityLevel?, int?>>((metric, _, _) => { seen.Add(metric); return 2; });
                 },
                 ReadBefore = (_, _, _) => Task.FromResult<string?>(null),
                 Write = (_, _, _) => { _writes++; return Task.FromResult<string?>(null); },
@@ -553,7 +553,7 @@ CREATE TRIGGER test_fail_every_done BEFORE INSERT ON config.investigation_journa
     private AgentActionHooks WriteHooks(ServerChangeSignal? signal, bool mark, Func<CancellationToken, Task>? then = null) => new()
     {
         ServerChange = signal,
-        LoadLevels = (_, _, _) => Task.FromResult<Func<string, int, int?>>((_, _) => 2),
+        LoadLevels = (_, _, _) => Task.FromResult<Func<string, int, PerformanceMonitor.Notifications.AlertSeverityLevel?, int?>>((_, _, _) => 2),
         ReadBefore = (_, _, _) => Task.FromResult<string?>("{\"state\":\"old\"}"),
         Write = async (_, _, token) =>
         {

Darling/Darling.Tests/AgentChangeUndoLiveTests.cs

@@ -277,7 +277,7 @@ await RunAsync(async (undo, connection, ct) =>
             var writes = 0;
             var hooks = new AgentActionHooks
             {
-                LoadLevels = (_, _, _) => Task.FromResult<Func<string, int, int?>>((_, _) => 2),
+                LoadLevels = (_, _, _) => Task.FromResult<Func<string, int, PerformanceMonitor.Notifications.AlertSeverityLevel?, int?>>((_, _, _) => 2),
                 ReadBefore = (_, _, _) => Task.FromResult<string?>(null),
                 Write = (_, _, _) => { writes++; return Task.FromResult<string?>("{}"); },
             };

Darling/Darling.Tests/AgentLevel2StoreTests.cs

@@ -527,7 +527,7 @@ private static async Task RunTheGateAsMcpAsync(NpgsqlConnection owner, ScratchPo
             "[{\"tool\":\"get_blocking\",\"arguments\":{},\"key_numbers\":\"12\"}]"), new AgentActionTarget(7, "srv-7", null, null));
         AgentActionHooks Hooks(int level) => new()
         {
-            LoadLevels = (_, _, _) => Task.FromResult<Func<string, int, int?>>((_, _) => level),
+            LoadLevels = (_, _, _) => Task.FromResult<Func<string, int, PerformanceMonitor.Notifications.AlertSeverityLevel?, int?>>((_, _, _) => level),
             ReadBefore = (_, _, _) => Task.FromResult<string?>(null),
             Write = (_, _, _) => { writes++; return Task.FromResult<string?>("{}"); },
         };

Darling/Darling.Tests/AgentSetupWebLiveTests.cs

@@ -246,6 +246,17 @@ await WithWebRoutesAsync("admin", async (owner, _, Send, ct) =>
             Assert.Equal("Analysis: Waits|ops@example.com|3|60|500|false", await ScalarAsync(owner,
                 $"SELECT metric_match || '|' || smtp_recipients || '|' || agent_level || '|' || agent_max_per_hour || '|' || agent_max_per_day || '|' || enabled::text FROM config_notification_routes WHERE route_id = {seeded}", ct));
 
+            /* #6928: the severity filter. A new route is Both, an edit sets it (any case), an edit that leaves it out keeps it. */
+            Assert.Equal("both", await ScalarAsync(owner, $"SELECT severities FROM config_notification_routes WHERE route_id = {seeded}", ct));
+            Assert.Equal(200, (await Send("PUT", $"/api/notification-routes/{seeded}", Json, "{\"severities\":\"Critical\"}")).Status);
+            Assert.Equal("critical", await ScalarAsync(owner, $"SELECT severities FROM config_notification_routes WHERE route_id = {seeded}", ct));
+            Assert.Equal(200, (await Send("PUT", $"/api/notification-routes/{seeded}", Json, "{\"enabled\":true}")).Status);
+            Assert.Equal("critical", await ScalarAsync(owner, $"SELECT severities FROM config_notification_routes WHERE route_id = {seeded}", ct));
+            var warningOnly = await Send("POST", "/api/notification-routes", Json, "{\"metric_match\":\"blocking\",\"agent_level\":1,\"severities\":\"warning\"}");
+            Assert.Equal(200, warningOnly.Status);
+            Assert.Equal("warning", await ScalarAsync(owner, $"SELECT severities FROM config_notification_routes WHERE route_id = {(int)warningOnly.Body!["route_id"]!}", ct));
+            Assert.Equal(200, (await Send("PUT", $"/api/notification-routes/{seeded}", Json, "{\"severities\":\"both\"}")).Status);
+
             /* Level 0 (Off) and Inherit are both settable. */
             Assert.Equal(200, (await Send("PUT", $"/api/notification-routes/{seeded}", Json, "{\"agent_level\":0}")).Status);
             Assert.Equal("0", await ScalarAsync(owner, $"SELECT agent_level FROM config_notification_routes WHERE route_id = {seeded}", ct));
@@ -401,6 +412,8 @@ async Task Refused(string method, string path, string json, string contains)
             await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\"}", "at least one destination or the agent channel");
             await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\",\"agent_level\":1,\"fallback_route_id\":99999}", "Fallback route 99999 does not exist");
             await Refused("PUT", $"/api/notification-routes/{seeded}", $"{{\"fallback_route_id\":{seeded}}}", "its own fallback");
+            await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\",\"agent_level\":1,\"severities\":\"major\"}", "severities is critical, warning or both");
+            await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\",\"agent_level\":1,\"severities\":3}", "severities must be text");
             await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\",\"agent_level\":\"one\"}", "agent_level must be a whole number");
             await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\",\"agent_level\":1,\"agent_max_per_hour\":1.5}", "agent_max_per_hour must be a whole number");
 

Darling/Darling.Tests/AlertResolutionDeliveryTests.cs

@@ -365,6 +365,35 @@ public async Task ASecondClearWithNoFireBetween_SendsNothing()
         Assert.Equal(AlertDelivery.ChannelNotApplicable, rig.History.Records[^1].NotificationType);
     }
 
+    /// <summary>#6928: a clear goes to the channels the fire reached, not to wherever the severity filter would send it now. The fire
+    /// was Critical and went to the Critical-only route's endpoint. The routes' filters are then swapped (the other endpoint's route has the lower id, so a re-match at the clear would pick it), and the clear still goes to
+    /// that endpoint and never to the other.</summary>
+    [Fact]
+    public async Task AClear_GoesToTheChannelTheFireReached_WhateverTheSeverityFiltersSayAtTheClear()
+    {
+        using var criticalEndpoint = new CapturingWebhookEndpoint();
+        using var warningEndpoint = new CapturingWebhookEndpoint();
+        var config = new DarlingConfig();
+        config.Webhooks.SlackSendResolutions = true;
+        NotificationRoute[] RoutesWith(string first, string second) => new[]
+        {
+            new NotificationRoute(2, Metric, "", criticalEndpoint.Url, "", "", "", true) { Severities = first },
+            new NotificationRoute(1, Metric, "", warningEndpoint.Url, "", "", "", true) { Severities = second },
+        };
+        config.NotificationRoutes = RoutesWith("critical", "warning");
+        var rig = Rig.Build(config);
+
+        await rig.Deliverer.DeliverAsync(Fire() with { Severity = AlertSeverityLevel.Critical }, TestContext.Current.CancellationToken);
+        Assert.Single(criticalEndpoint.Bodies);
+        Assert.Empty(warningEndpoint.Bodies);
+
+        config.NotificationRoutes = RoutesWith("warning", "critical");
+        await rig.Deliverer.DeliverResolutionAsync(CpuResolution(), cancellationToken: TestContext.Current.CancellationToken);
+
+        Assert.Equal(2, criticalEndpoint.Bodies.Count);
+        Assert.Empty(warningEndpoint.Bodies);
+    }
+
     private static AlertOutcome Fire() =>
         new(ServerKey, Server, Metric, "97%", "90%", Context: null, DetailText: null,
             NumericCurrentValue: 97, NumericThresholdValue: 90, Muted: false, Severity: null);