From 4b19caf730fb02546da7615434b4683498710d54 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:18:05 -0400 Subject: [PATCH 01/22] Hold threshold overrides for a server tag (#6926) A threshold override can now be set for a server tag. Precedence is database, then server, then the deepest tag the server is under (the notification routes' own rule), then global. Adds V227, the MCP tag argument, the get_alert_settings tag source, the Viewer and web Admin tag rows, and tests. --- Darling/Darling.Tests/RungRewindArtifacts.cs | 10 + .../ThresholdOverrideTagScopeTests.cs | 329 ++++++++++++++++++ .../DarlingThresholdOverrides.cs | 105 ++++-- .../DarlingWebThresholdOverrides.cs | 29 +- .../Mcp/DarlingMcpThresholdOverrideTools.cs | 151 +++++++- .../wwwroot/js/pages/admin.js | 55 +-- .../PgMigrations.cs | 44 +++ .../StorageVersion.cs | 2 +- .../TagThresholdResolver.cs | 71 ++++ .../ThresholdOverrideStore.TagScope.cs | 94 +++++ .../ThresholdOverrideStore.cs | 31 +- .../ThresholdOverridesViewModel.cs | 56 ++- .../ThresholdOverridesWindow.xaml | 10 +- .../ThresholdOverridesWindow.xaml.cs | 11 +- .../ViewerDataService.ThresholdOverrides.cs | 31 ++ .../ViewerDataService.cs | 14 +- docs/darling/admin.md | 1 + 17 files changed, 953 insertions(+), 91 deletions(-) create mode 100644 Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs create mode 100644 Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs create mode 100644 Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs diff --git a/Darling/Darling.Tests/RungRewindArtifacts.cs b/Darling/Darling.Tests/RungRewindArtifacts.cs index 375b56efdd..a4273f778b 100644 --- a/Darling/Darling.Tests/RungRewindArtifacts.cs +++ b/Darling/Darling.Tests/RungRewindArtifacts.cs @@ -577,6 +577,16 @@ ALTER TABLE config.config_alert_settings // so a re-run of V226 does everything again. Safe on a store that never had it. new(226, "hasServerAwsRdsWebOrExpress", @"ALTER TABLE collect.servers DROP COLUMN IF EXISTS is_aws_rds_web_or_express;"), + // V227 (#6926): the tag scope on the threshold overrides. The column is the probe's newest sentinel. The rewind removes the tag rows, + // the two constraints and the tag index, puts server_id back to NOT NULL and drops the column, so a re-run of V227 does everything + // again. Safe on a store that never had it. + new(227, "hasThresholdOverrideTagScope", + @"DELETE FROM config.config_threshold_overrides WHERE server_id IS NULL; +ALTER TABLE config.config_threshold_overrides DROP CONSTRAINT IF EXISTS ck_config_threshold_overrides_one_scope; +ALTER TABLE config.config_threshold_overrides DROP CONSTRAINT IF EXISTS ck_config_threshold_overrides_tag_no_database; +DROP INDEX IF EXISTS config.ux_config_threshold_overrides_tag; +ALTER TABLE config.config_threshold_overrides ALTER COLUMN server_id SET NOT NULL; +ALTER TABLE config.config_threshold_overrides DROP COLUMN IF EXISTS scope_tag_id;"), ]; /// diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs new file mode 100644 index 0000000000..1c5c98400b --- /dev/null +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Npgsql; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/* #1776 own-store: every live fact mints its own scratch database through ScratchPostgres and never touches another test's rows, + so the class is deliberately NOT [Collection("live-postgres")]. */ + +/// +/// #6926, a threshold override held for a server tag. The precedence (database, then server, then tag, then global), the tag rule +/// (the notification routes' own: the deepest tag a server is under, the lowest row id on a same-depth tie, a deleted tag matching +/// nothing) and the whole path through a live store: set_alert_threshold_override with a tag, get_alert_settings +/// naming the tag, the list marking an orphan, and V227 on a store that already holds server overrides. +/// +public sealed class ThresholdOverrideTagScopeTests +{ + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private static readonly DarlingMcpWritePolicy.Effective WritesArm = new(true, false); + private static readonly DarlingMcpWritePolicy.Effective AgentArm = new(false, false, Investigations: true); + + /* Tags: 1 prod (root) > 2 east > 3 rack1; 4 dev (root); 5 and 6 are two roots a server can sit under at once. */ + private static readonly Dictionary Parents = new() + { + [1] = null, [2] = 1, [3] = 2, [4] = null, [5] = null, [6] = null, + }; + + private static StoreConfigProvider.RouteScopeInputs Inputs(bool readable = true) => new( + new HashSet { 7, 8, 9, 10 }, + Parents, + new Dictionary> + { + [7] = new[] { 3 }, /* under rack1: prod > east > rack1 */ + [8] = new[] { 1 }, /* directly under prod */ + [9] = new[] { 4 }, /* dev */ + [10] = new[] { 5, 6 }, /* two roots at the same depth */ + }, + readable); + + private static DarlingThresholdOverrides Load( + (int Server, string? Database, string Key, double Value)[] direct, + (long RowId, int Tag, string Key, double Value)[] tags, + bool readable = true) + { + var overrides = new DarlingThresholdOverrides(); + overrides.Replace(direct, tags.Select(t => (t.RowId, t.Tag, t.Key, t.Value)), Inputs(readable)); + return overrides; + } + + [Fact] + public void Precedence_DatabaseBeatsServerBeatsTagBeatsGlobal() + { + const string Key = "pvs.threshold_percent"; + var o = Load( + [(7, "shop", Key, 60), (7, null, Key, 55)], + [(1, 1, Key, 50)]); + + Assert.Equal(60, o.ResolveWire(Key, "7", "shop", 40)); /* the database override */ + Assert.Equal(55, o.ResolveWire(Key, "7", "ledger", 40)); /* no override for that database: the server's */ + Assert.Equal(55, o.ResolveWire(Key, "7", null, 40)); + Assert.Equal(50, o.ResolveWire(Key, "8", null, 40)); /* no server override: the tag's */ + Assert.Equal(50, o.ResolveWire(Key, "8", "shop", 40)); + Assert.Equal(40, o.ResolveWire(Key, "9", null, 40)); /* a server outside the tag: global */ + Assert.Equal(40, o.ResolveWire("cpu.threshold_percent", "8", null, 40)); + } + + [Fact] + public void TagRule_DeepestTagWins_AndASameDepthTieGoesToTheLowestRowId() + { + const string Key = "cpu.threshold_percent"; + + /* Server 7 sits under rack1 (2 ancestors); prod (0 ancestors) also covers it. The deeper tag wins whatever the row ids. */ + var nested = Load([], [(1, 1, Key, 80), (2, 3, Key, 70), (3, 2, Key, 75)]); + Assert.Equal(70, nested.ResolveWire(Key, "7", null, 90)); + Assert.Equal(80, nested.ResolveWire(Key, "8", null, 90)); /* server 8 is only under prod */ + + /* Server 10 is under two roots of the same depth: the lower row id wins, the way the router takes the lower route id. */ + var tie = Load([], [(20, 5, Key, 65), (10, 6, Key, 85)]); + Assert.Equal(85, tie.ResolveWire(Key, "10", null, 90)); + var reversed = Load([], [(10, 5, Key, 65), (20, 6, Key, 85)]); + Assert.Equal(65, reversed.ResolveWire(Key, "10", null, 90)); + } + + [Fact] + public void ADeletedTag_MatchesNothing_AndUnreadableTagTablesMatchNothing() + { + const string Key = "cpu.threshold_percent"; + var orphan = Load([], [(1, 99, Key, 70)]); /* tag 99 is not in the tree */ + Assert.Equal(90, orphan.ResolveWire(Key, "7", null, 90)); + Assert.Equal(0, orphan.Count); + + var unreadable = Load([], [(1, 1, Key, 70)], readable: false); + Assert.Equal(90, unreadable.ResolveWire(Key, "8", null, 90)); + } + + [Fact] + public void ATagRowWithAnInvalidValue_IsSkipped_AndTheShallowerTagStillApplies() + { + const string Key = "cpu.threshold_percent"; + var o = Load([], [(1, 1, Key, 80), (2, 3, Key, 5000)]); /* 5000 is out of bounds for a percent */ + Assert.Equal(80, o.ResolveWire(Key, "7", null, 90)); + } + + [Fact] + public async Task Tool_SetsAndClearsATagOverride_TheLoaderAndTheServerViewNameTheTag() + { + await RunAsync(async (ds, connection, store, ct) => + { + var set = await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 95, tag: "Production", setBy: "erik"); + Assert.Equal("ok", Status(set)); + Assert.Equal("Production", JsonNode.Parse(set)!["tag"]!.GetValue()); + Assert.Equal("1", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides WHERE scope_tag_id = 1 AND server_id IS NULL", ct)); + Assert.Equal("erik", await ScalarAsync(connection, "SELECT set_by FROM config.config_threshold_overrides WHERE scope_tag_id = 1", ct)); + + /* A second call updates the same row (by tag id this time); the answer carries the undo. */ + var again = await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 92, tag: "1"); + Assert.Equal(95, JsonNode.Parse(again)!["previous_override"]!.GetValue()); + Assert.Equal("1", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides WHERE scope_tag_id = 1", ct)); + + /* A nested tag with its own value: the deeper tag wins for the server under it. */ + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 88, tag: "east"))); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(88, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "7", null, 80)); /* srv-7 is under east */ + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); /* srv-8 is directly under Production */ + Assert.Equal(80, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + + /* A server override beats the tag in the loader and in the view. */ + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, "srv-7", "cpu.threshold_percent", 60))); + await loader.LoadAsync(ds, ct); + Assert.Equal(60, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "7", null, 80)); + + await ExecAsync(connection, "INSERT INTO config.config_alert_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + await ExecAsync(connection, "INSERT INTO config.config_notification (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + JsonNode? Row(JsonNode view, string key) => view["thresholds"]!.AsArray().First(n => n!["key"]!.GetValue() == key); + + var seven = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-7", ct))!["server_view"]!; + Assert.Equal("server_override", Row(seven, "cpu.threshold_percent")!["source"]!.GetValue()); + + var eight = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-8", ct))!["server_view"]!; + var cpu = Row(eight, "cpu.threshold_percent")!; + Assert.Equal("tag_override", cpu["source"]!.GetValue()); + Assert.Equal("Production", cpu["tag"]!.GetValue()); + Assert.Equal(92, cpu["effective_value"]!.GetValue()); + Assert.Equal("global", Row(eight, "blocking.count_threshold")!["source"]!.GetValue()); + Assert.Equal(1, eight["tag_overrides"]!.AsArray().Count); + + var nine = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-9", ct))!["server_view"]!; + Assert.Equal("global", Row(nine, "cpu.threshold_percent")!["source"]!.GetValue()); + + /* Clear works the same way, by name or id. */ + Assert.Equal("cleared", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", null, tag: "east", clear: true))); + Assert.Equal("cleared", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", null, tag: "1", clear: true))); + Assert.Equal("0", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides WHERE scope_tag_id IS NOT NULL", ct)); + }); + } + + [Fact] + public async Task Tool_RefusesTagMisuse_AndLeavesTheTableUntouched() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("scope_conflict", Reason(await Set(ds, store, WritesArm, "srv-7", "cpu.threshold_percent", 90, tag: "Production"))); + Assert.Equal("scope_required", Reason(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 90))); + Assert.Equal("database_scope", Reason(await Set(ds, store, WritesArm, null, "pvs.threshold_percent", 70, database: "shop", tag: "Production"))); + Assert.Equal("tag_not_found", Reason(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 90, tag: "nope"))); + Assert.Equal("tag_writes_arm_only", Reason(await Set(ds, store, AgentArm, null, "cpu.threshold_percent", 90, tag: "Production"))); + Assert.Equal("0", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides", ct)); + + /* A name two tags share is refused until the id is used. */ + await ExecAsync(connection, "INSERT INTO config.server_tags (name, parent_id) VALUES ('Shared', 1), ('Shared', 4)", ct); + Assert.Equal("tag_not_found", Reason(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 90, tag: "shared"))); + }); + } + + [Fact] + public async Task ADeletedTag_ListsAsOrphaned_MatchesNothing_AndCanStillBeCleared() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 95, tag: "dev"))); + await ExecAsync(connection, "DELETE FROM config.server_tags WHERE id = 4", ct); + + var rows = await ThresholdOverrideStore.ListAsync(ds, k => null, ct); + var row = Assert.Single(rows); + Assert.True(row.IsTag); + Assert.True(row.Orphaned); + Assert.Equal(4, row.TagId); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(80, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + + Assert.Equal("cleared", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", null, tag: "4", clear: true))); + Assert.Empty(await ThresholdOverrideStore.ListAsync(ds, k => null, ct)); + }); + } + + [Fact] + public async Task ListShowsTagRowsWithTheirTagName() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 95, tag: "Production"))); + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, "srv-7", "cpu.threshold_percent", 90))); + var rows = await ThresholdOverrideStore.ListAsync(ds, k => null, ct); + Assert.Equal(2, rows.Count); + var tag = rows.Single(r => r.IsTag); + Assert.Equal("Production", tag.ServerName); + Assert.False(tag.Orphaned); + Assert.Equal("srv-7", rows.Single(r => !r.IsTag).ServerName); + + var body = await DarlingWebThresholdOverrides.ListBodyAsync(ds, ct); + Assert.Equal(2, body["overrides"]!.AsArray().Count); + Assert.Contains(body["tags"]!.AsArray(), t => t!["tag_name"]!.GetValue() == "Production"); + }); + } + + [Fact] + public async Task V227_OnAStoreThatHoldsServerOverrides_KeepsThem_AndTakesTagRows() + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to run the threshold override live tests."); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var store = new NpgsqlConnection(scratch.ConnectionString); + await store.OpenAsync(ct); + await PgMigrations.MigrateAsync(store, ct); + + var rung = PgMigrations.Scripts.Single(m => m.Name == "threshold-override-tag-scope").Version; + Assert.Equal(227, rung); + await RungRewindArtifacts.DropSentinelsAboveAsync(store, rung - 1, ct); + await ExecAsync(store, "DELETE FROM darling_schema_version WHERE version >= " + rung.ToString(CultureInfo.InvariantCulture), ct); + await ExecAsync(store, "INSERT INTO config.config_monitored_servers (server_id, name, host) VALUES (7, 'srv-7', 'h7')", ct); + await ExecAsync(store, "INSERT INTO config.config_threshold_overrides (server_id, database_name, threshold_key, value, set_by) VALUES " + + "(7, NULL, 'cpu.threshold_percent', 91, 'erik'), (7, 'shop', 'pvs.threshold_percent', 66, 'erik')", ct); + Assert.Equal("False", await ScalarAsync(store, + "SELECT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'config.config_threshold_overrides'::regclass AND attname = 'scope_tag_id' AND NOT attisdropped)", ct)); + + await PgMigrations.MigrateAsync(store, ct); + Assert.Equal("227", await ScalarAsync(store, "SELECT max(version) FROM darling_schema_version", ct)); + Assert.Equal("2", await ScalarAsync(store, "SELECT count(*) FROM config.config_threshold_overrides WHERE server_id = 7 AND scope_tag_id IS NULL", ct)); + Assert.Equal("91", await ScalarAsync(store, "SELECT value FROM config.config_threshold_overrides WHERE database_name IS NULL", ct)); + + /* A tag row is accepted; a row with neither scope, both scopes or a database on a tag is not; one value per tag and key. */ + await ExecAsync(store, "INSERT INTO config.server_tags (name) VALUES ('Production')", ct); + await ExecAsync(store, "INSERT INTO config.config_threshold_overrides (scope_tag_id, threshold_key, value, set_by) VALUES (1, 'cpu.threshold_percent', 80, 'erik')", ct); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (scope_tag_id, threshold_key, value, set_by) VALUES (1, 'cpu.threshold_percent', 70, 'erik')", ct)); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (threshold_key, value, set_by) VALUES ('cpu.threshold_percent', 70, 'erik')", ct)); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (server_id, scope_tag_id, threshold_key, value, set_by) VALUES (7, 1, 'blocking.count_threshold', 70, 'erik')", ct)); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (scope_tag_id, database_name, threshold_key, value, set_by) VALUES (1, 'shop', 'pvs.threshold_percent', 70, 'erik')", ct)); + + /* A second run changes nothing, and the version stamp is not repeated. */ + await ExecAsync(store, "DELETE FROM darling_schema_version WHERE version >= " + rung.ToString(CultureInfo.InvariantCulture), ct); + await PgMigrations.MigrateAsync(store, ct); + Assert.Equal("3", await ScalarAsync(store, "SELECT count(*) FROM config.config_threshold_overrides", ct)); + } + + private static async Task RunAsync(Func body) + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the threshold override live tests."); + + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await ExecAsync(connection, "INSERT INTO config.config_monitored_servers (server_id, name, host) VALUES (7, 'srv-7', 'h7'), (8, 'srv-8', 'h8'), (9, 'srv-9', 'h9')", ct); + await ExecAsync(connection, "INSERT INTO collect.servers (server_id, server_name, display_name, is_enabled, sql_major_version, created_date, modified_date) " + + "VALUES (7, 'srv-7', 'srv-7', TRUE, 15, now(), now()), (8, 'srv-8', 'srv-8', TRUE, 15, now(), now()), (9, 'srv-9', 'srv-9', TRUE, 15, now(), now())", ct); + /* Tag ids 1 Production, 2 east (under Production), 3 rack1 (under east), 4 dev; srv-7 is under east, srv-8 under Production, srv-9 under dev. */ + await ExecAsync(connection, "INSERT INTO config.server_tags (name, parent_id) VALUES ('Production', NULL), ('east', 1), ('rack1', 2), ('dev', NULL)", ct); + await ExecAsync(connection, "INSERT INTO config.server_tag_map (server_id, tag_id) VALUES (7, 2), (8, 1), (9, 4)", ct); + + var builder = new NpgsqlConnectionStringBuilder(scratch.ConnectionString) { SearchPath = PgSchemaGenerator.SearchPath }; + await using var dataSource = NpgsqlDataSource.Create(builder.ConnectionString); + var store = new InvestigationRequestStore(dataSource, 30, () => new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc)); + await body(dataSource, connection, store, ct); + } + + private static Task Set(NpgsqlDataSource ds, InvestigationRequestStore store, DarlingMcpWritePolicy.Effective policy, + string? server, string key, int? value, string? database = null, bool clear = false, string? setBy = null, string? tag = null) => + DarlingMcpThresholdOverrideTools.SetAsync(ds, policy, store, NullLogger.Instance, server, key, value, database, clear, setBy, null, + TestContext.Current.CancellationToken, null, tag); + + private static string Status(string json) => JsonNode.Parse(json)!["status"]!.GetValue(); + + private static string Reason(string json) => JsonNode.Parse(json)!["reason"]!.GetValue(); + + private static async Task ExecAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + return Convert.ToString(await command.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture) ?? ""; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs index aa54bd0fae..1ea10e46c2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs @@ -14,11 +14,12 @@ using Microsoft.Extensions.Logging; using Npgsql; using PerformanceMonitor.Alerting; +using PerformanceMonitor.Darling.Storage; namespace PerformanceMonitor.Darling.Service; /// -/// The per-server and per-database threshold overrides the alert engine and the PostgreSQL count checks resolve through +/// The per-server, per-database and per-tag threshold overrides the alert engine and the PostgreSQL count checks resolve through /// (#5916): the Darling loader behind . It holds one immutable snapshot of /// config.config_threshold_overrides and answers from it, so the sweep path does no I/O. The service reloads it at /// start and on every control-plane reload (the table is on the config_version beacon), and a failed load keeps the @@ -27,66 +28,101 @@ namespace PerformanceMonitor.Darling.Service; internal sealed class DarlingThresholdOverrides : IAlertThresholdOverrides { internal const string SelectSql = - "SELECT server_id, database_name, threshold_key, value FROM config.config_threshold_overrides"; + "SELECT id, server_id, database_name, scope_tag_id, threshold_key, value FROM config.config_threshold_overrides"; + + /// The direct (server and database) overrides, and the tag override that wins per server and key. + private sealed record Snapshot(Dictionary<(int Server, string Database, string Key), int> Direct, Dictionary<(int Server, string Key), int> Tag); private readonly ILogger? _logger; - private volatile Dictionary<(int Server, string Database, string Key), int> _snapshot = new(); + private volatile Snapshot _snapshot = new(new(), new()); internal DarlingThresholdOverrides(ILogger? logger = null) => _logger = logger; - /// How many overrides are in force. - internal int Count => _snapshot.Count; + /// How many overrides are in force: the server and database ones, and the tag ones that some server resolves to. + internal int Count => _snapshot.Direct.Count + _snapshot.Tag.Count; /// /// Replaces the snapshot with the table's rows. The read is awaited before the swap and nothing here catches, so a /// read that throws leaves the overrides already in force. A row whose key is not a tunable threshold, whose value is /// not an integer inside the key's bounds, or whose database name sits on a server-only key is skipped and logged, - /// never applied: the engine uses the value as-is. + /// never applied: the engine uses the value as-is. Tag overrides (#6926) are resolved with the tag tree and the server-to-tag + /// map, read only when the table holds a tag row; a tag-table read that fails leaves them matching nothing, as a tag-scoped route does. /// internal async Task LoadAsync(NpgsqlDataSource postgres, CancellationToken cancellationToken = default) { - var next = new Dictionary<(int, string, string), int>(); - await using var command = postgres.CreateCommand(SelectSql); - command.CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds; - await using var reader = await command.ExecuteReaderAsync(cancellationToken); - while (await reader.ReadAsync(cancellationToken)) + var direct = new List<(int ServerId, string? Database, string Key, double Value)>(); + var tag = new List<(long RowId, int TagId, string Key, double Value)>(); + await using (var command = postgres.CreateCommand(SelectSql)) { - var serverId = reader.GetInt32(0); - var database = reader.IsDBNull(1) ? string.Empty : reader.GetString(1); - var key = reader.GetString(2); - var raw = reader.GetDouble(3); - if (Accept(key, raw, database, out var value) is { } problem) + command.CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) { - LogSkipped(serverId, key, database, problem); - continue; + var key = reader.GetString(4); + var raw = reader.GetDouble(5); + if (!reader.IsDBNull(3)) + tag.Add((reader.GetInt64(0), reader.GetInt32(3), key, raw)); + else if (!reader.IsDBNull(1)) + direct.Add((reader.GetInt32(1), reader.IsDBNull(2) ? null : reader.GetString(2), key, raw)); } + } - next[(serverId, database, key)] = value; + StoreConfigProvider.RouteScopeInputs? inputs = null; + if (tag.Count > 0) + { + await using var connection = await postgres.OpenConnectionAsync(cancellationToken); + inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, _logger, cancellationToken); } - _snapshot = next; + Replace(direct, tag, inputs); } - private void LogSkipped(int serverId, string key, string database, string problem) => + private void LogSkipped(string scope, string key, string database, string problem) => _logger?.LogWarning( - "Threshold override for server {ServerId} key '{Key}' database '{Database}' was skipped: {Problem}", - serverId, key, database, problem); + "Threshold override for {Scope} key '{Key}' database '{Database}' was skipped: {Problem}", + scope, key, database, problem); /// Replaces the snapshot from rows already in hand (the loader's own tests, and any caller that read the table). A /// skipped row is logged the way logs it. - internal void Replace(IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows) + internal void Replace(IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows) => + Replace(rows, [], null); + + /// The same with tag overrides (#6926). carry the table row id, which breaks a tie between two + /// tags of the same depth, and the registry, tag tree and server-to-tag map the routes expand against + /// (null: the tag tables were not read, and a tag override matches nothing). + internal void Replace( + IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows, + IEnumerable<(long RowId, int TagId, string Key, double Value)> tagRows, + StoreConfigProvider.RouteScopeInputs? inputs) { - var next = new Dictionary<(int, string, string), int>(); + var direct = new Dictionary<(int, string, string), int>(); foreach (var (serverId, database, key, raw) in rows) { var db = database ?? string.Empty; if (Accept(key, raw, db, out var value) is { } problem) - LogSkipped(serverId, key, db, problem); + LogSkipped("server " + serverId.ToString(CultureInfo.InvariantCulture), key, db, problem); else - next[(serverId, db, key)] = value; + direct[(serverId, db, key)] = value; + } + + var accepted = new List(); + foreach (var (rowId, tagId, key, raw) in tagRows) + { + if (Accept(key, raw, string.Empty, out var value) is { } problem) + LogSkipped("tag " + tagId.ToString(CultureInfo.InvariantCulture), key, string.Empty, problem); + else + accepted.Add(new TagOverrideCandidate(rowId, tagId, key, value)); + } + + var tagWinners = new Dictionary<(int, string), int>(); + if (accepted.Count > 0 && inputs is not null) + { + foreach (var (slot, winner) in TagThresholdResolver.Resolve( + accepted, inputs.KnownServerIds, inputs.TagParents, inputs.ServerTags, inputs.TagsReadable)) + tagWinners[slot] = (int)winner.Value; } - _snapshot = next; + _snapshot = new Snapshot(direct, tagWinners); } private static string? Accept(string key, double raw, string database, out int value) @@ -106,18 +142,21 @@ public int Resolve(AlertThresholdKey key, string serverKey, string? database, in /// /// The value of the wire-named threshold for the server (and database): a database override first, then the - /// server's, else . is the store server id as text, the - /// engine's server key. This is the one resolution the PostgreSQL count thresholds use too. + /// server's, then the tag's (#6926: the deepest tag the server is under), else . + /// is the store server id as text, the engine's server key. This is the one resolution the + /// PostgreSQL count thresholds use too. /// internal int ResolveWire(string wire, string serverKey, string? database, int globalValue) { if (!int.TryParse(serverKey, NumberStyles.Integer, CultureInfo.InvariantCulture, out var serverId)) return globalValue; var snapshot = _snapshot; - if (snapshot.Count == 0) + if (snapshot.Direct.Count == 0 && snapshot.Tag.Count == 0) return globalValue; - if (!string.IsNullOrEmpty(database) && snapshot.TryGetValue((serverId, database, wire), out var perDatabase)) + if (!string.IsNullOrEmpty(database) && snapshot.Direct.TryGetValue((serverId, database, wire), out var perDatabase)) return perDatabase; - return snapshot.TryGetValue((serverId, string.Empty, wire), out var perServer) ? perServer : globalValue; + if (snapshot.Direct.TryGetValue((serverId, string.Empty, wire), out var perServer)) + return perServer; + return snapshot.Tag.TryGetValue((serverId, wire), out var perTag) ? perTag : globalValue; } } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs index 9c15d750b5..d8cc0d3eec 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs @@ -42,8 +42,9 @@ internal static string RefusedMessage(bool managedStore, bool adminWritesOff = f + "Darling/tools/provision-roles.sql re-run since the upgrade that added threshold overrides: run it once as a superuser (it is idempotent).", adminWritesOff); - /// The list body: {overrides: [{server_id, server_name, database, key, value, global_value, set_by, request_id, - /// updated_at}], keys: [{key, min, max, database_scope}], servers: [{server_id, server_name}], can_edit, note}. + /// The list body: {overrides: [{server_id, server_name, tag_id, tag_name, orphaned, database, key, value, global_value, set_by, + /// request_id, updated_at}], keys: [{key, min, max, database_scope}], servers: [{server_id, server_name}], tags: [{tag_id, tag_name}], + /// can_edit, note}. A tag override (#6926) has a tag_id, a null server_id and server_name, and orphaned true when its tag was deleted. /// can_edit is whether the store login holds the write grant, so a page can say list-only before anyone tries. internal static async Task ListBodyAsync(NpgsqlDataSource postgres, CancellationToken ct, NpgsqlDataSource? accessSource = null) { @@ -56,8 +57,11 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, { overrides.Add(new JsonObject { - ["server_id"] = r.ServerId, - ["server_name"] = r.ServerName, + ["server_id"] = r.IsTag ? null : r.ServerId, + ["server_name"] = r.IsTag ? null : r.ServerName, + ["tag_id"] = r.TagId, + ["tag_name"] = r.IsTag && !r.Orphaned ? r.ServerName : null, + ["orphaned"] = r.Orphaned, ["database"] = r.Database, ["key"] = r.Key, ["value"] = r.Value, @@ -95,11 +99,21 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, servers.Add(new JsonObject { ["server_id"] = reader.GetInt32(0), ["server_name"] = reader.GetString(1) }); } + var tags = new JsonArray(); + await using (var command = postgres.CreateCommand("SELECT id, name FROM config.server_tags ORDER BY name, id")) + { + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + tags.Add(new JsonObject { ["tag_id"] = reader.GetInt32(0), ["tag_name"] = reader.GetString(1) }); + } + return new JsonObject { ["overrides"] = overrides, ["keys"] = keys, ["servers"] = servers, + ["tags"] = tags, ["can_edit"] = await DarlingMcpThresholdOverrideTools.HoldsGrantAsync(accessSource ?? postgres, ct), ["note"] = note, }; @@ -116,6 +130,7 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, var server = body["server_name"]?.GetValueKind() == JsonValueKind.String ? body["server_name"]!.GetValue() : null; var key = body["key"]?.GetValueKind() == JsonValueKind.String ? body["key"]!.GetValue() : null; var database = body["database_name"]?.GetValueKind() == JsonValueKind.String ? body["database_name"]!.GetValue() : null; + var tag = body["tag"]?.GetValueKind() == JsonValueKind.String ? body["tag"]!.GetValue() : null; var clear = body["clear"]?.GetValueKind() == JsonValueKind.True; int? value = null; @@ -126,13 +141,13 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, value = parsed; } - if (string.IsNullOrWhiteSpace(server)) - return (400, new JsonObject { ["error"] = "server_name is required." }); + if (string.IsNullOrWhiteSpace(server) && string.IsNullOrWhiteSpace(tag)) + return (400, new JsonObject { ["error"] = "server_name or tag is required." }); var json = await DarlingMcpThresholdOverrideTools.SetAsync( postgres, new DarlingMcpWritePolicy.Effective(true, false), new InvestigationRequestStore(postgres, McpCommandDeadlines.ReadSeconds), logger, - server, key ?? string.Empty, value, database, clear, ThresholdOverrideStore.SetByFor(editor), null, ct); + server, key ?? string.Empty, value, database, clear, ThresholdOverrideStore.SetByFor(editor), null, ct, tag: tag); var answer = JsonNode.Parse(json)!.AsObject(); var status = answer["status"]?.GetValue(); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs index 521ad6b075..edab71bae2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.ComponentModel; using System.Globalization; +using System.Linq; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; @@ -67,12 +68,14 @@ public sealed class DarlingMcpThresholdOverrideTools "runs again) as a superuser. Managed and compose stores grant it themselves on start. Call again after the script has run."; [McpServerTool(Name = "set_alert_threshold_override"), Description( - "Sets or clears a threshold for ONE server (or one database on it), leaving the global value alone. Changes the " + + "Sets or clears a threshold for ONE server (or one database on it), or for a server TAG, leaving the global value alone. Changes the " + "monitoring store only; the service picks it up within its reload interval. <> " + "key is a tunable integer threshold named as get_alert_settings names it (for example cpu.threshold_percent, " + "blocking.pg_count_threshold); get_alert_settings with server_name lists the keys, the global value, the effective value " + "and where it comes from. value is an integer inside the same bounds the global setting has. clear: true removes the " + - "override so the global value applies again. database_name applies only to pvs.* and file_growth.rise_mb / " + + "override so the global value applies again. tag (a tag name or id, in place of server_name) holds the value for every server under that tag " + + "and every tag below it: the writes arm only, and no database_name. Precedence is database, then server, then tag (the deepest tag a server " + + "is under, as a notification route resolves it, the older override winning a same-depth tie), then global. database_name applies only to pvs.* and file_growth.rise_mb / " + "file_growth.volume_percent. Refused: a key outside that list, a self_alerts.*, enable, cooldown or route key, an " + "out-of-bounds value. To silence an alert, mute it. An agent cannot set a threshold to the value that turns its alert off (the top of the " + "key's range, or zero for the low_disk.* keys); that value is refused as threshold_disables_alert. A very high count below " + @@ -85,23 +88,24 @@ public static async Task SetAlertThresholdOverride( NpgsqlDataSource postgres, DarlingMcpServedPolicy served, ILogger logger, - [Description("Server name, or the start of it.")] string server_name, - [Description("A tunable threshold, as get_alert_settings names it, e.g. cpu.threshold_percent.")] string key, + [Description("Server name, or the start of it. Omit when tag is given.")] string? server_name = null, + [Description("A tunable threshold, as get_alert_settings names it, e.g. cpu.threshold_percent.")] string key = "", [Description("The override value (integer).")] int? value = null, [Description("Database on that server (pvs.*, file_growth.rise_mb, file_growth.volume_percent). Omit for the whole server.")] string? database_name = null, [Description("true removes the override.")] bool clear = false, [Description("Who made the change (writes arm). 1-64 of A-Za-z0-9._:-. Default mcp.")] string? set_by = null, [Description("Investigations arm only: the agent_action object (request_id, claim_id, agent_id, reason, evidence). Ignored under the writes arm.")] string? agent_action = null, + [Description("A server tag, by name or id, in place of server_name: the value applies to every server under the tag and the tags below it (#6926). Writes arm only; no database_name.")] string? tag = null, StoreOwnerSource? ownerSource = null, CancellationToken cancellationToken = default) => await SetAsync(postgres, served.Policy, new InvestigationRequestStore(postgres, ServiceCommandDeadlines.SerialLoopSeconds, agentActionSource: ownerSource?.Read()), - logger, server_name, key, value, database_name, clear, set_by, agent_action, cancellationToken, served.NamedAgentNames); + logger, server_name, key, value, database_name, clear, set_by, agent_action, cancellationToken, served.NamedAgentNames, tag); /// The tool body over a given store, so a test drives either arm. internal static async Task SetAsync( NpgsqlDataSource postgres, DarlingMcpWritePolicy.Effective policy, InvestigationRequestStore store, ILogger logger, - string server_name, string key, int? value, string? database_name, bool clear, string? set_by, string? agent_action, - CancellationToken cancellationToken, IReadOnlySet? namedAgentNames = null) + string? server_name, string key, int? value, string? database_name, bool clear, string? set_by, string? agent_action, + CancellationToken cancellationToken, IReadOnlySet? namedAgentNames = null, string? tag = null) { try { @@ -110,6 +114,15 @@ internal static async Task SetAsync( DarlingThresholdKeys.TryGet(key, out var entry); var wire = entry.Wire; var database = string.IsNullOrWhiteSpace(database_name) ? null : database_name.Trim(); + var hasTag = !string.IsNullOrWhiteSpace(tag); + if (hasTag && !string.IsNullOrWhiteSpace(server_name)) + return Refused("scope_conflict", "Give a server_name or a tag, not both."); + if (!hasTag && string.IsNullOrWhiteSpace(server_name)) + return Refused("scope_required", "server_name (or tag) is required."); + if (hasTag && database is not null) + return Refused("database_scope", "A tag override holds a server-level value, so it takes no database_name."); + if (hasTag && !policy.Writes) + return Refused("tag_writes_arm_only", "A tag override changes every server under the tag, so it is set on the writes arm, not by an investigation request's agent, whose change is for the request's server only."); if (!clear) { @@ -131,7 +144,10 @@ internal static async Task SetAsync( return Refused("set_by_invalid", "set_by must be 1-64 of A-Za-z0-9._:-."); } - var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name, cancellationToken, NameClash.Refuse); + if (hasTag) + return await SetTagAsync(postgres, tag!, wire, value, clear, person!, cancellationToken); + + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name!, cancellationToken, NameClash.Refuse); if (error is not null) return error; @@ -225,6 +241,52 @@ internal static async Task SetAsync( } } + /// The tag branch of (#6926), writes arm only: resolve the tag, then set or clear its value in one transaction. + /// A tag with no servers under it is allowed (servers are tagged later and take the value then). + private static async Task SetTagAsync( + NpgsqlDataSource postgres, string tagText, string wire, int? value, bool clear, string by, CancellationToken cancellationToken) + { + var (tagId, tagName, problem) = await ThresholdOverrideStore.ResolveTagAsync(postgres, tagText, cancellationToken); + /* A clear by id still works for an override whose tag was deleted (it is listed as orphaned): the row is found by its tag id below. */ + var orphanClear = problem is not null && clear + && int.TryParse(tagText.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var orphanId) && (tagId = orphanId) is not null; + if (problem is not null && !orphanClear) + return Refused("tag_not_found", problem); + if (!await HoldsGrantAsync(postgres, cancellationToken)) + return McpHelpers.Status("precondition", PreconditionMessage); + + double? previous; + await using (var connection = await postgres.OpenConnectionAsync(cancellationToken)) + await using (var transaction = await connection.BeginTransactionAsync(cancellationToken)) + { + previous = await ThresholdOverrideStore.ReadTagValueAsync(connection, transaction, tagId!.Value, wire, cancellationToken); + if (orphanClear && previous is null) + return Refused("tag_not_found", problem!); + if (clear) + await ThresholdOverrideStore.DeleteTagAsync(connection, transaction, tagId.Value, wire, cancellationToken); + else + await ThresholdOverrideStore.UpsertTagAsync(connection, transaction, tagId.Value, wire, value!.Value, by, null, cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + + return JsonSerializer.Serialize(new + { + status = clear ? "cleared" : "ok", + tag = tagName, + tag_id = tagId, + key = wire, + value = clear ? (int?)null : value, + previous_override = previous, + undo = DarlingMcpAlertTools.UndoCall(ToolName, previous is { } was + ? new System.Collections.Generic.Dictionary { ["tag"] = tagId!.Value.ToString(CultureInfo.InvariantCulture), ["key"] = wire, ["value"] = was } + : new System.Collections.Generic.Dictionary { ["tag"] = tagId!.Value.ToString(CultureInfo.InvariantCulture), ["key"] = wire, ["clear"] = true }), + set_by = by, + message = clear + ? "The tag override is removed; servers under the tag fall back to a shallower tag, or the global value." + : "The tag override is stored. The service applies it within its reload interval to every server under the tag that has no server or database override of its own." + }, McpHelpers.JsonOptions); + } + private sealed class Applied { internal double? Previous; @@ -264,8 +326,8 @@ internal static async Task HoldsGrantAsync(NpgsqlDataSource postgres, Canc /// /// The get_alert_settings view for one server (#5916): every tunable threshold with its global value, the value in - /// effect for this server and where that value comes from (global or server_override), and the database - /// overrides the server holds. A read of the overrides table, so it answers what the service will apply after its next reload. + /// effect for this server and where that value comes from (global, tag_override naming the tag, or server_override), + /// and the database overrides the server holds. A read of the overrides table, so it answers what the service will apply after its next reload. /// internal static async Task BuildServerViewAsync( NpgsqlDataSource postgres, DarlingAlertReader.AlertSettingsReadRow s, int serverId, string serverName, @@ -291,6 +353,56 @@ internal static async Task BuildServerViewAsync( note = PreconditionMessage; } + /* #6926: the tag overrides that reach this server, resolved by the notification routes' own tag rule (TagThresholdResolver), the same + one the service loads. A tag read that fails reaches no server: the view then shows what the engine would apply. */ + var tagWinners = new System.Collections.Generic.Dictionary(); + try + { + var candidates = new System.Collections.Generic.List(); + var meta = new System.Collections.Generic.Dictionary<(int Tag, string Key), (string SetBy, DateTime UpdatedAt)>(); + await using (var command = postgres.CreateCommand( + "SELECT id, scope_tag_id, threshold_key, value, set_by, updated_at FROM " + Table + " WHERE scope_tag_id IS NOT NULL")) + { + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var key = reader.GetString(2); + var raw = reader.GetDouble(3); + if (!DarlingThresholdKeys.TryGet(key, out _) || raw != Math.Floor(raw) || raw < int.MinValue || raw > int.MaxValue) + continue; + candidates.Add(new TagOverrideCandidate(reader.GetInt64(0), reader.GetInt32(1), key, raw)); + meta[(reader.GetInt32(1), key)] = (reader.GetString(4), reader.GetDateTime(5)); + } + } + + if (candidates.Count > 0) + { + await using var connection = await postgres.OpenConnectionAsync(cancellationToken); + var inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, null, cancellationToken); + var names = new System.Collections.Generic.Dictionary(); + await using (var command = new NpgsqlCommand("SELECT id, name FROM config.server_tags", connection) { CommandTimeout = McpCommandDeadlines.ReadSeconds }) + await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) + { + while (await reader.ReadAsync(cancellationToken)) + names[reader.GetInt32(0)] = reader.GetString(1); + } + + foreach (var ((server, key), winner) in TagThresholdResolver.Resolve( + candidates, inputs.KnownServerIds, inputs.TagParents, inputs.ServerTags, inputs.TagsReadable)) + { + if (server != serverId) + continue; + var m = meta[(winner.TagId, key)]; + tagWinners[key] = (winner.TagId, names.GetValueOrDefault(winner.TagId, string.Empty), winner.Value, m.SetBy, m.UpdatedAt); + } + } + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.InsufficientPrivilege) + { + note ??= PreconditionMessage; + } + /* One number per key, read the way the engine reads it (#5916): the global value and the value with this server's override applied each go through the same pair clamp the enclosing get_alert_settings payload applies, so an override that inverts a warning/critical pair shows the pair the engine evaluates, not the stored one. */ @@ -305,19 +417,25 @@ internal static async Task BuildServerViewAsync( return null; } + int? TagOverride(string wire) => tagWinners.TryGetValue(wire, out var t) ? (int)t.Value : null; + var thresholds = new System.Collections.Generic.List(); foreach (var entry in DarlingThresholdKeys.Entries) { var global = GlobalValueAsEngineReadsIt(s, entry.Wire); var server = ServerOverride(entry.Wire); - var effective = Evaluated(entry.Wire, w => ServerOverride(w) ?? GlobalValue(s, w)); + var effective = Evaluated(entry.Wire, w => ServerOverride(w) ?? TagOverride(w) ?? GlobalValue(s, w)); + var found = tagWinners.TryGetValue(entry.Wire, out var tw); + var tagHit = server is null && found; thresholds.Add(new { key = entry.Wire, global_value = global, effective_value = effective, - source = server is null ? "global" : "server_override", + source = server is not null ? "server_override" : tagHit ? "tag_override" : "global", + tag = tagHit ? tw.TagName : null, + tag_id = tagHit ? tw.TagId : (int?)null, database_scope = entry.Scope == PerformanceMonitor.Alerting.AlertThresholdScope.ServerAndDatabase }); } @@ -338,10 +456,15 @@ internal static async Task BuildServerViewAsync( { server = serverName, server_id = serverId, - note = note ?? "effective_value is the global value with this server's override applied; a database override (database_overrides) applies to that database only. Set one with set_alert_threshold_override.", + note = note ?? "effective_value is the global value with this server's override applied, else the deepest server tag's (source tag_override, naming the tag; tag_overrides lists the ones that reach this server), else global; a database override (database_overrides) applies to that database only. Set one with set_alert_threshold_override.", thresholds, server_overrides = serverOverrides, - database_overrides = databaseOverrides + database_overrides = databaseOverrides, + tag_overrides = tagWinners.Select(t => (object)new + { + key = t.Key, tag = t.Value.TagName, tag_id = t.Value.TagId, value = t.Value.Value, set_by = t.Value.SetBy, + updated_at = t.Value.UpdatedAt.ToString("o", CultureInfo.InvariantCulture) + }).ToList() }; } diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js index 0abc05db92..ee27ef9ebf 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js @@ -1807,12 +1807,12 @@ async function buildPending(body, generation) { /* ─────────────────────────── threshold overrides (#5916) ─────────────────────────── */ -/* A threshold held for one server (or one database on it) that differs from the global value. The list is GET +/* A threshold held for one server (or one database on it), or for a server tag (#6926), that differs from the global value. The list is GET /api/threshold-overrides; a change is PUT /api/threshold-overrides, accepted only under a store login with the admin role. A managed or compose store connects this page as the viewer role, so there the grid is a list: the form and the Remove buttons are not drawn, and the note says where to change one. The service checks the key and the bounds again. */ export const OVERRIDE_COLUMNS = [ - { key: "server_name", label: "Server" }, + { key: "scope_text", label: "Scope" }, { key: "database_text", label: "Database" }, { key: "key", label: "Key" }, { key: "value", label: "Value" }, @@ -1823,10 +1823,10 @@ export const OVERRIDE_COLUMNS = [ ]; const OVERRIDES_NOTE_EDITING = - "A threshold held for one server, or for one database on it, that differs from the global value. The alert engine uses it for that server only " + - "and applies a change within its reload interval. To silence an alert, mute it."; + "A threshold held for one server, for one database on it, or for a server tag (every server under the tag and the tags below it), that differs from the global value. " + + "A database override wins over a server override, then the deepest tag the server is under, then the global value. A change applies within the alert engine's reload interval. To silence an alert, mute it."; const overridesNoteReadOnly = (managed, adminWritesOff) => - "A threshold held for one server, or for one database on it, that differs from the global value. " + + "A threshold held for one server, for one database on it, or for a server tag, that differs from the global value. " + listOnlyNote(managed, "this page lists them only", "To change an override", adminWritesOff); const OVERRIDE_DENIED = "Changing a threshold override needs"; @@ -1834,8 +1834,12 @@ const OVERRIDE_DENIED = "Changing a threshold override needs"; export function overrideRows(data) { const list = data && Array.isArray(data.overrides) ? data.overrides : []; return list.map((o) => ({ - server_id: o.server_id, - server_name: o.server_name, + server_id: o.server_id == null ? null : o.server_id, + server_name: o.server_name == null ? null : o.server_name, + tag_id: o.tag_id == null ? null : o.tag_id, + tag_name: o.tag_name == null ? null : o.tag_name, + orphaned: o.orphaned === true, + scope_text: o.tag_id == null ? o.server_name : (o.orphaned === true ? "Tag (deleted): orphaned, matches nothing" : "Tag: " + (o.tag_name || "")), database: o.database || null, database_text: o.database || "Whole server", key: o.key, @@ -1851,7 +1855,10 @@ export function overrideRows(data) { inside its bounds, and a database only on a key that takes one. The service applies the same rules again. */ export function validateOverride(keys, form) { const list = Array.isArray(keys) ? keys : []; - if (!form || !String(form.server || "").trim()) return "Choose a server."; + const hasServer = !!(form && String(form.server || "").trim()); + const hasTag = !!(form && String(form.tag || "").trim()); + if (hasServer && hasTag) return "Choose a server or a tag, not both."; + if (!hasServer && !hasTag) return "Choose a server or a tag."; const entry = list.find((k) => k.key === String(form.key || "").trim()); if (!entry) return "Choose a key from the list: only a tunable threshold can be overridden."; const text = String(form.value == null ? "" : form.value).trim(); @@ -1860,6 +1867,9 @@ export function validateOverride(keys, form) { if (!Number.isSafeInteger(n) || n < entry.min || n > entry.max) { return entry.key + " must be a whole number between " + entry.min + " and " + entry.max + ", the same bounds the global setting has."; } + if (hasTag && String(form.database || "").trim()) { + return "A tag override holds a server-level value, so it takes no database."; + } if (String(form.database || "").trim() && entry.database_scope !== true) { return entry.key + " is compared against a server-level figure, so it takes a server override only, not a database."; } @@ -1887,7 +1897,7 @@ export function overrideOutcome(verb, res) { } /* The form's entries and the last answer, in module scope so the 60 s repaint keeps what was typed. */ -const EMPTY_OVERRIDE_FORM = { server: "", key: "", database: "", value: "" }; +const EMPTY_OVERRIDE_FORM = { server: "", tag: "", key: "", database: "", value: "" }; let overrideEntries = { ...EMPTY_OVERRIDE_FORM }; let overrideStatus = null; let overrideBusy = false; @@ -1903,7 +1913,9 @@ async function changeOverride(verb, entry) { overrideBusy = true; let out; try { - const payload = { server_name: entry.server, key: entry.key, database_name: entry.database || null }; + const payload = { key: entry.key, database_name: entry.database || null }; + if (entry.tag) payload.tag = String(entry.tag); + else payload.server_name = entry.server; if (verb === "clear") payload.clear = true; else payload.value = Number(String(entry.value).trim()); out = overrideOutcome(verb, await apiWrite("PUT", "/api/threshold-overrides", payload)); @@ -1916,19 +1928,22 @@ async function changeOverride(verb, entry) { await redrawOverrides(); } -function overrideFormNode(servers, keys) { +function overrideFormNode(servers, keys, tags) { const field = (label, control) => el("label", { class: "tag-field" }, [el("span", { class: "mute-label", text: label }), control]); const choose = (items, textOf) => [el("option", { value: "", text: "(choose)" })].concat(items.map((i) => el("option", { value: textOf(i), text: textOf(i) }))); const server = el("select", { class: "tag-input", "data-field": "server" }, choose(servers, (s) => s.server_name)); + const tag = el("select", { class: "tag-input", "data-field": "tag" }, + [el("option", { value: "", text: "(none)" })].concat((Array.isArray(tags) ? tags : []).map((t) => el("option", { value: String(t.tag_id), text: t.tag_name })))); const key = el("select", { class: "tag-input", "data-field": "key" }, choose(keys, (k) => k.key)); const database = el("input", { type: "text", class: "tag-input", "data-field": "database", placeholder: "blank = whole server" }); const value = el("input", { type: "text", class: "tag-input", "data-field": "value", inputmode: "numeric" }); server.value = overrideEntries.server; + tag.value = overrideEntries.tag; key.value = overrideEntries.key; database.value = overrideEntries.database; value.value = overrideEntries.value; - const sync = () => { overrideEntries = { server: server.value, key: key.value, database: database.value, value: value.value }; }; - for (const c of [server, key, database, value]) c.addEventListener("input", sync); + const sync = () => { overrideEntries = { server: server.value, tag: tag.value, key: key.value, database: database.value, value: value.value }; }; + for (const c of [server, tag, key, database, value]) c.addEventListener("input", sync); const save = el("button", { class: "btn primary", type: "button", text: "Save override", "data-action": "save-override", onClick: async () => { @@ -1943,21 +1958,21 @@ function overrideFormNode(servers, keys) { }, }); return el("div", { class: "tag-form", "data-override-form": "1" }, [ - field("Server:", server), field("Key:", key), field("Database:", database), field("Value:", value), save, + field("Server:", server), field("or Tag:", tag), field("Key:", key), field("Database:", database), field("Value:", value), save, ]); } function overrideActions(row) { const edit = el("button", { - class: "btn", type: "button", text: "Edit", "aria-label": "Edit " + row.key + " for " + row.server_name, + class: "btn", type: "button", text: "Edit", "aria-label": "Edit " + row.key + " for " + row.scope_text, onClick: async () => { - overrideEntries = { server: row.server_name, key: row.key, database: row.database || "", value: String(row.value) }; + overrideEntries = { server: row.tag_id == null ? row.server_name : "", tag: row.tag_id == null ? "" : String(row.tag_id), key: row.key, database: row.database || "", value: String(row.value) }; await redrawOverrides(); }, }); const remove = el("button", { - class: "btn", type: "button", text: "Remove", "aria-label": "Remove " + row.key + " for " + row.server_name, "data-action": "remove-override", - onClick: () => changeOverride("clear", { server: row.server_name, key: row.key, database: row.database }), + class: "btn", type: "button", text: "Remove", "aria-label": "Remove " + row.key + " for " + row.scope_text, "data-action": "remove-override", + onClick: () => changeOverride("clear", { server: row.tag_id == null ? row.server_name : "", tag: row.tag_id == null ? "" : String(row.tag_id), key: row.key, database: row.database }), }); return el("span", { class: "row-actions" }, [edit, remove]); } @@ -1981,11 +1996,11 @@ async function buildOverrides(body, generation) { const keys = Array.isArray(res.data && res.data.keys) ? res.data.keys : []; nodes.push(noticeStrip(canEdit ? OVERRIDES_NOTE_EDITING : overridesNoteReadOnly(session.managed_store === true, session.admin_writes_off === true))); if (res.data && typeof res.data.note === "string" && res.data.note) nodes.push(noticeStrip(res.data.note)); - if (canEdit) nodes.push(overrideFormNode(Array.isArray(res.data.servers) ? res.data.servers : [], keys)); + if (canEdit) nodes.push(overrideFormNode(Array.isArray(res.data.servers) ? res.data.servers : [], keys, res.data.tags)); nodes.push(VIZ.table({ overrides: overrideRows(res.data) }, { rowsKey: "overrides", columns: overrideColumns(canEdit), - emptyText: "No server holds an override: every server uses the global values.", + emptyText: "No server or tag holds an override: every server uses the global values.", })); mount(body, nodes); } diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index c65424e31c..8ca7eb987e 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -2975,8 +2975,52 @@ holds the ordering. */ new Migration(224, "identity-capacity", V224Sql), new Migration(225, "server-aws-rds", V225Sql), new Migration(226, "server-aws-rds-web-or-express", V226Sql), + new Migration(227, "threshold-override-tag-scope", V227Sql), }; + /// + /// V227 (#6926): a threshold override can be held for a server tag. config.config_threshold_overrides gains the nullable + /// scope_tag_id (a config.server_tags id), server_id becomes nullable, and a CHECK says exactly one of the two is set, + /// so a stored row (server set, tag NULL) keeps its meaning and stays valid. A tag row holds no database (CHECK), because a tag value is + /// the server-level value for every server under the tag. A second unique index keeps one value per tag and key, since the V191 index is + /// on server_id and NULLs never collide. + /// + /// No foreign key on the tag, deliberately, as V181 did for a route: an override whose tag was deleted stays as written and + /// shows as orphaned, and the service resolves it to nothing. A cascade would delete the person's setting, and SET NULL would break the + /// exactly-one CHECK. The table's reload trigger (V191) and the tag tables' (V181) already bump the config version, so the service + /// re-reads when a tag moves. No GRANT: the table-level grants of V191 (admin and mcp write, viewer read) cover the new column. Every + /// statement is idempotent. No Lite twin: Lite has no overrides. + /// + private const string V227Sql = @" +ALTER TABLE config.config_threshold_overrides + ADD COLUMN IF NOT EXISTS scope_tag_id integer; + +ALTER TABLE config.config_threshold_overrides + ALTER COLUMN server_id DROP NOT NULL; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ck_config_threshold_overrides_one_scope' + AND conrelid = 'config.config_threshold_overrides'::regclass) THEN + ALTER TABLE config.config_threshold_overrides + ADD CONSTRAINT ck_config_threshold_overrides_one_scope + CHECK ((server_id IS NOT NULL) <> (scope_tag_id IS NOT NULL)); + END IF; + + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ck_config_threshold_overrides_tag_no_database' + AND conrelid = 'config.config_threshold_overrides'::regclass) THEN + ALTER TABLE config.config_threshold_overrides + ADD CONSTRAINT ck_config_threshold_overrides_tag_no_database + CHECK (scope_tag_id IS NULL OR database_name IS NULL); + END IF; +END $$; + +CREATE UNIQUE INDEX IF NOT EXISTS ux_config_threshold_overrides_tag + ON config.config_threshold_overrides (scope_tag_id, threshold_key) + WHERE scope_tag_id IS NOT NULL;"; + /// /// V225 (#6894): collect.servers.is_aws_rds, whether the target is an Amazon RDS for SQL Server instance (the connect query's /// DB_ID('rdsadmin') IS NOT NULL), written on every connect. The collectors that RDS gates off (running jobs, Agent status, Agent alerts) diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs index bfebc69b62..e669ce62e7 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs @@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class StorageVersion { - public const int SchemaVersion = 226; + public const int SchemaVersion = 227; } diff --git a/Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs b/Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs new file mode 100644 index 0000000000..2a6f88ae5c --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using PerformanceMonitor.Notifications; + +namespace PerformanceMonitor.Darling.Storage; + +/// One stored tag override: the table row's id, its tag, the wire key and the value. +public sealed record TagOverrideCandidate(long RowId, int TagId, string Key, double Value); + +/// The tag override that applies to a server for one key: the tag it comes from and its value. +public sealed record TagOverrideWinner(int TagId, double Value); + +/// +/// Which tag override applies to a server (#6926). It is the notification routes' rule, run by the routes' own code: each tag that +/// holds an override is expanded by exactly as a tag-scoped route is (a tag covers the servers +/// mapped to it or to any tag below it, a tag that no longer exists is orphaned and covers nothing, and tag tables that could not be read +/// cover nothing and are not called orphaned). When a server is covered by several tags that hold the same key, the deepest tag wins +/// (, the number of ancestors) and a tie goes to the lowest row id, the way the router's +/// ordering takes the lowest route id after depth. Pure: the service, the MCP view and the tests share this one resolution. +/// +public static class TagThresholdResolver +{ + /// The winning tag override per (server, key) for every server some tag covers. + public static IReadOnlyDictionary<(int ServerId, string Key), TagOverrideWinner> Resolve( + IReadOnlyCollection rows, + IReadOnlyCollection knownServerIds, + IReadOnlyDictionary tagParents, + IReadOnlyDictionary> serverTags, + bool tagsReadable = true) + { + ArgumentNullException.ThrowIfNull(rows); + if (rows.Count == 0) + return new Dictionary<(int ServerId, string Key), TagOverrideWinner>(); + + var tagIds = rows.Select(r => r.TagId).Distinct().ToList(); + var routes = tagIds + .Select(t => new NotificationRoute(t, NotificationRoute.WildcardMatch, "", "", "", "", "", true) { ScopeTagId = t }) + .ToList(); + var expanded = NotificationRouteScopeExpander.Expand(routes, knownServerIds, tagParents, serverTags, tagsReadable); + var byTag = new Dictionary(); + for (var i = 0; i < tagIds.Count; i++) + byTag[tagIds[i]] = expanded[i]; + + var winners = new Dictionary<(int ServerId, string Key), (int Depth, long RowId, TagOverrideWinner Winner)>(); + foreach (var row in rows) + { + var route = byTag[row.TagId]; + if (route.Orphaned || route.ExpandedServerIds is null) + continue; + foreach (var serverId in route.ExpandedServerIds) + { + var slot = (serverId, row.Key); + if (winners.TryGetValue(slot, out var current) + && (current.Depth > route.ScopeTagDepth || (current.Depth == route.ScopeTagDepth && current.RowId <= row.RowId))) + continue; + winners[slot] = (route.ScopeTagDepth, row.RowId, new TagOverrideWinner(row.TagId, row.Value)); + } + } + + return winners.ToDictionary(p => p.Key, p => p.Value.Winner); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs new file mode 100644 index 0000000000..b4d9a5ab65 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using NpgsqlTypes; + +namespace PerformanceMonitor.Darling.Storage; + +/// The tag-scoped statements of (#6926): a threshold held for a server tag, the deepest tag +/// a server is under winning as a notification route's does (). +public static partial class ThresholdOverrideStore +{ + private static NpgsqlCommand TagCommand(NpgsqlConnection connection, NpgsqlTransaction transaction, string sql, int tagId, string wire) + { + var command = new NpgsqlCommand(sql, connection) { Transaction = transaction, CommandTimeout = CommandTimeoutSeconds }; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = tagId }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = wire }); + return command; + } + + /// The tag's stored value for a key, or null. + public static async Task ReadTagValueAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, int tagId, string wire, CancellationToken ct) + { + await using var command = TagCommand(connection, transaction, + "SELECT value FROM " + Table + " WHERE scope_tag_id = $1 AND threshold_key = $2", tagId, wire); + return await command.ExecuteScalarAsync(ct) is double d ? d : null; + } + + public static async Task DeleteTagAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, int tagId, string wire, CancellationToken ct) + { + await using var command = TagCommand(connection, transaction, + "DELETE FROM " + Table + " WHERE scope_tag_id = $1 AND threshold_key = $2", tagId, wire); + await command.ExecuteNonQueryAsync(ct); + } + + public static async Task UpsertTagAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, int tagId, string wire, int value, + string setBy, Guid? requestId, CancellationToken ct) + { + await using var command = TagCommand(connection, transaction, + "INSERT INTO " + Table + " (scope_tag_id, threshold_key, value, set_by, request_id) VALUES ($1, $2, $3, $4, $5) " + + "ON CONFLICT (scope_tag_id, threshold_key) WHERE scope_tag_id IS NOT NULL DO UPDATE " + + "SET value = EXCLUDED.value, set_by = EXCLUDED.set_by, request_id = EXCLUDED.request_id, updated_at = now()", + tagId, wire); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Double, Value = (double)value }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = setBy }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Uuid, Value = (object?)requestId ?? DBNull.Value }); + await command.ExecuteNonQueryAsync(ct); + } + + /// Finds a tag from the text a person or an agent gave: a number is a tag id, anything else is a name matched without regard + /// to case. A name two tags share, or one that matches nothing, comes back as a refusal sentence. + public static async Task<(int? TagId, string? Name, string? Problem)> ResolveTagAsync(NpgsqlDataSource postgres, string text, CancellationToken ct) + { + var wanted = text.Trim(); + if (wanted.Length == 0) + return (null, null, "A tag name or id is required."); + var matches = new List<(int Id, string Name)>(); + await using var command = postgres.CreateCommand( + "SELECT id, name FROM config.server_tags WHERE id::text = $1 OR lower(name) = lower($1) ORDER BY id"); + command.CommandTimeout = CommandTimeoutSeconds; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = wanted }); + await using (var reader = await command.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + matches.Add((reader.GetInt32(0), reader.GetString(1))); + } + + var byId = matches.Where(m => m.Id.ToString(CultureInfo.InvariantCulture) == wanted).ToList(); + if (byId.Count == 1) + return (byId[0].Id, byId[0].Name, null); + if (matches.Count == 1) + return (matches[0].Id, matches[0].Name, null); + return matches.Count == 0 + ? (null, null, $"No server tag is named or numbered '{wanted}'.") + : (null, null, $"More than one server tag is named '{wanted}' (ids {string.Join(", ", matches.Select(m => m.Id))}): use the tag id."); + } + + /// The tag list statement: every tag override with its tag's name. A tag that was deleted has no name and is orphaned. + public const string TagListSql = + "SELECT o.scope_tag_id, t.name, o.threshold_key, o.value, o.set_by, o.request_id, o.updated_at " + + "FROM " + Table + " AS o LEFT JOIN config.server_tags AS t ON t.id = o.scope_tag_id " + + "WHERE o.scope_tag_id IS NOT NULL ORDER BY t.name NULLS LAST, o.scope_tag_id, o.threshold_key"; +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs index 3d11d4b0b6..9eb8a4584a 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs @@ -20,7 +20,12 @@ namespace PerformanceMonitor.Darling.Storage; /// override replaces (null when the store has no global row or no such column yet). public sealed record ThresholdOverrideRow( int ServerId, string ServerName, string? Database, string Key, double Value, double? GlobalValue, - string SetBy, Guid? RequestId, DateTime UpdatedAt); + string SetBy, Guid? RequestId, DateTime UpdatedAt, int? TagId = null, bool Orphaned = false) +{ + /// True for a row held for a tag (#6926): is set, is 0, + /// holds the tag's name (empty when the tag was deleted) and says the tag no longer exists, so the row matches nothing. + public bool IsTag => TagId.HasValue; +} /// /// The statements behind config.config_threshold_overrides (#5916), shared by the set_alert_threshold_override tool, @@ -28,7 +33,7 @@ public sealed record ThresholdOverrideRow( /// and transaction (the tool journals its write in the same transaction) or a data source (the list). Key and bound checks are /// not here: they are DarlingThresholdKeys' (PerformanceMonitor.Alerting), applied by every caller before any write. /// -public static class ThresholdOverrideStore +public static partial class ThresholdOverrideStore { /// The table, schema-qualified; the mcp role holds SELECT, INSERT, UPDATE and DELETE on it (#5920). public const string Table = "config.config_threshold_overrides"; @@ -141,6 +146,28 @@ public static async Task> ListAsync( reader.GetString(5), reader.IsDBNull(6) ? null : reader.GetGuid(6), reader.GetDateTime(7))); } + await reader.DisposeAsync(); + + await using var tagCommand = postgres.CreateCommand(TagListSql); + tagCommand.CommandTimeout = CommandTimeoutSeconds; + await using var tagReader = await tagCommand.ExecuteReaderAsync(ct); + while (await tagReader.ReadAsync(ct)) + { + var key = tagReader.GetString(2); + double? globalValue = null; + if (global is { } g && columnFor(key) is { } column + && g.TryGetProperty(column, out var element) && element.ValueKind == JsonValueKind.Number) + { + globalValue = element.GetDouble(); + } + + var name = tagReader.IsDBNull(1) ? null : tagReader.GetString(1); + rows.Add(new ThresholdOverrideRow( + 0, name ?? string.Empty, null, key, tagReader.GetDouble(3), globalValue, + tagReader.GetString(4), tagReader.IsDBNull(5) ? null : tagReader.GetGuid(5), tagReader.GetDateTime(6), + tagReader.GetInt32(0), name is null)); + } + return rows; } } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs index f5e4df60fc..a99e6db89c 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs @@ -25,10 +25,18 @@ public interface IThresholdOverrideGateway Task SaveThresholdOverrideAsync(int serverId, string? database, string key, int value, string setBy, CancellationToken cancellationToken = default); Task RemoveThresholdOverrideAsync(int serverId, string? database, string key, CancellationToken cancellationToken = default); + + /// Sets a tag's override (#6926): the value for every server under the tag and the tags below it. + Task SaveThresholdOverrideForTagAsync(int tagId, string key, int value, string setBy, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + Task RemoveThresholdOverrideForTagAsync(int tagId, string key, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); } -/// One server the add form can pick. -public sealed record ThresholdOverrideServerChoice(int ServerId, string Name); +/// One scope the add form can pick: a server, or ( set, #6926) a server tag, whose +/// reads Tag: name and whose is 0. +public sealed record ThresholdOverrideServerChoice(int ServerId, string Name, int? TagId = null); /// The grid's row: the stored override as the window shows it. public sealed class ThresholdOverrideViewRow @@ -39,6 +47,11 @@ public sealed class ThresholdOverrideViewRow public int ServerId => Row.ServerId; public string ServerName => Row.ServerName; + public bool IsTag => Row.IsTag; + public int? TagId => Row.TagId; + /// The Scope column (#6926): the server's name, or Tag: name for a tag override. A tag override whose tag was deleted reads + /// Tag: (deleted) - orphaned, matches nothing, as an orphaned notification route is marked. + public string ScopeText => !Row.IsTag ? Row.ServerName : Row.Orphaned ? "Tag: (deleted) - orphaned, matches nothing" : "Tag: " + Row.ServerName; public string? Database => Row.Database; public string DatabaseText => Row.Database ?? "Whole server"; public string Key => Row.Key; @@ -91,6 +104,18 @@ public ThresholdOverridesViewModel(IThresholdOverrideGateway gateway, string log return DarlingThresholdKeys.ValidateValue(entry, value, database)?.Message; } + /// Why a tag entry is refused, or null: a tag must be chosen, the key must be a tunable threshold, the value a whole number + /// inside its bounds, and no database is given, because a tag value is the server-level value for every server under the tag. + public static string? ValidateForTag(int? tagId, string? key, string? valueText, string? databaseText, out int value) + { + value = 0; + if (tagId is null) + return "Choose a tag."; + if (!string.IsNullOrWhiteSpace(databaseText)) + return "A tag override holds a server-level value, so it takes no database. Clear the Database box."; + return Validate(0, key, valueText, null, out value); + } + public async Task> LoadAsync(CancellationToken cancellationToken = default) => (await _gateway.ListThresholdOverridesAsync(cancellationToken)).Select(r => new ThresholdOverrideViewRow(r)).ToList(); @@ -115,6 +140,26 @@ public async Task SaveAsync( return new ThresholdOverrideOutcome(true, $"Override saved: {entry.Wire} = {value.ToString(CultureInfo.CurrentCulture)}. The service applies it within its reload interval."); } + /// Checks a tag entry, then writes it (#6926). An entry that fails the check changes nothing. + public async Task SaveForTagAsync( + int? tagId, string? key, string? valueText, string? databaseText, CancellationToken cancellationToken = default) + { + if (ValidateForTag(tagId, key, valueText, databaseText, out var value) is { } refusal) + return new ThresholdOverrideOutcome(false, refusal); + + DarlingThresholdKeys.TryGet(key, out var entry); + try + { + await _gateway.SaveThresholdOverrideForTagAsync(tagId!.Value, entry.Wire, value, _setBy, cancellationToken); + } + catch (ViewerReadOnlyException ex) + { + return new ThresholdOverrideOutcome(false, ex.Message); + } + + return new ThresholdOverrideOutcome(true, $"Tag override saved: {entry.Wire} = {value.ToString(CultureInfo.CurrentCulture)}. The service applies it within its reload interval to every server under the tag that has no override of its own."); + } + /// Removes the chosen override so the global value applies again. public async Task RemoveAsync(ThresholdOverrideViewRow? row, CancellationToken cancellationToken = default) { @@ -122,13 +167,16 @@ public async Task RemoveAsync(ThresholdOverrideViewRow return new ThresholdOverrideOutcome(false, "Select an override to remove."); try { - await _gateway.RemoveThresholdOverrideAsync(row.ServerId, row.Database, row.Key, cancellationToken); + if (row.TagId is { } tagId) + await _gateway.RemoveThresholdOverrideForTagAsync(tagId, row.Key, cancellationToken); + else + await _gateway.RemoveThresholdOverrideAsync(row.ServerId, row.Database, row.Key, cancellationToken); } catch (ViewerReadOnlyException ex) { return new ThresholdOverrideOutcome(false, ex.Message); } - return new ThresholdOverrideOutcome(true, $"Override removed: {row.Key} on {row.ServerName} uses the global value again."); + return new ThresholdOverrideOutcome(true, $"Override removed: {row.Key} on {row.ScopeText} uses the global value again."); } } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml index d2e7718283..7984e4da4f 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml @@ -30,7 +30,7 @@ @@ -40,7 +40,7 @@ IsReadOnly="True" SelectionChanged="OverridesGrid_SelectionChanged"> - + @@ -51,7 +51,7 @@ - - + + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs index c45baa5ac9..28ea7dcb56 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs @@ -59,9 +59,13 @@ private async Task LoadAsync() try { var servers = await _dataService.GetMonitoredServersAsync(); + var tags = await _dataService.GetServerTagsAsync(); ServerCombo.ItemsSource = servers .OrderBy(s => s.Name, StringComparer.OrdinalIgnoreCase) .Select(s => new ThresholdOverrideServerChoice(s.ServerId, s.Name)) + .Concat(tags + .OrderBy(t => t.Name, StringComparer.OrdinalIgnoreCase) + .Select(t => new ThresholdOverrideServerChoice(0, "Tag: " + t.Name, t.Id))) .ToList(); _rows.Clear(); foreach (var row in await _model.LoadAsync()) @@ -85,7 +89,8 @@ private void OverridesGrid_SelectionChanged(object sender, SelectionChangedEvent } /* Selecting a row loads it into the form, so Save changes it and Remove removes it. */ - ServerCombo.SelectedItem = ServerCombo.Items.OfType().FirstOrDefault(s => s.ServerId == row.ServerId); + ServerCombo.SelectedItem = ServerCombo.Items.OfType() + .FirstOrDefault(s => row.IsTag ? s.TagId == row.TagId : s.TagId is null && s.ServerId == row.ServerId); KeyCombo.SelectedItem = row.Key; DatabaseBox.Text = row.Database ?? ""; ValueBox.Text = row.Row.Value.ToString("0.##", System.Globalization.CultureInfo.CurrentCulture); @@ -98,7 +103,9 @@ private async void Save_Click(object sender, RoutedEventArgs e) try { var server = ServerCombo.SelectedItem as ThresholdOverrideServerChoice; - var outcome = await _model.SaveAsync(server?.ServerId, KeyCombo.SelectedItem as string, ValueBox.Text, DatabaseBox.Text); + var outcome = server?.TagId is { } tagId + ? await _model.SaveForTagAsync(tagId, KeyCombo.SelectedItem as string, ValueBox.Text, DatabaseBox.Text) + : await _model.SaveAsync(server?.ServerId, KeyCombo.SelectedItem as string, ValueBox.Text, DatabaseBox.Text); SetStatus(outcome.Message, !outcome.Ok); if (outcome.Ok) { diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs index a9a422a345..20ae73c0de 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs @@ -44,6 +44,37 @@ public async Task SaveThresholdOverrideAsync( } } + public async Task SaveThresholdOverrideForTagAsync( + int tagId, string key, int value, string setBy, CancellationToken cancellationToken = default) + { + try + { + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken); + await ThresholdOverrideStore.UpsertTagAsync(connection, transaction, tagId, key, value, setBy, null, cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + catch (PostgresException ex) when (ex.SqlState == InsufficientPrivilegeSqlState) + { + throw new ViewerReadOnlyException(ex); + } + } + + public async Task RemoveThresholdOverrideForTagAsync(int tagId, string key, CancellationToken cancellationToken = default) + { + try + { + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken); + await ThresholdOverrideStore.DeleteTagAsync(connection, transaction, tagId, key, cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + catch (PostgresException ex) when (ex.SqlState == InsufficientPrivilegeSqlState) + { + throw new ViewerReadOnlyException(ex); + } + } + public async Task RemoveThresholdOverrideAsync( int serverId, string? database, string key, CancellationToken cancellationToken = default) { diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs index 7b42e0ab90..3d9e58a532 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs @@ -1377,7 +1377,10 @@ per the V71 finding. */ EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds' AND NOT attisdropped), /* V226 (#6893) adds the AWS RDS Web or Express fact to the server registry. Found by catalog like the V225 column above. Named only in this probe line, never in prose, per the V71 finding. */ - EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped)"; + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped), + /* V227 (#6926) adds the tag scope to the threshold overrides. Found by catalog like the V225 column above. Named only in this probe line, never in + prose, per the V71 finding. */ + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.config_threshold_overrides') AND attname = 'scope_tag_id' AND NOT attisdropped)"; /// The store schema version this viewer build requires — the highest migration it knows /// (). The connect-time gate blocks a store below this. @@ -1399,7 +1402,7 @@ per the V71 finding. */ await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201)); + return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201), reader.GetBoolean(202)); } return null; @@ -1424,7 +1427,7 @@ per the V71 finding. */ /// is unit-tested without a live store; any schema bump past the newest arm trips the pinning test that keeps /// this in step with . /// - internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false) + internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false, bool hasThresholdOverrideTagScope = false) { /* V71 (the PostgreSQL blocking-edges rung): a table-existence sentinel and now the newest-first arm. A collector table would ordinarily get no arm at all — see the V63-V69 note below — but the TOP @@ -1633,6 +1636,11 @@ store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through t /* V226 (#6893): the AWS RDS Web or Express fact on the server registry. Column-existence sentinel, newest-first, and the TOP rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 225 and showing a spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + /* V227 (#6926): the tag scope on the threshold overrides. Catalog sentinel, the top rung, newest first. */ + if (hasThresholdOverrideTagScope) + { + return 227; + } if (hasServerAwsRdsWebOrExpress) { return 226; diff --git a/docs/darling/admin.md b/docs/darling/admin.md index df819a8722..1668dc998b 100644 --- a/docs/darling/admin.md +++ b/docs/darling/admin.md @@ -189,6 +189,7 @@ An older build ignores what those migrations added, so it can read some newer da - **V219**: an older build does not know the Query Store self-statement keys ([#6750](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6750)). Rolling back leaves `collect.query_store_self_keys`, `collect.query_store_self_keys_state` and the two triggers on `collect.query_store_text` in place. The older build ignores the tables, and the triggers keep the keys current whenever it writes statement text, so when you roll forward again the Query Store regressions read still hides the monitor's own statement. Nothing needs to be done before you roll back. - **V220**: an older build does not know the Job History statistics index ([#6798](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6798)). Rolling back leaves `idx_job_history_job_success` on `collect.job_history` in place. The older build's Job History read can still use it, and the collector keeps it current on every insert. Nothing needs doing before you roll back. Leave the index in place: the viewer recognizes a V220 store by it, so a store whose index was dropped answers 219 to the viewer while its schema version says 220, the viewer refuses it, and the migrator does not build the index again. - **V221**: an older build does not know the Query Store hash-hourly table ([#6800](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6800)). Rolling back leaves `collect.query_store_hash_hourly`, the day tables `collect.query_store_hash_daily`, `collect.query_store_hash_daily_built` and `collect.query_store_hash_daily_failures`, and the `hash_built_seq` column of `collect.query_store_compose_stamp_built` in place. The older build neither reads nor fills them, so the tables stop growing and their hours and days go stale. Nothing to do before rolling back. If you roll forward again, the next hourly passes rebuild the stale hours, newest first, and the next cleanup removes the hash-hourly rows of any hour the older build retired without deleting them (it deletes by the retention floor, not by the hours it just retired) +- **V227**: an older build does not know a threshold override held for a server tag ([#6926](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6926)). Rolling back leaves the `scope_tag_id` column on `config.config_threshold_overrides` (and `server_id` nullable), and any tag override rows stay. An older build reads the whole overrides table when it loads, and a row with no server stops that read, so it applies **no** threshold override at all (server or database ones included) until the tag rows are gone. Before rolling back past V227, remove every tag override (the Viewer's Threshold Overrides window, the web Admin tab, or `set_alert_threshold_override` with `clear` and a tag), or run `DELETE FROM config.config_threshold_overrides WHERE server_id IS NULL;` as the owner. ### Upgrading from a build that predates the script From 20e14a16627173aadc53de5d4cde8a5401416557 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:18:08 -0400 Subject: [PATCH 02/22] Notification routes: match on alert severity, store and MCP half (#6928) A route gains a severities filter (both, critical or warning, default both) in V229, the route functions and grants, the router, the MCP create, update and get tools, and the external-alert PagerDuty close. Viewer and web Routes list editing, tests and docs follow. --- .../ExternalAlertDrainLifecycleLiveTests.cs | 2 +- .../ExternalAlertDrainLiveTests.cs | 2 +- .../DarlingAlertDeliverer.cs | 4 +- .../DarlingManagedRoles.cs | 41 ++++++++---- .../DarlingWorker.cs | 2 +- .../ExternalAlertDrain.cs | 16 +++-- .../Mcp/DarlingMcpAlertTools.RouteAdmin.cs | 64 ++++++++++++++++--- .../Mcp/DarlingMcpAlertTools.cs | 4 ++ .../PgNotificationRouteStore.cs | 6 +- .../StoreConfigProvider.cs | 4 +- .../NotificationRouteWriter.cs | 26 ++++++-- .../PgMigrations.cs | 22 +++++++ Darling/tools/provision-roles.sql | 61 ++++++++++++++---- .../AlertSeverity.cs | 5 ++ .../EmailSendCore.cs | 3 +- .../InvestigationDispatchCore.cs | 9 +-- .../NotificationRoute.cs | 8 +++ .../NotificationRouteKinds.cs | 29 +++++++++ .../NotificationRouter.cs | 18 ++++-- .../WebhookAlertService.cs | 9 ++- 20 files changed, 273 insertions(+), 62 deletions(-) diff --git a/Darling/Darling.Tests/ExternalAlertDrainLifecycleLiveTests.cs b/Darling/Darling.Tests/ExternalAlertDrainLifecycleLiveTests.cs index 1eb7863501..cf7849c8d7 100644 --- a/Darling/Darling.Tests/ExternalAlertDrainLifecycleLiveTests.cs +++ b/Darling/Darling.Tests/ExternalAlertDrainLifecycleLiveTests.cs @@ -512,7 +512,7 @@ await History.RecordAlertAsync(new AlertHistoryRecord( return delivery; }, - (_, _, _, _, _) => Task.FromResult(PagerDutyAnswer), + (_, _, _, _, _, _) => Task.FromResult(PagerDutyAnswer), History, context => IsMuted(context), name => string.Equals(name, "SQL01", StringComparison.OrdinalIgnoreCase) ? new ExternalAlertServer(7, Sql01DisplayName) : null, diff --git a/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs b/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs index f3f818adcb..f235c3f532 100644 --- a/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs +++ b/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs @@ -577,7 +577,7 @@ await History.RecordAlertAsync(new AlertHistoryRecord( AlertContextSerializer.Serialize(outcome.Context!))); return delivery; }, - (metric, serverName, serverKey, _, _) => + (metric, serverName, serverKey, _, _, _) => { PagerDutyResolves.Add(metric.Split(':')[2]); PagerDutyIdentities.Add(serverName + "|" + serverKey); diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs index 47de15553a..b27f3abe38 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs @@ -112,8 +112,8 @@ public DarlingAlertDeliverer( /// public Task TryResolvePagerDutyAsync( string firingMetric, string serverName, string serverKey, string? detailText, - CancellationToken cancellationToken = default) => - _webhooks.TrySendPagerDutyResolveAsync(firingMetric, serverName, serverKey, detailText, cancellationToken); + AlertSeverityLevel? firedSeverity = null, CancellationToken cancellationToken = default) => + _webhooks.TrySendPagerDutyResolveAsync(firingMetric, serverName, serverKey, detailText, firedSeverity, cancellationToken); public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) => DeliverAndReportAsync(outcome, cancellationToken); diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs index 2a4d0333f2..26ea2ae9e2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs @@ -273,6 +273,8 @@ column that exists precisely so a role denied the URLs can still say which chann "send_resolutions", /* V193 (#5997): the fallback route (D8) and when the previous signing key stops being sent. Non-secret. */ "fallback_route_id", "previous_signing_key_until", + /* V229 (#6928): the route's severity filter (both, critical or warning). Non-secret. */ + "severities", }, SecretColumns: new[] { "teams_url", "slack_url", "generic_url", "pagerduty_routing_key", "signing_key", "previous_signing_key" }), }; @@ -1808,8 +1810,8 @@ GRANT UPDATE ({McpMonitoredServerUpdateColumns}) -- (the address list is plain text). All three re-read the row FOR UPDATE and refuse an agent route inside the transaction, -- unless the actor the service passes is an operator-class token (#6542 part 2). Granted to mcp only. {BuildNotificationRouteFunctionsSql(config)} -GRANT EXECUTE ON FUNCTION {config}.create_notification_route(text, boolean, integer, integer, boolean, integer) TO {mcp}; -GRANT EXECUTE ON FUNCTION {config}.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer) TO {mcp}; +GRANT EXECUTE ON FUNCTION {config}.create_notification_route(text, boolean, integer, integer, boolean, integer, text) TO {mcp}; +GRANT EXECUTE ON FUNCTION {config}.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer, text) TO {mcp}; GRANT EXECUTE ON FUNCTION {config}.set_notification_route_destinations(integer, text, text[], text, text, text, text, text) TO {mcp}; GRANT EXECUTE ON FUNCTION {config}.set_notification_route_agent(integer, text, text[], integer, integer, integer, integer, integer, integer, boolean, jsonb) TO {mcp}; -- #6394: the MCP generate_webhook_signing_key and remove_webhook_signing_key tools. mcp holds no UPDATE on a signing key column of {config}.config_notification @@ -2829,13 +2831,16 @@ IF v_last_status IN ('pending', 'in_progress') THEN /// agent route inside the transaction, unless the actor the service passes is an operator-class token (#6542 part 2: MCP is full admin). Created and REVOKEd from PUBLIC here; the caller adds the GRANT EXECUTE to mcp only. /// internal static string BuildNotificationRouteFunctionsSql(string config) => $@" +-- #6928 (V229): p_severities joins the argument list, so the six-argument function is dropped and stays uncallable. NULL is both. +DROP FUNCTION IF EXISTS {config}.create_notification_route(text, boolean, integer, integer, boolean, integer); CREATE OR REPLACE FUNCTION {config}.create_notification_route( p_metric_match text, p_enabled boolean, p_scope_server_id integer, p_scope_tag_id integer, p_send_resolutions boolean, - p_fallback_route_id integer) + p_fallback_route_id integer, + p_severities text DEFAULT NULL) RETURNS integer LANGUAGE plpgsql SECURITY DEFINER @@ -2843,8 +2848,12 @@ SECURITY DEFINER AS $fn$ DECLARE v_match text := btrim(COALESCE(p_metric_match, '')); + v_severities text := lower(btrim(COALESCE(p_severities, 'both'))); v_id integer; BEGIN + IF v_severities NOT IN ('both', 'critical', 'warning') THEN + RAISE EXCEPTION 'severities is critical, warning or both.' USING ERRCODE = 'DL400'; + END IF; IF v_match = '' THEN RAISE EXCEPTION 'Choose a family or enter an exact metric name.' USING ERRCODE = 'DL400'; END IF; @@ -2870,17 +2879,20 @@ AND NOT EXISTS (SELECT 1 FROM {config}.config_notification_routes WHERE route_id -- goes where it went before (the next matching route, else the Settings channels), and no caller of this function can name a URL, a key or an address list. -- The agent columns keep their defaults too: this function never makes an agent route. INSERT INTO {config}.config_notification_routes - (metric_match, enabled, scope_server_id, scope_tag_id, send_resolutions, fallback_route_id, modified_at) + (metric_match, enabled, scope_server_id, scope_tag_id, send_resolutions, fallback_route_id, severities, modified_at) VALUES - (v_match, COALESCE(p_enabled, true), p_scope_server_id, p_scope_tag_id, p_send_resolutions, p_fallback_route_id, + (v_match, COALESCE(p_enabled, true), p_scope_server_id, p_scope_tag_id, p_send_resolutions, p_fallback_route_id, v_severities, (now() AT TIME ZONE 'UTC')) RETURNING route_id INTO v_id; RETURN v_id; END $fn$; -REVOKE ALL ON FUNCTION {config}.create_notification_route(text, boolean, integer, integer, boolean, integer) FROM PUBLIC; +REVOKE ALL ON FUNCTION {config}.create_notification_route(text, boolean, integer, integer, boolean, integer, text) FROM PUBLIC; -- #6542 part 2: the actor is an argument now and the before row comes back, so the old signature is dropped and stays uncallable. DROP FUNCTION IF EXISTS {config}.update_notification_route(integer, text[], text, boolean, integer, integer, boolean, integer); +-- #6928 (V229): p_severities joins the argument list, so the nine-argument function is dropped and stays uncallable. It is read only when +-- p_columns names severities. +DROP FUNCTION IF EXISTS {config}.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer); CREATE OR REPLACE FUNCTION {config}.update_notification_route( p_route_id integer, p_actor text, @@ -2890,7 +2902,8 @@ INSERT INTO {config}.config_notification_routes p_scope_server_id integer, p_scope_tag_id integer, p_send_resolutions boolean, - p_fallback_route_id integer) + p_fallback_route_id integer, + p_severities text DEFAULT NULL) RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER @@ -2907,11 +2920,12 @@ SECURITY DEFINER v_tag integer; v_resolutions boolean; v_fallback integer; + v_severities text; BEGIN IF p_columns IS NULL OR cardinality(p_columns) = 0 OR EXISTS (SELECT 1 FROM unnest(p_columns) AS c(name) - WHERE c.name IS NULL OR c.name NOT IN ('metric_match', 'enabled', 'scope', 'send_resolutions', 'fallback_route_id')) THEN - RAISE EXCEPTION 'The columns to change are metric_match, enabled, scope, send_resolutions and fallback_route_id.' USING ERRCODE = 'DL400'; + WHERE c.name IS NULL OR c.name NOT IN ('metric_match', 'enabled', 'scope', 'send_resolutions', 'fallback_route_id', 'severities')) THEN + RAISE EXCEPTION 'The columns to change are metric_match, enabled, scope, send_resolutions, fallback_route_id and severities.' USING ERRCODE = 'DL400'; END IF; -- The row is read again here, under its lock, so the agent-route refusal and every check below are on what is stored now -- and not on what the caller read a moment ago. @@ -2925,7 +2939,7 @@ IF NOT FOUND THEN -- What a route matches and which servers it covers decide which servers an agent route covers, so a route with an -- explicit agent setting (Off included) keeps both in the Viewer. A route's own destinations do not stop a change -- to what it matches: the destinations are set over MCP too. (#6231) - IF ('metric_match' = ANY (p_columns) OR 'scope' = ANY (p_columns)) AND v_row.agent_level IS NOT NULL AND NOT v_operator THEN + IF ('metric_match' = ANY (p_columns) OR 'scope' = ANY (p_columns) OR 'severities' = ANY (p_columns)) AND v_row.agent_level IS NOT NULL AND NOT v_operator THEN RAISE EXCEPTION 'That route has an agent setting (Off), so what it matches and which servers it covers are changed in the Viewer or on the web dashboard, not over MCP. Turning it on, send_resolutions and fallback_route_id can still be changed here.' USING ERRCODE = 'DL403'; END IF; -- An Off route keeps servers away from a broader agent route, so turning it off widens agent coverage: that is done in the Viewer @@ -2939,6 +2953,10 @@ IF NOT FOUND THEN v_tag := CASE WHEN 'scope' = ANY (p_columns) THEN p_scope_tag_id ELSE v_row.scope_tag_id END; v_resolutions := CASE WHEN 'send_resolutions' = ANY (p_columns) THEN p_send_resolutions ELSE v_row.send_resolutions END; v_fallback := CASE WHEN 'fallback_route_id' = ANY (p_columns) THEN p_fallback_route_id ELSE v_row.fallback_route_id END; + v_severities := CASE WHEN 'severities' = ANY (p_columns) THEN lower(btrim(COALESCE(p_severities, ''))) ELSE v_row.severities END; + IF v_severities NOT IN ('both', 'critical', 'warning') THEN + RAISE EXCEPTION 'severities is critical, warning or both.' USING ERRCODE = 'DL400'; + END IF; IF v_match = '' THEN RAISE EXCEPTION 'Choose a family or enter an exact metric name.' USING ERRCODE = 'DL400'; END IF; @@ -2972,6 +2990,7 @@ AND NOT EXISTS (SELECT 1 FROM {config}.config_notification_routes WHERE route_id scope_tag_id = v_tag, send_resolutions = v_resolutions, fallback_route_id = v_fallback, + severities = v_severities, modified_at = (now() AT TIME ZONE 'UTC') WHERE route_id = p_route_id; -- The row as it was before this change, in the journal's words: set_notification_route_agent journals it as the before of a @@ -2982,7 +3001,7 @@ RETURN jsonb_build_object( 'agent_lease_minutes', v_row.agent_lease_minutes, 'agent_expiry_minutes', v_row.agent_expiry_minutes, 'enabled', v_row.enabled); END $fn$; -REVOKE ALL ON FUNCTION {config}.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer) FROM PUBLIC; +REVOKE ALL ON FUNCTION {config}.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer, text) FROM PUBLIC; -- #6542: a named operator token sets a route's agent level, its four limits and its fallback over MCP through this function, and -- the change is journaled as route_agent_changed under the token's name (operator:), in the same transaction. Levels 0 to 3 are -- set here (MCP is full admin). A level 3 server action still needs that server's switch (config_server_actions) and its remediation login. diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs index efed41c5d0..d294c5b083 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs @@ -3763,7 +3763,7 @@ name stays a label in the server_id 0 bucket. */ externalAlertStore, alertsEnabled: () => alertSettings.AlertsEnabled, (outcome, token) => deliverer.DeliverAndReportAsync(outcome, token), - (metric, serverName, serverKey, detail, token) => deliverer.TryResolvePagerDutyAsync(metric, serverName, serverKey, detail, token), + (metric, serverName, serverKey, detail, severity, token) => deliverer.TryResolvePagerDutyAsync(metric, serverName, serverKey, detail, severity, token), historyStore, muteRuleService.IsAlertMuted, name => diff --git a/Darling/PerformanceMonitor.Darling.Service/ExternalAlertDrain.cs b/Darling/PerformanceMonitor.Darling.Service/ExternalAlertDrain.cs index 1071d31f35..00ffe85a02 100644 --- a/Darling/PerformanceMonitor.Darling.Service/ExternalAlertDrain.cs +++ b/Darling/PerformanceMonitor.Darling.Service/ExternalAlertDrain.cs @@ -75,7 +75,7 @@ public sealed class ExternalAlertDrain private readonly ExternalAlertStore _store; private readonly Func _alertsEnabled; private readonly Func> _deliver; - private readonly Func> _resolvePagerDuty; + private readonly Func> _resolvePagerDuty; private readonly IAlertHistoryStore _history; private readonly Func _isMuted; private readonly Func _resolveServer; @@ -86,7 +86,7 @@ public sealed class ExternalAlertDrain /// The live master switch every alert family consults (#3464). Off, a pass only expires rows. /// The record-and-send seam: . - /// (firing metric, server name, server key, detail, token): closes the PagerDuty incident + /// (firing metric, server name, server key, detail, the severity the fire went out at, token): closes the PagerDuty incident /// of a fire, and only with auto-resolve on: . /// The mute check every family shares (MuteRuleService.IsAlertMuted). /// Maps the submitted server name to a monitored server, or null when it is not registered; @@ -95,7 +95,7 @@ public ExternalAlertDrain( ExternalAlertStore store, Func alertsEnabled, Func> deliver, - Func> resolvePagerDuty, + Func> resolvePagerDuty, IAlertHistoryStore history, Func isMuted, Func resolveServer, @@ -354,6 +354,13 @@ await TryMarkAsync(row.PendingId, ExternalAlertOutcome.Delivered, return ExternalAlertOutcome.Delivered; } + /// The tier a fire went out at (#6928), read back from the state's stored word, so the close is routed as the fire was: + /// to the destination of the route that took that severity. Null when none was confirmed, which every route takes. + private static AlertSeverityLevel? SeverityOfFired(string? firedSeverity) => + string.Equals(firedSeverity, "critical", StringComparison.OrdinalIgnoreCase) ? AlertSeverityLevel.Critical + : string.Equals(firedSeverity, "warning", StringComparison.OrdinalIgnoreCase) ? AlertSeverityLevel.Warning + : null; + private async Task HandleClearAsync(ExternalAlertPendingRow row, CancellationToken cancellationToken) { var server = Identify(row); @@ -376,7 +383,8 @@ fire must not move the incident this clear closes. */ /* The PagerDuty resolve goes first: it posts only with auto-resolve on and a PagerDuty destination for this alert, and if it fails the firing state is left alone so the retry still finds a fire to close. */ - var pagerDuty = await _resolvePagerDuty(firingMetric, firedLabel, firedKey, message, cancellationToken); + var pagerDuty = await _resolvePagerDuty( + firingMetric, firedLabel, firedKey, message, SeverityOfFired(state.FiredSeverity), cancellationToken); if (pagerDuty == AlertChannelOutcome.Failed) { return await RetryOrFailAsync(row, "the PagerDuty resolve failed"); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.RouteAdmin.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.RouteAdmin.cs index d48b45dd0b..3534215f48 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.RouteAdmin.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.RouteAdmin.cs @@ -48,7 +48,7 @@ public sealed partial class DarlingMcpAlertTools "Creates a route that decides WHICH alerts it covers and, when you pass them, where they go. " + DestinationsHelp + " " + "With no destination field the route is stored with every destination empty, so it has none of its own and each channel goes where it went before: to the next less specific matching route that sets that channel, else to the Settings > Notifications channels. " + "What it can set: metric_match (" + MatchHelp + "), a scope of one server or one tag (not both; * only with a scope), enabled, " + - "send_resolutions (inherit, always or never: the clear message to the channels this route resolves to) and fallback_route_id (an existing route). " + + "send_resolutions (inherit, always or never: the clear message to the channels this route resolves to), fallback_route_id (an existing route) and severities (" + SeveritiesHelp + "). " + "The database role this tool runs as has no INSERT on the routes table and no read or write on a destination column: the route is written by the store's own " + "config.create_notification_route function and its destinations by config.set_notification_route_destinations, in one transaction, so a destination that is refused leaves no route behind. " + "A destination is sealed for this route's own row, so it cannot be copied to another route. scope_server_id and scope_tag_id are ids from get_notification_routes or the server and tag reads; an id that names nothing is invalid. " + @@ -61,6 +61,7 @@ public static Task CreateNotificationRoute( [Description("Scope the route to this server tag id and every tag under it. Omit for fleet-wide. Not with scope_server_id.")] int? scope_tag_id = null, [Description("inherit (default: each channel's own setting), always (send the clear message to the channels this route resolves to) or never.")] string? send_resolutions = null, [Description("The route_id of an existing route an expired investigation request goes to first. Omit for none.")] int? fallback_route_id = null, + [Description("Which alerts the route takes by severity: critical, warning or both (default, case-insensitive). An alert that is not Critical counts as Warning.")] string? severities = null, [Description("The route's own Teams webhook URL (absolute http or https). Sealed for this route; never shown back. Omit for none.")] string? teams_webhook_url = null, [Description("The route's own Slack webhook URL (absolute http or https). Sealed for this route; never shown back. Omit for none.")] string? slack_webhook_url = null, [Description("The route's own generic webhook URL (absolute http or https). Sealed for this route; never shown back. Omit for none.")] string? generic_webhook_url = null, @@ -80,13 +81,13 @@ public static Task CreateNotificationRoute( new RouteDestinationRequest(teams_webhook_url, slack_webhook_url, generic_webhook_url, pagerduty_routing_key, email_recipients), DarlingPasswordKey.Current, logger, cancellationToken, new RouteAgentRequest(agent_level, agent_max_per_hour, agent_max_per_day, agent_lease_minutes, agent_expiry_minutes), - served?.Policy.OperatorActor, StoreRolesAreRunByOperator(postgresConfig)); + served?.Policy.OperatorActor, StoreRolesAreRunByOperator(postgresConfig), severities); /// The create: the route and its destinations in one transaction (a sealed value is bound to the route's id, so the id comes first). internal static async Task CreateNotificationRouteAsync( NpgsqlDataSource postgres, string metric_match, bool enabled, int? scope_server_id, int? scope_tag_id, string? send_resolutions, int? fallback_route_id, RouteDestinationRequest destinations, IPasswordKeyRing ring, ILogger? logger, CancellationToken cancellationToken, - RouteAgentRequest? agent = null, string? operatorActor = null, bool selfManagedStore = true) + RouteAgentRequest? agent = null, string? operatorActor = null, bool selfManagedStore = true, string? severities = null) { try { @@ -95,6 +96,11 @@ internal static async Task CreateNotificationRouteAsync( return Outcome("invalid", badResolutions); } + if (ParseSeverities(severities, out var severitiesWord) is { } badSeverities) + { + return Outcome("invalid", badSeverities); + } + var match = NormalizeRouteMatch(metric_match); agent ??= RouteAgentRequest.None; if (agent.Any) @@ -122,7 +128,7 @@ internal static async Task CreateNotificationRouteAsync( : new WebhookProxies("", "", "", ""); await using var connection = await postgres.OpenConnectionAsync(cancellationToken); await using var transaction = await connection.BeginTransactionAsync(cancellationToken); - const string createSql = "SELECT config.create_notification_route($1, $2, $3, $4, $5, $6)"; + const string createSql = "SELECT config.create_notification_route($1, $2, $3, $4, $5, $6, $7)"; await using (var command = new NpgsqlCommand(createSql, connection) { Transaction = transaction }) { command.CommandTimeout = McpCommandDeadlines.ReadSeconds; @@ -134,6 +140,7 @@ internal static async Task CreateNotificationRouteAsync( ? new NpgsqlParameter { TypedValue = resolutions.Value } : new NpgsqlParameter { Value = DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Boolean }); NotificationRouteWriter.AddNullableInt(command, fallback_route_id); + command.Parameters.Add(new NpgsqlParameter { TypedValue = severitiesWord }); routeId = Convert.ToInt32(await command.ExecuteScalarAsync(cancellationToken), System.Globalization.CultureInfo.InvariantCulture); } @@ -158,7 +165,7 @@ internal static async Task CreateNotificationRouteAsync( return Outcome("invalid", ex.Message); } - var fields = "metric_match, enabled, scope, send_resolutions, fallback_route_id" + (change is { Fields.Count: > 0 } ? ", " + DestinationFieldNames(change) : "") + var fields = "metric_match, enabled, scope, send_resolutions, fallback_route_id, severities" + (change is { Fields.Count: > 0 } ? ", " + DestinationFieldNames(change) : "") + (agent.Any ? ", " + string.Join(", ", agent.Columns) : ""); LogRouteWrite(logger, "create_notification_route", CallerOf(), routeId, fields); var stored = (await new PgNotificationRouteStore(postgres).LoadSummariesAsync(cancellationToken)).FirstOrDefault(r => r.RouteId == routeId); @@ -188,7 +195,7 @@ internal static async Task CreateNotificationRouteAsync( "An operator token changes any route, an agent route included: its match, scope, enabled flag, send_resolutions, fallback, destinations and agent fields. A change to the agent fields, or to the match, scope or enabled flag of a route with an agent setting, is journaled as route_agent_changed under operator: and the token's name. " + "A route's own destinations do not stop a change to what it matches. " + "Fields: metric_match (" + MatchHelp + "), enabled, scope (scope_server_id or scope_tag_id, the other is cleared; fleet_wide true clears both), " + - "send_resolutions (inherit, always or never) and fallback_route_id (an existing route; clear_fallback true removes it), plus the destination fields. A field left out keeps its stored value. " + + "send_resolutions (inherit, always or never), severities (" + SeveritiesHelp + ") and fallback_route_id (an existing route; clear_fallback true removes it), plus the destination fields. A field left out keeps its stored value. " + "An operator token sets agent_level 0 to 3, or clear_agent_level true to put the level back to Inherit (null: the next level decides), not both. Clearing the level keeps the four limits, as the Viewer's Inherit does. " + "The same rules the Viewer's route window applies hold: * needs a scope, a route is scoped to one server or one tag, a route is not its own fallback, a webhook is an absolute http or https URL, recipients are plain addresses. " + "A route whose generic webhook has a signing key keeps its generic URL until the key is removed (remove_webhook_signing_key, or the Viewer). " + @@ -210,6 +217,7 @@ public static Task UpdateNotificationRoute( [Description("inherit, always or never. Omit to keep.")] string? send_resolutions = null, [Description("The route_id of an existing route an expired investigation request goes to first. Omit to keep.")] int? fallback_route_id = null, [Description("true removes the fallback route.")] bool? clear_fallback = null, + [Description("Which alerts the route takes by severity: critical, warning or both (case-insensitive). Omit to keep.")] string? severities = null, [Description("The route's own Teams webhook URL (absolute http or https). Sealed for this route; never shown back. Omit to keep; - removes it.")] string? teams_webhook_url = null, [Description("The route's own Slack webhook URL (absolute http or https). Sealed for this route; never shown back. Omit to keep; - removes it.")] string? slack_webhook_url = null, [Description("The route's own generic webhook URL (absolute http or https). Sealed for this route; never shown back. Omit to keep; - removes it.")] string? generic_webhook_url = null, @@ -230,14 +238,14 @@ public static Task UpdateNotificationRoute( new RouteDestinationRequest(teams_webhook_url, slack_webhook_url, generic_webhook_url, pagerduty_routing_key, email_recipients), DarlingPasswordKey.Current, logger, cancellationToken, new RouteAgentRequest(agent_level, agent_max_per_hour, agent_max_per_day, agent_lease_minutes, agent_expiry_minutes, clear_agent_level == true), - served?.Policy.OperatorActor, StoreRolesAreRunByOperator(postgresConfig)); + served?.Policy.OperatorActor, StoreRolesAreRunByOperator(postgresConfig), severities); /// The edit: the match, scope and flag columns and the destinations in one transaction. internal static async Task UpdateNotificationRouteAsync( NpgsqlDataSource postgres, int route_id, string? metric_match, bool? enabled, int? scope_server_id, int? scope_tag_id, bool? fleet_wide, string? send_resolutions, int? fallback_route_id, bool? clear_fallback, RouteDestinationRequest destinations, IPasswordKeyRing ring, ILogger? logger, CancellationToken cancellationToken, RouteAgentRequest? agent = null, string? operatorActor = null, - bool selfManagedStore = true) + bool selfManagedStore = true, string? severities = null) { try { @@ -327,6 +335,21 @@ internal static async Task UpdateNotificationRouteAsync( } } + var severitiesNow = existing.Severities; + if (severities is not null) + { + if (ParseSeverities(severities, out var parsedSeverities) is { } badSeverities) + { + return Outcome("invalid", badSeverities); + } + + if (parsedSeverities != existing.Severities) + { + severitiesNow = parsedSeverities; + columns.Add("severities"); + } + } + var fallback = existing.FallbackRouteId; if (clear_fallback == true) { @@ -393,7 +416,7 @@ internal static async Task UpdateNotificationRouteAsync( string? before = null; if (columns.Count > 0) { - const string updateSql = "SELECT config.update_notification_route($1, $2, $3, $4, $5, $6, $7, $8, $9)::text"; + const string updateSql = "SELECT config.update_notification_route($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)::text"; await using var command = new NpgsqlCommand(updateSql, connection) { Transaction = transaction }; command.CommandTimeout = McpCommandDeadlines.ReadSeconds; command.Parameters.Add(new NpgsqlParameter { TypedValue = route_id }); @@ -407,6 +430,7 @@ internal static async Task UpdateNotificationRouteAsync( ? new NpgsqlParameter { TypedValue = resolutions.Value } : new NpgsqlParameter { Value = DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Boolean }); NotificationRouteWriter.AddNullableInt(command, fallback); + command.Parameters.Add(new NpgsqlParameter { TypedValue = severitiesNow }); before = await command.ExecuteScalarAsync(cancellationToken) as string; } @@ -618,6 +642,7 @@ internal static void LogRouteWrite(ILogger? logger, string tool, string caller, ["metric_match"] = existing.MetricMatch, ["enabled"] = existing.Enabled, ["send_resolutions"] = existing.SendResolutions switch { true => "always", false => "never", _ => "inherit" }, + ["severities"] = existing.Severities, }; if (existing.ScopeServerId.HasValue) { @@ -673,6 +698,27 @@ private static string NormalizeRouteMatch(string? match) return AlertFamily.NormalizeFamily(text) ?? text; } + private const string SeveritiesHelp = "critical, warning or both; both is the default and takes every alert. An alert that is not Critical counts as Warning. A route whose filter excludes an alert does not match it, and matching goes on to the next route and ends at the default channels"; + + /// The severity filter word (#6928): critical, warning or both, any case. Blank is both. Anything else is refused with the sentence returned. + private static string? ParseSeverities(string? text, out string word) + { + word = NotificationRouteKinds.SeveritiesBoth; + var trimmed = (text ?? "").Trim().ToLowerInvariant(); + if (trimmed.Length == 0) + { + return null; + } + + if (trimmed is NotificationRouteKinds.SeveritiesBoth or NotificationRouteKinds.SeveritiesCritical or NotificationRouteKinds.SeveritiesWarning) + { + word = trimmed; + return null; + } + + return "severities is critical, warning or both."; + } + private static string? ParseSendResolutions(string? text, out bool? value) { value = null; diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs index b2686932c9..81dda2d9df 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs @@ -698,6 +698,7 @@ than a second opt-in behind one. The thresholds take the house _threshold_ "and agent_max_per_day; Darling only, Lite has no agent channel); it never changes a destination. " + "send_resolutions says whether the route's destinations also get the message that an alert has cleared: null " + "follows each channel's setting (get_alert_settings delivery.send_resolutions), true always sends it, false never does. " + + "severities (critical, warning or both) says which alerts the route takes by severity: both is the default and takes every alert, and a route whose filter excludes an alert does not match it, so matching goes on to the next route. " + "Routes are authored in the Viewer's Settings window or here (create_notification_route / update_notification_route); " + "they can also be disabled or deleted (set_notification_route_enabled / delete_notification_route).")] public static async Task GetNotificationRoutes( @@ -972,6 +973,7 @@ private static object RouteRecreateUndo(PgNotificationRouteStore.RouteSummary ro ["metric_match"] = route.MetricMatch, ["enabled"] = route.Enabled, ["send_resolutions"] = route.SendResolutions switch { true => "always", false => "never", _ => "inherit" }, + ["severities"] = route.Severities, }; if (route.ScopeServerId.HasValue) { @@ -1036,6 +1038,8 @@ private static object RouteRecreateUndo(PgNotificationRouteStore.RouteSummary ro /* #5825: null follows each channel's "send resolutions" setting (get_alert_settings, delivery.send_resolutions); true or false overrides it for this route's destinations. */ send_resolutions = r.SendResolutions, + /* #6928 (V229): which alerts the route takes by severity: both, critical or warning. */ + severities = r.Severities, /* #6036 (D8 of #5632 Phase C): the route an expired investigation request goes to first. Null = none; the name is the fallback route's metric_match. */ fallback_route_id = r.FallbackRouteId, diff --git a/Darling/PerformanceMonitor.Darling.Service/PgNotificationRouteStore.cs b/Darling/PerformanceMonitor.Darling.Service/PgNotificationRouteStore.cs index 6b6a3116ac..065b5f88db 100644 --- a/Darling/PerformanceMonitor.Darling.Service/PgNotificationRouteStore.cs +++ b/Darling/PerformanceMonitor.Darling.Service/PgNotificationRouteStore.cs @@ -89,6 +89,9 @@ public sealed record RouteSummary( /// The route an expired investigation request goes to first (#6036, D8 of #5632 Phase C), or null for none. public int? FallbackRouteId { get; init; } + /// The severity filter (V229, #6928): both, critical or warning. + public string Severities { get; init; } = NotificationRouteKinds.SeveritiesBoth; + /// The fallback route's name (its metric_match), or null when there is no fallback. public string? FallbackRouteName { get; init; } @@ -103,7 +106,7 @@ public sealed record RouteSummary( /// the one a superuser-run test never notices. public const string SelectSummariesSql = @" SELECT route_id, metric_match, configured_channels, smtp_recipients, enabled, modified_at, scope_server_id, scope_tag_id, - agent_level, agent_max_per_hour, agent_max_per_day, agent_lease_minutes, agent_expiry_minutes, send_resolutions, fallback_route_id + agent_level, agent_max_per_hour, agent_max_per_day, agent_lease_minutes, agent_expiry_minutes, send_resolutions, fallback_route_id, severities FROM config_notification_routes ORDER BY route_id"; @@ -170,6 +173,7 @@ public async Task> LoadSummariesAsync(CancellationTo AgentExpiryMinutes = reader.GetInt32(12), SendResolutions = reader.IsDBNull(13) ? null : reader.GetBoolean(13), FallbackRouteId = reader.IsDBNull(14) ? null : reader.GetInt32(14), + Severities = NotificationRouteKinds.NormalizeSeverities(reader.GetString(15)), }); } diff --git a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs index 965aa93a3a..ada08f1a56 100644 --- a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs +++ b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs @@ -2778,7 +2778,7 @@ internal static void MarkSlotsTheFileDeclares(MonitoredServer server, DarlingCon public const string NotificationRoutesSelectSql = "SELECT route_id, metric_match, teams_url, slack_url, generic_url, pagerduty_routing_key, smtp_recipients, enabled, scope_server_id, scope_tag_id, " + "agent_level, agent_max_per_hour, agent_max_per_day, agent_lease_minutes, agent_expiry_minutes, send_resolutions, " - + NotificationRouteSigningColumns + ", fallback_route_id " + + NotificationRouteSigningColumns + ", fallback_route_id, severities " + "FROM config_notification_routes ORDER BY route_id"; /// A route's signing key columns (V193, #5998), last in both the service's and the viewer's route SELECT. @@ -2901,6 +2901,8 @@ internal static async Task> ReadNotificationRou clockAhead), /* V193 (#6003, D8): where an unclaimed, expired investigation request goes instead. */ FallbackRouteId = reader.IsDBNull(19) ? null : reader.GetInt32(19), + /* V229 (#6928): the severity filter; a word this build does not know reads as both. */ + Severities = NotificationRouteKinds.NormalizeSeverities(reader.GetString(20)), }); } } diff --git a/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs b/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs index 07184ec9d9..27433670af 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs @@ -62,6 +62,9 @@ public sealed class NotificationRouteDraft public bool? SendResolutions { get; set; } public int? FallbackRouteId { get; set; } + /// The severity filter (V229, #6928): both, critical or warning. + public string Severities { get; set; } = NotificationRouteKinds.SeveritiesBoth; + /// When the route was last saved, as the writer's screen read it (the store's modified_at, UTC). On a stored draft /// handed to it is the version the save expects: a route saved by anyone since is refused /// as a conflict instead of overwritten (#6274). Null or default skips the check. @@ -93,8 +96,8 @@ public static class NotificationRouteWriter /// Inserts one route. route_id is GENERATED ALWAYS AS IDENTITY, so it is never in the column list. public const string InsertSql = @" INSERT INTO config_notification_routes - (metric_match, " + DestinationColumns + @", enabled, scope_server_id, scope_tag_id, " + AgentColumns + @", send_resolutions, fallback_route_id, modified_at) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, (now() AT TIME ZONE 'UTC')) + (metric_match, " + DestinationColumns + @", enabled, scope_server_id, scope_tag_id, " + AgentColumns + @", send_resolutions, fallback_route_id, severities, modified_at) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, (now() AT TIME ZONE 'UTC')) RETURNING route_id"; public const string UpdateSql = @" @@ -115,6 +118,7 @@ UPDATE config_notification_routes SET agent_expiry_minutes = $15, send_resolutions = $16, fallback_route_id = $17, + severities = $18, modified_at = (now() AT TIME ZONE 'UTC') WHERE route_id = $1"; @@ -215,7 +219,7 @@ await InvestigationPersonJournal.InsertAsync( /// plaintext) and the address list. Only a writer with the service's login reads them. public const string SelectStoredSql = "SELECT route_id, metric_match, teams_url, slack_url, generic_url, pagerduty_routing_key, smtp_recipients, enabled, " - + "scope_server_id, scope_tag_id, " + AgentColumns + ", send_resolutions, fallback_route_id, modified_at " + + "scope_server_id, scope_tag_id, " + AgentColumns + ", send_resolutions, fallback_route_id, modified_at, severities " + "FROM config_notification_routes WHERE route_id = $1"; /// The proxy of each webhook channel on the settings row (the row id 1 the Viewer reads). @@ -272,6 +276,11 @@ await InvestigationPersonJournal.InsertAsync( return "A route cannot be its own fallback route."; } + if (SeveritiesRefusal(row.Severities) is { } severities) + { + return severities; + } + if (row.AgentLevel is { } level && level is not (NotificationRoute.AgentLevelOff or NotificationRoute.AgentLevelInvestigate or NotificationRoute.AgentLevelAct or NotificationRoute.AgentLevelServerAct)) { return "The agent channel is Inherit, Off, Investigate, Act or Act on servers."; @@ -392,6 +401,12 @@ public static (int Count, string[] Domains) DescribeRecipients(string? recipient return await command.ExecuteScalarAsync(cancellationToken) is null ? $"Fallback route {fallbackId} does not exist." : null; } + /// The sentence for a severity filter that is not both, critical or warning (case-insensitive), else null (#6928). + public static string? SeveritiesRefusal(string? severities) => + severities is not null && NotificationRouteKinds.NormalizeSeverities(severities) != severities.Trim().ToLowerInvariant() + ? "severities is critical, warning or both." + : null; + /// Binds the agent columns, the resolution override and the fallback, in the order every insert and update takes them. public static void AddAgentColumns(NpgsqlCommand command, NotificationRouteDraft row) { @@ -410,6 +425,8 @@ public static void AddAgentColumns(NpgsqlCommand command, NotificationRouteDraft : new NpgsqlParameter { Value = DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Boolean }); /* #6036: the fallback route, bound last. NULL is none. */ AddNullableInt(command, row.FallbackRouteId); + /* #6928 (V229): the severity filter, bound after the fallback. */ + command.Parameters.Add(new NpgsqlParameter { TypedValue = NotificationRouteKinds.NormalizeSeverities(row.Severities) }); } /// Binds an integer that may be SQL NULL. @@ -633,6 +650,7 @@ private static NotificationRouteDraft ReadStoredRow(NpgsqlDataReader reader) SendResolutions = reader.IsDBNull(15) ? null : reader.GetBoolean(15), FallbackRouteId = reader.IsDBNull(16) ? null : reader.GetInt32(16), ModifiedAtUtc = reader.IsDBNull(17) ? null : DateTime.SpecifyKind(reader.GetDateTime(17), DateTimeKind.Utc), + Severities = reader.GetString(18), }; } @@ -727,7 +745,7 @@ public static async Task SaveAsync( update.Parameters.Add(new NpgsqlParameter { TypedValue = resolved.Enabled }); // $8 AddNullableInt(update, resolved.ScopeServerId); // $9 AddNullableInt(update, resolved.ScopeTagId); // $10 - AddAgentColumns(update, resolved); // $11-$17 + AddAgentColumns(update, resolved); // $11-$18 await update.ExecuteNonQueryAsync(cancellationToken); if (login is not null && AgentChange(before, resolved) is { } change) { diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index c65424e31c..984edc5715 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -2975,6 +2975,7 @@ holds the ordering. */ new Migration(224, "identity-capacity", V224Sql), new Migration(225, "server-aws-rds", V225Sql), new Migration(226, "server-aws-rds-web-or-express", V226Sql), + new Migration(229, "notification-route-severities", V229Sql), }; /// @@ -3002,6 +3003,27 @@ ALTER TABLE collect.servers ALTER TABLE collect.servers ADD COLUMN IF NOT EXISTS is_aws_rds_web_or_express boolean;"; + /// + /// V229 (#6928): config.config_notification_routes.severities, the severity filter of a notification route: both (the + /// default, Warning and Critical alerts alike, which is what every route did before), critical or warning. NOT NULL with + /// the default, so every stored route reads both and keeps its effect, and a CHECK holds the three words. An alert that is not + /// Critical counts as Warning. A route whose filter excludes an alert does not match it, and matching goes on to the next route. + /// Idempotent: the column and the constraint are each added only when absent. No backfill and no view refresh: a registry table. + /// + private const string V229Sql = @" +ALTER TABLE config.config_notification_routes + ADD COLUMN IF NOT EXISTS severities text NOT NULL DEFAULT 'both'; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'ck_config_notification_routes_severities' + AND conrelid = 'config.config_notification_routes'::regclass) THEN + ALTER TABLE config.config_notification_routes + ADD CONSTRAINT ck_config_notification_routes_severities + CHECK (severities IN ('both', 'critical', 'warning')); + END IF; +END +$$;"; + /// /// V189 (#5867): collect.query_store_compose_stamp_failures, the failure record of the Query Store compose builder: one row /// per hour whose last build threw, with its consecutive failures, last error and next attempt, so a build that cannot finish backs diff --git a/Darling/tools/provision-roles.sql b/Darling/tools/provision-roles.sql index 934dd85c4e..cd7761d109 100644 --- a/Darling/tools/provision-roles.sql +++ b/Darling/tools/provision-roles.sql @@ -599,6 +599,15 @@ $v184$; GRANT SELECT (send_resolutions) ON config.config_notification_routes TO viewer; $v187$; END IF; + -- V229 (#6928): a route's severity filter. Non-secret. Granted on its own column inside the routes table's guard, so a + -- store below V229 still commits. + IF EXISTS (SELECT 1 FROM pg_catalog.pg_attribute + WHERE attrelid = 'config.config_notification_routes'::regclass + AND attname = 'severities' AND NOT attisdropped) THEN + EXECUTE $v229$ +GRANT SELECT (severities) ON config.config_notification_routes TO viewer; +$v229$; + END IF; END IF; END $$; -- V187 (#5825): the four per-channel "send resolutions" flags. Non-secret (behaviour toggles, like @@ -715,6 +724,15 @@ $v184$; GRANT SELECT (send_resolutions) ON config.config_notification_routes TO mcp; $v187$; END IF; + -- V229 (#6928): a route's severity filter. Non-secret. Granted on its own column inside the routes table's guard, so a + -- store below V229 still commits. + IF EXISTS (SELECT 1 FROM pg_catalog.pg_attribute + WHERE attrelid = 'config.config_notification_routes'::regclass + AND attname = 'severities' AND NOT attisdropped) THEN + EXECUTE $v229$ +GRANT SELECT (severities) ON config.config_notification_routes TO mcp; +$v229$; + END IF; END IF; END $$; -- V187 (#5825): the four per-channel "send resolutions" flags. Non-secret (behaviour toggles, like @@ -1533,13 +1551,16 @@ GRANT EXECUTE ON FUNCTION config.request_analyze_now(integer, text) TO viewer; -- (the address list is plain text). All three re-read the row FOR UPDATE and refuse an agent route inside the transaction, -- unless the actor the service passes is an operator-class token (#6542 part 2). Granted to mcp only. BEGIN; +-- #6928 (V229): p_severities joins the argument list, so the six-argument function is dropped and stays uncallable. NULL is both. +DROP FUNCTION IF EXISTS config.create_notification_route(text, boolean, integer, integer, boolean, integer); CREATE OR REPLACE FUNCTION config.create_notification_route( p_metric_match text, p_enabled boolean, p_scope_server_id integer, p_scope_tag_id integer, p_send_resolutions boolean, - p_fallback_route_id integer) + p_fallback_route_id integer, + p_severities text DEFAULT NULL) RETURNS integer LANGUAGE plpgsql SECURITY DEFINER @@ -1547,8 +1568,12 @@ SET search_path = pg_catalog, pg_temp AS $fn$ DECLARE v_match text := btrim(COALESCE(p_metric_match, '')); + v_severities text := lower(btrim(COALESCE(p_severities, 'both'))); v_id integer; BEGIN + IF v_severities NOT IN ('both', 'critical', 'warning') THEN + RAISE EXCEPTION 'severities is critical, warning or both.' USING ERRCODE = 'DL400'; + END IF; IF v_match = '' THEN RAISE EXCEPTION 'Choose a family or enter an exact metric name.' USING ERRCODE = 'DL400'; END IF; @@ -1574,17 +1599,20 @@ BEGIN -- goes where it went before (the next matching route, else the Settings channels), and no caller of this function can name a URL, a key or an address list. -- The agent columns keep their defaults too: this function never makes an agent route. INSERT INTO config.config_notification_routes - (metric_match, enabled, scope_server_id, scope_tag_id, send_resolutions, fallback_route_id, modified_at) + (metric_match, enabled, scope_server_id, scope_tag_id, send_resolutions, fallback_route_id, severities, modified_at) VALUES - (v_match, COALESCE(p_enabled, true), p_scope_server_id, p_scope_tag_id, p_send_resolutions, p_fallback_route_id, + (v_match, COALESCE(p_enabled, true), p_scope_server_id, p_scope_tag_id, p_send_resolutions, p_fallback_route_id, v_severities, (now() AT TIME ZONE 'UTC')) RETURNING route_id INTO v_id; RETURN v_id; END $fn$; -REVOKE ALL ON FUNCTION config.create_notification_route(text, boolean, integer, integer, boolean, integer) FROM PUBLIC; +REVOKE ALL ON FUNCTION config.create_notification_route(text, boolean, integer, integer, boolean, integer, text) FROM PUBLIC; -- #6542 part 2: the actor is an argument now and the before row comes back, so the old signature is dropped and stays uncallable. DROP FUNCTION IF EXISTS config.update_notification_route(integer, text[], text, boolean, integer, integer, boolean, integer); +-- #6928 (V229): p_severities joins the argument list, so the nine-argument function is dropped and stays uncallable. It is read only when +-- p_columns names severities. +DROP FUNCTION IF EXISTS config.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer); CREATE OR REPLACE FUNCTION config.update_notification_route( p_route_id integer, p_actor text, @@ -1594,7 +1622,8 @@ CREATE OR REPLACE FUNCTION config.update_notification_route( p_scope_server_id integer, p_scope_tag_id integer, p_send_resolutions boolean, - p_fallback_route_id integer) + p_fallback_route_id integer, + p_severities text DEFAULT NULL) RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER @@ -1611,11 +1640,12 @@ DECLARE v_tag integer; v_resolutions boolean; v_fallback integer; + v_severities text; BEGIN IF p_columns IS NULL OR cardinality(p_columns) = 0 OR EXISTS (SELECT 1 FROM unnest(p_columns) AS c(name) - WHERE c.name IS NULL OR c.name NOT IN ('metric_match', 'enabled', 'scope', 'send_resolutions', 'fallback_route_id')) THEN - RAISE EXCEPTION 'The columns to change are metric_match, enabled, scope, send_resolutions and fallback_route_id.' USING ERRCODE = 'DL400'; + WHERE c.name IS NULL OR c.name NOT IN ('metric_match', 'enabled', 'scope', 'send_resolutions', 'fallback_route_id', 'severities')) THEN + RAISE EXCEPTION 'The columns to change are metric_match, enabled, scope, send_resolutions, fallback_route_id and severities.' USING ERRCODE = 'DL400'; END IF; -- The row is read again here, under its lock, so the agent-route refusal and every check below are on what is stored now -- and not on what the caller read a moment ago. @@ -1629,7 +1659,7 @@ BEGIN -- What a route matches and which servers it covers decide which servers an agent route covers, so a route with an -- explicit agent setting (Off included) keeps both in the Viewer. A route's own destinations do not stop a change -- to what it matches: the destinations are set over MCP too. (#6231) - IF ('metric_match' = ANY (p_columns) OR 'scope' = ANY (p_columns)) AND v_row.agent_level IS NOT NULL AND NOT v_operator THEN + IF ('metric_match' = ANY (p_columns) OR 'scope' = ANY (p_columns) OR 'severities' = ANY (p_columns)) AND v_row.agent_level IS NOT NULL AND NOT v_operator THEN RAISE EXCEPTION 'That route has an agent setting (Off), so what it matches and which servers it covers are changed in the Viewer or on the web dashboard, not over MCP. Turning it on, send_resolutions and fallback_route_id can still be changed here.' USING ERRCODE = 'DL403'; END IF; -- An Off route keeps servers away from a broader agent route, so turning it off widens agent coverage: that is done in the Viewer @@ -1643,6 +1673,10 @@ BEGIN v_tag := CASE WHEN 'scope' = ANY (p_columns) THEN p_scope_tag_id ELSE v_row.scope_tag_id END; v_resolutions := CASE WHEN 'send_resolutions' = ANY (p_columns) THEN p_send_resolutions ELSE v_row.send_resolutions END; v_fallback := CASE WHEN 'fallback_route_id' = ANY (p_columns) THEN p_fallback_route_id ELSE v_row.fallback_route_id END; + v_severities := CASE WHEN 'severities' = ANY (p_columns) THEN lower(btrim(COALESCE(p_severities, ''))) ELSE v_row.severities END; + IF v_severities NOT IN ('both', 'critical', 'warning') THEN + RAISE EXCEPTION 'severities is critical, warning or both.' USING ERRCODE = 'DL400'; + END IF; IF v_match = '' THEN RAISE EXCEPTION 'Choose a family or enter an exact metric name.' USING ERRCODE = 'DL400'; END IF; @@ -1676,6 +1710,7 @@ BEGIN scope_tag_id = v_tag, send_resolutions = v_resolutions, fallback_route_id = v_fallback, + severities = v_severities, modified_at = (now() AT TIME ZONE 'UTC') WHERE route_id = p_route_id; -- The row as it was before this change, in the journal's words: set_notification_route_agent journals it as the before of a @@ -1686,7 +1721,7 @@ BEGIN 'agent_lease_minutes', v_row.agent_lease_minutes, 'agent_expiry_minutes', v_row.agent_expiry_minutes, 'enabled', v_row.enabled); END $fn$; -REVOKE ALL ON FUNCTION config.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer) FROM PUBLIC; +REVOKE ALL ON FUNCTION config.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer, text) FROM PUBLIC; -- #6542: a named operator token sets a route's agent level, its four limits and its fallback over MCP through this function, and -- the change is journaled as route_agent_changed under the token's name (operator:), in the same transaction. Levels 0 to 3 are -- set here (MCP is full admin). A level 3 server action still needs that server's switch (config_server_actions) and its remediation login. @@ -1896,8 +1931,8 @@ BEGIN END $fn$; REVOKE ALL ON FUNCTION config.set_notification_route_destinations(integer, text, text[], text, text, text, text, text) FROM PUBLIC; -GRANT EXECUTE ON FUNCTION config.create_notification_route(text, boolean, integer, integer, boolean, integer) TO mcp; -GRANT EXECUTE ON FUNCTION config.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer) TO mcp; +GRANT EXECUTE ON FUNCTION config.create_notification_route(text, boolean, integer, integer, boolean, integer, text) TO mcp; +GRANT EXECUTE ON FUNCTION config.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer, text) TO mcp; GRANT EXECUTE ON FUNCTION config.set_notification_route_destinations(integer, text, text[], text, text, text, text, text) TO mcp; GRANT EXECUTE ON FUNCTION config.set_notification_route_agent(integer, text, text[], integer, integer, integer, integer, integer, integer, boolean, jsonb) TO mcp; -- #6394: the MCP generate_webhook_signing_key and remove_webhook_signing_key tools. mcp holds no UPDATE on a signing key column of config.config_notification @@ -3407,8 +3442,8 @@ BEGIN 'config.request_snapshot_now(integer, text, text)', 'config.request_active_queries_live(integer, text)', 'config.request_analyze_now(integer, text)', - 'config.create_notification_route(text, boolean, integer, integer, boolean, integer)', - 'config.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer)', + 'config.create_notification_route(text, boolean, integer, integer, boolean, integer, text)', + 'config.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer, text)', 'config.set_notification_route_agent(integer, text, text[], integer, integer, integer, integer, integer, integer, boolean, jsonb)', 'config.set_notification_route_destinations(integer, text, text[], text, text, text, text, text)', 'config.request_mcp_command(text, integer, integer, text)', diff --git a/PerformanceMonitor.Notifications/AlertSeverity.cs b/PerformanceMonitor.Notifications/AlertSeverity.cs index d0ca0bda9f..2b2084462e 100644 --- a/PerformanceMonitor.Notifications/AlertSeverity.cs +++ b/PerformanceMonitor.Notifications/AlertSeverity.cs @@ -28,6 +28,11 @@ public enum AlertSeverityLevel /// internal static class AlertSeverity { + /// The tier a notification route's severity filter sees for an alert (#6928): the tier graded at the fire site + /// when it carries one, else the metric's own badge (red is Critical, every other badge is Warning). + public static AlertSeverityLevel RouteLevelOf(string metricName, AlertSeverityLevel? overrideLevel) => + overrideLevel ?? (ForMetric(metricName).HexColor == "#DC2626" ? AlertSeverityLevel.Critical : AlertSeverityLevel.Warning); + /// /// When non-null, forces the tier regardless of — for metrics /// graded at runtime (#1136). null uses the per-metric map below. diff --git a/PerformanceMonitor.Notifications/EmailSendCore.cs b/PerformanceMonitor.Notifications/EmailSendCore.cs index 03d5e92815..04239cb977 100644 --- a/PerformanceMonitor.Notifications/EmailSendCore.cs +++ b/PerformanceMonitor.Notifications/EmailSendCore.cs @@ -209,7 +209,8 @@ reuses DerivePagerDutyDedupKey so the link correlates with the SAME incident Pag and a recipient list without them is not a configuration. #4751: `smtpConfigured` above only asks that SOME enabled route names recipients, so the fall-through to the parent's list can now be blank — a firing no route covers, with no default list, has nobody to send to. */ - emailRoute = NotificationRouter.Resolve(metricName, _settings.NotificationRoutes, _settings, NotificationRouter.ParseServerId(serverId)); + emailRoute = NotificationRouter.Resolve(metricName, _settings.NotificationRoutes, _settings, NotificationRouter.ParseServerId(serverId), + AlertSeverity.RouteLevelOf(metricName, context?.SeverityOverride)); var recipients = emailRoute.Email.Destination ?? _settings.SmtpRecipients; if (string.IsNullOrWhiteSpace(recipients)) diff --git a/PerformanceMonitor.Notifications/InvestigationDispatchCore.cs b/PerformanceMonitor.Notifications/InvestigationDispatchCore.cs index 2b402d5a4a..64140ac520 100644 --- a/PerformanceMonitor.Notifications/InvestigationDispatchCore.cs +++ b/PerformanceMonitor.Notifications/InvestigationDispatchCore.cs @@ -131,9 +131,9 @@ public static InvestigationAlertFacts Facts( /// #5821: whether the generic webhook this alert posts to has {{investigation_json}} in its body template, resolved /// through the same router the send uses. Only then is the request read back: the token is in no shipped template. /// - public bool GenericPostCarriesToken(string serverId, string metricName) => + public bool GenericPostCarriesToken(string serverId, string metricName, AlertSeverityLevel? severity = null) => WebhookAlertService.TemplateCarriesInvestigationToken(_settings.GenericWebhookBodyTemplate) - && NotificationRouter.Resolve(metricName, _settings.NotificationRoutes, _settings, NotificationRouter.ParseServerId(serverId)) + && NotificationRouter.Resolve(metricName, _settings.NotificationRoutes, _settings, NotificationRouter.ParseServerId(serverId), severity) .Generic.Destination is not null; /// @@ -165,7 +165,8 @@ public bool GenericPostCarriesToken(string serverId, string metricName) => } var routeServerId = NotificationRouter.ParseServerId(serverId); - var route = NotificationRouter.ResolveAgent(metricName, _settings.NotificationRoutes, routeServerId); + var route = NotificationRouter.ResolveAgent(metricName, _settings.NotificationRoutes, routeServerId, + AlertSeverity.RouteLevelOf(metricName, context?.SeverityOverride)); if (route is null) { return null; @@ -303,7 +304,7 @@ ruling Q3 (#3653) is "label, not discount". */ try { (InvestigationRequestRow Row, string? Brief)? read = null; - if (readRequest is not null && GenericPostCarriesToken(serverId, metricName)) + if (readRequest is not null && GenericPostCarriesToken(serverId, metricName, AlertSeverity.RouteLevelOf(metricName, context?.SeverityOverride))) { try { diff --git a/PerformanceMonitor.Notifications/NotificationRoute.cs b/PerformanceMonitor.Notifications/NotificationRoute.cs index 3cecbaef73..dbb796035c 100644 --- a/PerformanceMonitor.Notifications/NotificationRoute.cs +++ b/PerformanceMonitor.Notifications/NotificationRoute.cs @@ -166,6 +166,14 @@ public bool IsOffFor(string channel) => /// each channel's own setting, true sends it and false does not, whatever the channel says. public bool? SendResolutions { get; init; } + /// The severity filter (severities, #6928): (the default, every route + /// before the filter), or . + /// A route whose filter excludes an alert's severity does not match it, and matching goes on to the next route. + public string Severities { get; init; } = NotificationRouteKinds.SeveritiesBoth; + + /// Whether this route's severity filter takes an alert of (#6928). Null takes every route. + public bool AcceptsSeverity(AlertSeverityLevel? severity) => NotificationRouteKinds.SeveritiesAccept(Severities, severity); + /// Whether this route sets the agent channel at all (#5632): either level, as opposed to inheriting. public bool HasAgentLevel => AgentLevel.HasValue; diff --git a/PerformanceMonitor.Notifications/NotificationRouteKinds.cs b/PerformanceMonitor.Notifications/NotificationRouteKinds.cs index 91e72e79e4..d65cd772d8 100644 --- a/PerformanceMonitor.Notifications/NotificationRouteKinds.cs +++ b/PerformanceMonitor.Notifications/NotificationRouteKinds.cs @@ -48,4 +48,33 @@ public static string MatchKind(string? metricMatch) => public static string? FamilyOf(string? metricMatch) => IsWildcard(metricMatch) ? null : AlertFamily.NormalizeFamily(metricMatch) ?? AlertFamily.Of(metricMatch ?? string.Empty); + + /// severities value (#6928): a route that takes Warning and Critical alerts alike. The default, and what every route was before the filter existed. + public const string SeveritiesBoth = "both"; + + /// severities value (#6928): a route that takes Critical alerts only. + public const string SeveritiesCritical = "critical"; + + /// severities value (#6928): a route that takes Warning alerts only. Every alert that is not Critical counts as Warning here. + public const string SeveritiesWarning = "warning"; + + /// The stored word for a route's severity filter: both, critical or warning. Blank, null or + /// anything unknown reads as both, so a value this build does not know widens to today's behaviour and never hides a route. + public static string NormalizeSeverities(string? value) => + value?.Trim().ToLowerInvariant() switch + { + SeveritiesCritical => SeveritiesCritical, + SeveritiesWarning => SeveritiesWarning, + _ => SeveritiesBoth, + }; + + /// Whether a route with severity filter takes an alert of . + /// A null severity (a caller that does not know the alert's tier) is taken by every route. + public static bool SeveritiesAccept(string? severities, AlertSeverityLevel? severity) => + severity is null || NormalizeSeverities(severities) switch + { + SeveritiesCritical => severity == AlertSeverityLevel.Critical, + SeveritiesWarning => severity == AlertSeverityLevel.Warning, + _ => true, + }; } diff --git a/PerformanceMonitor.Notifications/NotificationRouter.cs b/PerformanceMonitor.Notifications/NotificationRouter.cs index e8252783b7..54f49741de 100644 --- a/PerformanceMonitor.Notifications/NotificationRouter.cs +++ b/PerformanceMonitor.Notifications/NotificationRouter.cs @@ -199,12 +199,16 @@ public static class NotificationRouter /// (#5750) is the product's server id for the firing - Lite's hash id, Darling's /// registry id - or null for a firing with no server (a fleet-level preview): only fleet-wide routes can match /// then. Optional so a caller with no server keeps compiling. + /// (#6928) is the alert's tier: a route whose severity filter excludes it is dropped before + /// the order below runs, so matching goes on to the next route and ends at the defaults. Null (a caller that does not + /// know the tier) is taken by every route, which is today's behaviour. /// public static NotificationRouteDecision Resolve( string metricName, IReadOnlyList? routes, IAlertSettings defaults, - int? serverId = null) + int? serverId = null, + AlertSeverityLevel? severity = null) { if (metricName is null) { @@ -217,7 +221,7 @@ public static NotificationRouteDecision Resolve( } var family = AlertFamily.Of(metricName); - var ordered = OrderedCandidates(metricName, routes, serverId); + var ordered = OrderedCandidates(metricName, routes, serverId, severity); /* The winning route for the ledger's headline: the most specific one that matched at all, whether or not it ended up supplying every channel. Per-channel provenance sits on each destination. */ @@ -261,14 +265,15 @@ or not it ended up supplying every channel. Per-channel provenance sits on each public static NotificationRoute? ResolveAgent( string metricName, IReadOnlyList? routes, - int? serverId = null) + int? serverId = null, + AlertSeverityLevel? severity = null) { if (metricName is null) { throw new ArgumentNullException(nameof(metricName)); } - foreach (var (route, _, _, _) in OrderedCandidates(metricName, routes, serverId)) + foreach (var (route, _, _, _) in OrderedCandidates(metricName, routes, serverId, severity)) { if (route.AgentLevel is { } level) { @@ -294,14 +299,15 @@ or not it ended up supplying every channel. Per-channel provenance sits on each private static List<(NotificationRoute Route, RouteScopeTier Tier, int Level, RouteSource Source)> OrderedCandidates( string metricName, IReadOnlyList? routes, - int? serverId) + int? serverId, + AlertSeverityLevel? severity) { var candidates = new List<(NotificationRoute Route, RouteScopeTier Tier, int Level, RouteSource Source)>(); if (routes is not null) { foreach (var route in routes) { - if (route is null || !route.CanMatch || !route.AppliesTo(serverId)) + if (route is null || !route.CanMatch || !route.AppliesTo(serverId) || !route.AcceptsSeverity(severity)) { continue; } diff --git a/PerformanceMonitor.Notifications/WebhookAlertService.cs b/PerformanceMonitor.Notifications/WebhookAlertService.cs index 5ed0ecb5d0..aba02322ce 100644 --- a/PerformanceMonitor.Notifications/WebhookAlertService.cs +++ b/PerformanceMonitor.Notifications/WebhookAlertService.cs @@ -240,7 +240,8 @@ the alert log on first touch per key (#1145), unless the store is null (no seedi comes from this one snapshot. With an adapter that does not override SnapshotForDelivery (Lite) it is the live settings. */ var delivery = _settings.SnapshotForDelivery(); - var route = NotificationRouter.Resolve(metricName, delivery.NotificationRoutes, delivery, NotificationRouter.ParseServerId(serverId)); + var route = NotificationRouter.Resolve(metricName, delivery.NotificationRoutes, delivery, NotificationRouter.ParseServerId(serverId), + AlertSeverity.RouteLevelOf(metricName, context?.SeverityOverride)); /* #5469: the PagerDuty auto-resolve behaviour reaches the cooldown only when PagerDuty will actually be sent this alert: the flag on AND a destination for this metric. The flag is read @@ -2905,7 +2906,7 @@ private static bool IsWellFormedJson(string payload, out string? error) /// public async Task TrySendPagerDutyResolveAsync( string firingMetric, string serverName, string serverId, string? detailText, - CancellationToken cancellationToken = default) + AlertSeverityLevel? firedSeverity = null, CancellationToken cancellationToken = default) { try { @@ -2915,7 +2916,9 @@ public async Task TrySendPagerDutyResolveAsync( return AlertChannelOutcome.NotAttempted; } - var route = NotificationRouter.Resolve(firingMetric, delivery.NotificationRoutes, delivery, NotificationRouter.ParseServerId(serverId)); + /* #6928: routed at the severity the fire went out at, so the close lands on the key the fire's route carried, not on a + route the severity filter would send another tier to. */ + var route = NotificationRouter.Resolve(firingMetric, delivery.NotificationRoutes, delivery, NotificationRouter.ParseServerId(serverId), firedSeverity); if (route.PagerDuty.Destination is not { } pagerDutyKey) { return AlertChannelOutcome.NotAttempted; From 679302b2efc0ca14d9a75f110a3816ea8fbd0303 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:25:55 -0400 Subject: [PATCH 03/22] Settable blocking bands so the colours agree with the alerts (#6927) --- .../Darling.Tests/BlockingBandTiersTests.cs | 81 ++++++++++++++++++ Darling/Darling.Tests/RungRewindArtifacts.cs | 8 ++ .../DarlingAlertSettings.cs | 9 ++ .../DarlingConfig.cs | 21 +++++ .../FleetSweepEngine.cs | 25 ++++-- .../Mcp/DarlingAlertReader.cs | 14 +++- .../Mcp/DarlingFleetReader.cs | 47 +++++++++-- .../Mcp/DarlingHealthReader.cs | 8 +- .../Mcp/DarlingMcpAlertTools.cs | 82 +++++++++++++++++- .../StoreConfigProvider.cs | 18 +++- .../wwwroot/js/pages/admin.js | 3 +- .../wwwroot/js/pages/fleet.js | 6 +- .../BlockingBandStore.cs | 46 ++++++++++ .../PgMigrations.cs | 19 +++++ .../StorageVersion.cs | 2 +- .../MainWindow.xaml.cs | 13 ++- .../SettingsWindow.xaml | 20 +++++ .../SettingsWindow.xaml.cs | 60 ++++++++++++++ .../ViewerDataService.AlertSettings.cs | 29 ++++++- .../ViewerDataService.DailySummary.cs | 1 + .../ViewerDataService.Overview.cs | 21 ++++- .../ViewerDataService.cs | 16 +++- PerformanceMonitor.Common/DailyHealthBand.cs | 9 +- .../ServerHealthBands.cs | 83 +++++++++++++++++-- 24 files changed, 598 insertions(+), 43 deletions(-) create mode 100644 Darling/Darling.Tests/BlockingBandTiersTests.cs create mode 100644 Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs diff --git a/Darling/Darling.Tests/BlockingBandTiersTests.cs b/Darling/Darling.Tests/BlockingBandTiersTests.cs new file mode 100644 index 0000000000..7bd72a3314 --- /dev/null +++ b/Darling/Darling.Tests/BlockingBandTiersTests.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace PerformanceMonitor.Darling.Tests; + +/// #6927: the blocking band reads settable tiers whose defaults are the constants every earlier build used. +public sealed class BlockingBandTiersTests +{ + private static readonly TimeSpan Day = TimeSpan.FromHours(24); + + [Fact] + public void Defaults_EqualTheCompileTimeConstants() + { + var d = BlockingBandThresholds.Default; + Assert.Equal(ServerHealthThresholds.BlockingWarnPerHour, d.WarnPerHour); + Assert.Equal(ServerHealthThresholds.BlockingCriticalPerHour, d.CriticalPerHour); + Assert.Equal(ServerHealthThresholds.BlockingWarnWaitSeconds, d.WarnWaitSeconds); + Assert.Equal(ServerHealthThresholds.BlockingCriticalWaitSeconds, d.CriticalWaitSeconds); + Assert.Equal( + ServerHealthClassifier.BlockingSeverity(1434, 162.0, Day), + ServerHealthClassifier.BlockingSeverity(1434, 162.0, Day, d)); + } + + [Fact] + public void ADayTunedTo100EventsAnd180Seconds_IsNotCritical_ButIsUnderTheDefaults() + { + /* 2026-10-23 in the issue: 1,434 reports over a day is 59.75 an hour here (73.5 on the day's own window), peak block 2.7 minutes. */ + const long reports = 1764; // 73.5 an hour over 24 hours + const double peakSeconds = 162.0; // 2.7 minutes + + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(reports, peakSeconds, Day)); + + var tuned = new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(reports, peakSeconds, Day, tuned)); + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(2400, 100.0, Day, tuned)); // 100 an hour + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(1, 180.0, Day, tuned)); + } + + [Fact] + public void TheDailyClassifier_UsesTheTunedTiers() + { + var signals = new DailyHealthSignals + { + Window = Day, + BlockingReports = 1764, + BlockingEvents = 1764, + PeakBlockWaitMs = 162_000, + }; + + Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(signals)); + var tuned = new DailyHealthThresholds { BlockingBands = new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0) }; + Assert.NotEqual(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(signals, tuned)); + } + + [Fact] + public void RawValuesClampToTheFloorsAndCeilings_AndADefaultStructClampsToTheFloorNotZero() + { + var wild = new BlockingBandThresholds(-5, double.NaN, 0, 1e12); + Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, wild.WarnPerHour); + Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, wild.CriticalPerHour); + Assert.Equal(ServerHealthThresholds.BlockingWaitSecondsFloor, wild.WarnWaitSeconds); + Assert.Equal(ServerHealthThresholds.BlockingWaitSecondsCeiling, wild.CriticalWaitSeconds); + Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, default(BlockingBandThresholds).WarnPerHour); + } + + [Fact] + public void TheFleetTotalsTiles_AreBandedByTheClassifier_NotOnAnyNonZeroCount() + { + var now = new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc); + var empty = DarlingFleetReader.BuildRollup(Array.Empty(), now, now.AddHours(-1), now); + Assert.Equal(HealthSeverity.Healthy, empty.TotalDeadlockSeverity); + + var tuned = DarlingFleetReader.BuildRollup( + new List(), now, now.AddHours(-1), now, + deadlockTiers: new DeadlockRateThresholds(50.0, 100.0), blockingTiers: new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0)); + Assert.Equal(HealthSeverity.Healthy, tuned.TotalDeadlockSeverity); + } +} diff --git a/Darling/Darling.Tests/RungRewindArtifacts.cs b/Darling/Darling.Tests/RungRewindArtifacts.cs index 375b56efdd..eadcce8d4c 100644 --- a/Darling/Darling.Tests/RungRewindArtifacts.cs +++ b/Darling/Darling.Tests/RungRewindArtifacts.cs @@ -577,6 +577,14 @@ ALTER TABLE config.config_alert_settings // so a re-run of V226 does everything again. Safe on a store that never had it. new(226, "hasServerAwsRdsWebOrExpress", @"ALTER TABLE collect.servers DROP COLUMN IF EXISTS is_aws_rds_web_or_express;"), + // V228 (#6927): the four blocking band columns on the alert settings row. The warn per hour column is the probe's newest sentinel. The + // rewind drops all four, so a re-run of V228 adds them again. Safe on a store that never had them. + new(228, "hasBlockingBands", + @"ALTER TABLE config.config_alert_settings + DROP COLUMN IF EXISTS blocking_warn_per_hour, + DROP COLUMN IF EXISTS blocking_critical_per_hour, + DROP COLUMN IF EXISTS blocking_warn_wait_seconds, + DROP COLUMN IF EXISTS blocking_critical_wait_seconds;"), ]; /// diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs index add0198c81..048c648cc8 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs @@ -296,6 +296,15 @@ public IReadOnlyList WebhookChannelProblems _config.Alerts.DeadlockWarnPerHour, _config.Alerts.DeadlockCriticalPerHour); + /// #6927 (V228): the blocking health band's four tiers as ONE value, for the reason + /// is one. The clamp lives on + /// , so the viewer's card and the service's read clamp alike. + public BlockingBandThresholds BlockingBandThresholds => new( + _config.Alerts.BlockingWarnPerHour, + _config.Alerts.BlockingCriticalPerHour, + _config.Alerts.BlockingWarnWaitSeconds, + _config.Alerts.BlockingCriticalWaitSeconds); + public int CollectionFailureThreshold => Math.Clamp(_config.Alerts.CollectionFailureThreshold, 1, 1000); /* #1984: percent clamped like low-disk's (0 = off); the GB floor merely floored at 0 — unlike diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs index 02690d7096..1d765cde4f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs @@ -1105,6 +1105,27 @@ public sealed class AlertsConfig [JsonPropertyName("deadlockCriticalPerHour")] public double DeadlockCriticalPerHour { get; set; } = ServerHealthThresholds.DeadlockCriticalPerHourDefault; + /// #6927 (V228): the blocking health band's WARNING tier, blocked-process reports per hour over the window a card + /// counted them in. Seeded from , named rather than restated, so a store + /// that never sets it bands as every earlier build did. BlockingBandThresholds clamps it on read. + [JsonPropertyName("blockingWarnPerHour")] + public double BlockingWarnPerHour { get; set; } = ServerHealthThresholds.BlockingWarnPerHour; + + /// #6927 (V228): the blocking health band's CRITICAL tier, reports per hour, seeded from + /// . + [JsonPropertyName("blockingCriticalPerHour")] + public double BlockingCriticalPerHour { get; set; } = ServerHealthThresholds.BlockingCriticalPerHour; + + /// #6927 (V228): the blocking band's WARNING bar on the longest single block, seconds, seeded from + /// . + [JsonPropertyName("blockingWarnWaitSeconds")] + public double BlockingWarnWaitSeconds { get; set; } = ServerHealthThresholds.BlockingWarnWaitSeconds; + + /// #6927 (V228): the blocking band's CRITICAL bar on the longest single block, seconds, seeded from + /// . + [JsonPropertyName("blockingCriticalWaitSeconds")] + public double BlockingCriticalWaitSeconds { get; set; } = ServerHealthThresholds.BlockingCriticalWaitSeconds; + [JsonPropertyName("longRunningJobEnabled")] public bool LongRunningJobEnabled { get; set; } = true; diff --git a/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs b/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs index 54d5b52a91..3e219964e0 100644 --- a/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs +++ b/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs @@ -197,7 +197,8 @@ public static FleetSweepComposition Compose( IReadOnlyList previousVerdicts, IReadOnlyList activeWatchItems, FleetSweepInstrumentCounters instruments, - DeadlockRateThresholds deadlockRateTiers) + DeadlockRateThresholds deadlockRateTiers, + BlockingBandThresholds? blockingBandTiers = null) { ArgumentNullException.ThrowIfNull(readings); ArgumentNullException.ThrowIfNull(previousVerdicts); @@ -209,7 +210,12 @@ public static FleetSweepComposition Compose( a caller that kept the old call would compile and silently band on the shipped pair while the Overview card used the store's. Built once, because the banding thresholds must be one configuration for the whole sweep. */ - var banding = new DailyHealthThresholds { DeadlockRates = deadlockRateTiers }; + /* #6927: the blocking band's tiers travel the same way; null is today's constants. */ + var banding = new DailyHealthThresholds + { + DeadlockRates = deadlockRateTiers, + BlockingBands = blockingBandTiers ?? BlockingBandThresholds.Default, + }; var sweepId = nowUtc.Ticks; var inSettleWindow = nowUtc - instruments.ServiceStartedUtc < PostRestartSettleWindow; @@ -785,26 +791,26 @@ The production read fills both from one MAX. */ unit its rate tiers were measured in; BlockingEvents counts WAITS and is what the evidence shows as the count. Classify reads the same pair, so the ledger still cannot page on a count the verdict did not band. */ var bandedBlocking = signals.BlockingReports ?? signals.BlockingEvents; - if (ServerHealthClassifier.BlockingSeverity(bandedBlocking, peakBlockSeconds, signals.Window) + if (ServerHealthClassifier.BlockingSeverity(bandedBlocking, peakBlockSeconds, signals.Window, thresholds.BlockingBands) == HealthSeverity.Critical) { - yield return (FamilyBlocking, BlockingEvidence(signals.BlockingEvents, bandedBlocking, signals.BlockingReports.HasValue, signals.Window, peakBlockSeconds)); + yield return (FamilyBlocking, BlockingEvidence(signals.BlockingEvents, bandedBlocking, signals.BlockingReports.HasValue, signals.Window, peakBlockSeconds, thresholds.BlockingBands)); } } /// The blocking family's evidence (#3539 A3): the arm that banded is the one named. A 60 s /// block is the wait arm's Critical whatever the rate; otherwise the RATE is the value, with the raw /// count as its own member for the operator reconciling against the blocking grid. - private static string BlockingEvidence(long count, long bandedCount, bool bandedOnReports, TimeSpan window, double peakBlockSeconds) + private static string BlockingEvidence(long count, long bandedCount, bool bandedOnReports, TimeSpan window, double peakBlockSeconds, BlockingBandThresholds tiers) { - if (peakBlockSeconds >= ServerHealthThresholds.BlockingCriticalWaitSeconds) + if (peakBlockSeconds >= tiers.CriticalWaitSeconds) { return JsonSerializer.Serialize(new { derivation = "summary-scoring critical trigger under alerts_enabled: false — not an alert-engine replay", trigger = "longest single block in the sweep span, seconds", value = peakBlockSeconds, - threshold = ServerHealthThresholds.BlockingCriticalWaitSeconds, + threshold = tiers.CriticalWaitSeconds, blocking_count = count, }); } @@ -819,7 +825,7 @@ rate is present by construction. */ ? "blocked-process reports per hour over the sweep span" : "blocking events per hour over the sweep span", value = ratePerHour, - threshold = ServerHealthThresholds.BlockingCriticalPerHour, + threshold = tiers.CriticalPerHour, blocking_count = count, }); } @@ -924,6 +930,7 @@ public static async Task RunAsync( — an engine-seam read, so a fault here loudly costs this sweep slot rather than quietly banding the fleet on the shipped pair. */ var deadlockRateTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken).ConfigureAwait(false); + var blockingBandTiers = await BlockingBandStore.ReadAsync(postgres, McpCommandDeadlines.ReadSeconds, cancellationToken).ConfigureAwait(false); var nowUtc = DateTime.UtcNow; var spanStartUtc = ComputeSpanStart(nowUtc, interval, previousRun); @@ -942,7 +949,7 @@ at a time and a herd all at once — and the sweep is a background errand racing var composition = Compose( nowUtc, spanStartUtc, alertsEnabled, servers.Count, readings, - previousRun, previousVerdicts, activeItems, ReadInstrumentCounters(), deadlockRateTiers); + previousRun, previousVerdicts, activeItems, ReadInstrumentCounters(), deadlockRateTiers, blockingBandTiers); await FleetSweepStore.RecordSweepAsync( postgres, composition.Run, composition.Verdicts, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs index 846a2a2dcc..b33da5ef24 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs @@ -542,7 +542,12 @@ homes decided (DarlingAlertSettings.ResolveUncorroboratedRoute takes this beside /* #6875 (V224): the Identity Capacity switch (boolean NOT NULL, default off) and its warning and critical bars (integer NOT NULL, 70 and 90). APPENDED, same reason. */ bool IdentityCapacityEnabled, int IdentityCapacityWarningPercent, - int IdentityCapacityCriticalPercent); + int IdentityCapacityCriticalPercent, + /* #6927 (V228): the four blocking band tiers (double precision NOT NULL, 5, 20 reports per hour and 10, 60 seconds). */ + double BlockingWarnPerHour, + double BlockingCriticalPerHour, + double BlockingWarnWaitSeconds, + double BlockingCriticalWaitSeconds); /// The single global alert-settings row (id=1) — the viewer's AlertSettingsSelectSql. The /// columns are read in the SAME order the service reads them (StoreConfigProvider), and @@ -582,7 +587,8 @@ homes decided (DarlingAlertSettings.ResolveUncorroboratedRoute takes this beside excluded_volumes, client_timeouts_warning_multiple, client_timeouts_critical_multiple, agent_alert_gaps_enabled, backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, - identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent + identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, + blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1"; @@ -653,7 +659,9 @@ FROM config_alert_settings /* #6874 (V222): the Agent Alert Gaps switch at 79. #6873 (V223): the three database protection switches at 80-82. */ reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), /* #6875 (V224): the Identity Capacity switch and bars at 83-85. */ - reader.GetBoolean(83), reader.GetInt32(84), reader.GetInt32(85)); + reader.GetBoolean(83), reader.GetInt32(84), reader.GetInt32(85), + /* #6927 (V228): the four blocking band tiers at 86-89. */ + reader.GetDouble(86), reader.GetDouble(87), reader.GetDouble(88), reader.GetDouble(89)); } /* ─────────────────────── delivery cooldown (a SECOND config table) ─────────────────────── */ diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index ea394f5ddb..5fcb31de44 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -930,6 +930,8 @@ the production photo this answers. */ servers on the old pair and the rest on the new one — a mixed reading no configuration ever held, and the band counts and the worst-first ranking would be derived from it. */ var deadlockTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken); + /* #6927 (V228): the blocking band's tiers, read once per roll-up for the same reason. */ + var blockingTiers = await BlockingBandStore.ReadAsync(postgres, McpCommandDeadlines.ReadSeconds, cancellationToken); var identityCapacity = await ReadIdentityCapacityAsync(postgres, now, cancellationToken); var cards = new List(servers.Count); @@ -967,11 +969,11 @@ server the read reported with an empty window reads Offline or never-collected b cards.Add(BuildCard( server, c, pg, m, mp, t, b, deadlock, pgDeadlock, collected.LastCollection, collectors, serverTags, now, - windowEndUtc - windowStartUtc, deadlockTiers, collected.RegisteredAt, collected.OlderCollection, serverAgentAlertGaps, serverDatabaseProtection, serverIdentityCapacity)); + windowEndUtc - windowStartUtc, deadlockTiers, collected.RegisteredAt, collected.OlderCollection, serverAgentAlertGaps, serverDatabaseProtection, serverIdentityCapacity, blockingTiers)); } return BuildRollup(cards, now, windowStartUtc, windowEndUtc, worstCount, tagForest, collectionHealthAgeSeconds, - await ReadCollectionStateAsync(postgres, logger, cancellationToken)); + await ReadCollectionStateAsync(postgres, logger, cancellationToken), deadlockTiers, blockingTiers); } /// Builds one pre-banded card from a server's raw cross-server reads (pure — the reduction the WPF @@ -1093,7 +1095,8 @@ internal static FleetServerCard BuildCard( DateTime? olderCollection = null, IReadOnlyList? agentAlertGaps = null, IReadOnlyList? databaseProtectionOverdue = null, - FleetIdentityCapacity? identityCapacity = null) + FleetIdentityCapacity? identityCapacity = null, + BlockingBandThresholds? blockingTiers = null) { /* Lite's XE-preferred / DMV-fallback, per server: XE when it has any row this window, else the DMV @@ -1188,6 +1191,8 @@ because cpuForAlert there is percent of an allocation that moves. The source is arm, one metric over. */ DeadlockWindow = deadlockWindow, DeadlockRateThresholds = deadlockTiers, + /* #6927: the blocking tiers travel with the count and the window, for the reason the deadlock ones do. */ + BlockingBandThresholds = blockingTiers, TotalThreads = threads.TotalThreads, AvailableThreads = availableThreads, ThreadsWaitingForCpu = threads.RunnableTasks, @@ -1268,7 +1273,7 @@ deliberately not surfaced. */ ? ServerHealthClassifier.BlockingRatePerHour(blockingForBand.Value, deadlockWindow) : null, BlockingWindow = deadlockWindow, - BlockingSeverity = ServerHealthClassifier.BlockingSeverity(blockingForBand, maxBlockedSeconds, deadlockWindow), + BlockingSeverity = ServerHealthClassifier.BlockingSeverity(blockingForBand, maxBlockedSeconds, deadlockWindow, blockingTiers), DeadlockCount = deadlockCount, DeadlockLastSeen = deadlockLastSeen, DeadlockMeasured = deadlockMeasured, @@ -1281,6 +1286,7 @@ over. Null exactly when the PostgreSQL arm took no difference (#3539). */ : null, DeadlockWindow = deadlockWindow, DeadlockRateThresholds = deadlockTiers, + BlockingBandThresholds = blockingTiers, /* #6875: set only when a column is at the warning bar or past it. The Band below is not read from it. */ IdentityCapacity = identityCapacity, DeadlockSeverity = ServerHealthClassifier.DeadlockSeverity( @@ -1323,7 +1329,9 @@ public static FleetOverviewResult BuildRollup( int worstCount = DefaultWorstCount, IReadOnlyList? tags = null, int collectionHealthAgeSeconds = 0, - FleetCollectionState? collection = null) + FleetCollectionState? collection = null, + DeadlockRateThresholds? deadlockTiers = null, + BlockingBandThresholds? blockingTiers = null) { var healthy = 0; var warning = 0; @@ -1332,6 +1340,8 @@ public static FleetOverviewResult BuildRollup( var failures = 0; long totalBlocking = 0; long totalDeadlocks = 0; + long? totalBlockingReports = null; + var maxBlockedSeconds = 0.0; var deadlockSourcesRead = 0; var deadlockPostgresTargets = 0; var deadlockCollectorsSilent = 0; @@ -1375,6 +1385,14 @@ attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on totalBlocking += card.BlockingCount; totalDeadlocks += card.DeadlockCount; + /* #6927: the blocking tile bands on the reports the cards band on, and the longest block anywhere. A PostgreSQL card + has no blocking source, so it adds nothing and does not make the tile measured. */ + var cardMetrics = card.ToHealthMetrics(); + if (cardMetrics.BlockingCount.HasValue) + { + totalBlockingReports = (totalBlockingReports ?? 0) + cardMetrics.BlockingCount.Value; + maxBlockedSeconds = Math.Max(maxBlockedSeconds, cardMetrics.MaxBlockedSeconds); + } } var problems = cards @@ -1410,6 +1428,12 @@ attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on ServersWithCollectionFailures = failures, TotalBlockingEvents = totalBlocking, TotalDeadlocks = totalDeadlocks, + /* #6927: the two totals tiles band through the same classifier and the store's tiers as the cards, over the window the totals + were read in, instead of the page colouring them on any non-zero count. */ + TotalBlockingSeverity = ServerHealthClassifier.BlockingSeverity( + totalBlockingReports, maxBlockedSeconds, windowEndUtc - windowStartUtc, blockingTiers), + TotalDeadlockSeverity = ServerHealthClassifier.DeadlockSeverity( + totalDeadlocks, windowEndUtc - windowStartUtc, deadlockTiers ?? DeadlockRateThresholds.Default), DeadlockCoverage = new FleetDeadlockCoverage { ServersRead = deadlockSourcesRead, @@ -2950,6 +2974,11 @@ public sealed class FleetServerCard [JsonIgnore] public DeadlockRateThresholds DeadlockRateThresholds { get; init; } + /// The blocking tiers this card banded on (#6927), or null for today's constants. Carried, not serialized, for + /// , so the worst-first score re-bands on the card's own tiers. + [JsonIgnore] + public BlockingBandThresholds? BlockingBandThresholds { get; init; } + /// This server's deadlock-source collector band over the trailing seven days of collection /// health (#3017) — the fact that explains a of zero. The collector is /// deadlocks on SQL Server and pg_database_stats on PostgreSQL (#3539). Null when that @@ -3080,6 +3109,7 @@ was taken - see DeadlockMeasured. */ dot used the store's. */ DeadlockWindow = DeadlockWindow, DeadlockRateThresholds = DeadlockRateThresholds, + BlockingBandThresholds = BlockingBandThresholds, TotalThreads = TotalThreads, AvailableThreads = AvailableThreads, ThreadsWaitingForCpu = ThreadsWaitingForCpu, @@ -3145,6 +3175,13 @@ public sealed class FleetOverviewResult [JsonPropertyName("total_blocking_events")] public long TotalBlockingEvents { get; init; } [JsonPropertyName("total_deadlocks")] public long TotalDeadlocks { get; init; } + /// The Blocking totals tile's band (#6927): the fleet's blocked-process reports per hour and longest block against the + /// store's blocking tiers, through the same classifier the cards use. Unknown when no card has a blocking source. + [JsonPropertyName("total_blocking_severity")] public HealthSeverity TotalBlockingSeverity { get; init; } + + /// The Deadlocks totals tile's band (#6927): the fleet's deadlocks per hour against the store's deadlock tiers. + [JsonPropertyName("total_deadlock_severity")] public HealthSeverity TotalDeadlockSeverity { get; init; } + /// How much of the fleet actually read a deadlock source for, with /// the causes named (#3017). Never null — a total with no denominator beside it is the defect. [JsonPropertyName("deadlock_coverage")] public FleetDeadlockCoverage DeadlockCoverage { get; init; } = new(); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs index ef80e8d186..8026178ac3 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs @@ -275,6 +275,10 @@ public sealed record DailySummaryReadRow( /// defaults holds anyway. public DeadlockRateThresholds RateTiers { get; init; } = DeadlockRateThresholds.Default; + /// The blocking band's tiers the day is banded on (#6927), the store's values for the same reason + /// is. + public BlockingBandThresholds BlockingTiers { get; init; } = BlockingBandThresholds.Default; + /// The clock the still-forming day's window clamps against (#3525 review): anchored /// reads hand their resolved window end so a backdated as_of clamps against its own "now"; /// unanchored reads (the explicit-date tool, the viewer path) band against the wall clock. @@ -337,7 +341,7 @@ have not happened yet (review finding on #3525). The fleet sweep does NOT read t }; public DailyHealthBand HealthBand => - DailyHealthBandCalculator.Classify(ToSignals(), new DailyHealthThresholds { DeadlockRates = RateTiers }); + DailyHealthBandCalculator.Classify(ToSignals(), new DailyHealthThresholds { DeadlockRates = RateTiers, BlockingBands = BlockingTiers }); /// Human label for the band ("Healthy" / "Warning" / "Critical" / "No Data"). public string OverallHealth => DailyHealthBandCalculator.Label(HealthBand); @@ -496,6 +500,7 @@ a floor moves back only on a backfill (stale, it under-claims, so a window stays DarlingFleetReader's own hoist argument: a settings write mid-read must not band some days on the old pair and the rest on the new one. */ var rateTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken); + var blockingTiers = await BlockingBandStore.ReadAsync(postgres, McpCommandDeadlines.ReadSeconds, cancellationToken); /* #3541 A9: the retention horizon, from the store's effective retention and the READER's wall clock. The clock is deliberately NOT the caller's anchor — a purge is a wall-clock event and a backdated @@ -592,6 +597,7 @@ twice McpCommandDeadlines.ReadSeconds. */ var results = rawResults.Select(row => row with { RateTiers = rateTiers, + BlockingTiers = blockingTiers, ReferenceUtc = referenceUtc ?? DateTime.UtcNow, DataState = DailySummaryRetention.StateFor(row.SummaryDate, row.CollectionRuns, row.SignalSourcesPresent, horizon), RetentionHorizon = horizon, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs index b2686932c9..ff5a1824ea 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs @@ -477,7 +477,12 @@ invite tuning one and expecting the other to move. */ health_bands = new { deadlock_warn_per_hour = s.DeadlockWarnPerHour, - deadlock_critical_per_hour = s.DeadlockCriticalPerHour + deadlock_critical_per_hour = s.DeadlockCriticalPerHour, + /* #6927 (V228): the blocking band, beside the deadlock one: reports per hour and the longest single block in seconds. */ + blocking_warn_per_hour = s.BlockingWarnPerHour, + blocking_critical_per_hour = s.BlockingCriticalPerHour, + blocking_warn_wait_seconds = s.BlockingWarnWaitSeconds, + blocking_critical_wait_seconds = s.BlockingCriticalWaitSeconds }, poison_wait = new { @@ -1205,6 +1210,11 @@ internal static JsonObject BuildMuteRuleWithState(MuteRule r, DateTime nowUtc) "per hour is the tightest setting that is still a rate, so no value here can restore the 'any deadlock " + "is Critical' reading these tiers replaced. Setting critical BELOW warn is accepted and means every " + "banded rate is Critical. " + + "health_bands.blocking_warn_per_hour and health_bands.blocking_critical_per_hour are the blocking band's tiers in blocked-process reports per HOUR " + + "(accepting 1 to 10000), and health_bands.blocking_warn_wait_seconds and health_bands.blocking_critical_wait_seconds are its bars on the longest " + + "single block in seconds (accepting 1 to 86400). They default to 5 and 20 reports per hour and 10 and 60 seconds, and they decide the colour " + + "of the fleet cards, the Overview tab, the Daily Health Calendar and get_fleet_overview, so set them to match the blocking alert's own " + + "threshold. Unlike the deadlock tiers, each blocking pair must keep critical at or above warn, and a write that would leave it below is refused. " + "file_growth.rise_mb is megabytes per HOUR averaged over file_growth.lookback_minutes (a rate — the same 10240 is 10 GB/hr on any lookback; the engine scales it to the window), so shortening the lookback does not tighten the rise gate and lengthening it does not loosen it; only the rate does. " + "Two keys govern the PostgreSQL versions of the two count alerts and are NOT the same numbers as their SQL Server neighbours: deadlocks.pg_count_threshold and blocking.pg_count_threshold, both accepting 1 upward. They sit inside those groups rather than a section of their own so both engines' figures are visible together, but tuning deadlocks.count_threshold does NOT move the PostgreSQL gate and tuning deadlocks.pg_count_threshold does NOT move the SQL Server one. The enabled switch in each group DOES govern both engines. They are separate because the two engines count with different instruments - SQL Server's figure is captured deadlock graphs, the PostgreSQL one is deadlocks parsed from the server log (get_pg_deadlocks) - and an operator tuning one should not silently move the other. Both engines' fleet cards now band deadlocks through the SAME health_bands.deadlock_warn_per_hour tiers (the PostgreSQL card differences the server's own pg_stat_database.deadlocks counter over the window), so the move - raising a fire gate to meet the band's Warning bar so a page and an amber dot describe the same server - is available on either knob. On the blocking side the SQL Server count is engine-recorded blocked-process reports and the PostgreSQL one is distinct root blockers in a periodic SAMPLE of pg_stat_activity. Both PostgreSQL keys are ignored on a store with no PostgreSQL targets. " + "The fleet_sweep group is NOT an alert family and the alerts_enabled master switch does not govern it: " + @@ -1384,6 +1394,29 @@ Widening that allowlist to admit a transaction would have let a real target comm } } + /* #6927: the same lone-bar rule for the two blocking band pairs. */ + if (updates.Any(u => u.Column is BlockingWarnPerHourColumn or BlockingCriticalPerHourColumn or BlockingWarnWaitColumn or BlockingCriticalWaitColumn)) + { + const string blockingBandSql = "SELECT blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1"; + await using var blockingBandCommand = new NpgsqlCommand(blockingBandSql, connection) { Transaction = transaction }; + blockingBandCommand.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var blockingBandReader = await blockingBandCommand.ExecuteReaderAsync(); + if (await blockingBandReader.ReadAsync()) + { + if (blockingBandReader.GetDouble(1) < blockingBandReader.GetDouble(0)) + { + return Outcome("invalid", BlockingRateBarsOrderMessage + + $" (stored after this write: warn {blockingBandReader.GetDouble(0).ToString("0.###", CultureInfo.InvariantCulture)}, critical {blockingBandReader.GetDouble(1).ToString("0.###", CultureInfo.InvariantCulture)}; nothing was written.)"); + } + + if (blockingBandReader.GetDouble(3) < blockingBandReader.GetDouble(2)) + { + return Outcome("invalid", BlockingWaitBarsOrderMessage + + $" (stored after this write: warn {blockingBandReader.GetDouble(2).ToString("0.###", CultureInfo.InvariantCulture)}, critical {blockingBandReader.GetDouble(3).ToString("0.###", CultureInfo.InvariantCulture)}; nothing was written.)"); + } + } + } + /* #5640: the same lone-bar rule for the Client Timeouts pair. */ if (updates.Any(u => u.Column == ClientTimeoutsWarningColumn || u.Column == ClientTimeoutsCriticalColumn)) { @@ -2586,6 +2619,14 @@ private static string AlreadyExists(MuteRule existing) => "'client_timeouts.critical_multiple' must not be lower than 'client_timeouts.warning_multiple' (the Critical bar can never sit below the Warning bar)."; private const string ClientTimeoutsBarsOrderMessage = "'client_timeouts.critical_per_minute' must not be lower than 'client_timeouts.warning_per_minute' (the Critical bar can never sit below the Warning bar)."; + private const string BlockingWarnPerHourColumn = "blocking_warn_per_hour"; + private const string BlockingCriticalPerHourColumn = "blocking_critical_per_hour"; + private const string BlockingWarnWaitColumn = "blocking_warn_wait_seconds"; + private const string BlockingCriticalWaitColumn = "blocking_critical_wait_seconds"; + private const string BlockingRateBarsOrderMessage = + "'health_bands.blocking_critical_per_hour' must not be lower than 'health_bands.blocking_warn_per_hour' (the Critical bar can never sit below the Warning bar)."; + private const string BlockingWaitBarsOrderMessage = + "'health_bands.blocking_critical_wait_seconds' must not be lower than 'health_bands.blocking_warn_wait_seconds' (the Critical bar can never sit below the Warning bar)."; private const string RunnableQueueBarsOrderMessage = "'runnable_queue.critical_per_scheduler' must not be lower than 'runnable_queue.warning_per_scheduler' (the Critical bar can never sit below the Warning bar)."; @@ -2963,6 +3004,27 @@ threshold that cannot reach a bare-count band. */ ServerHealthThresholds.DeadlockRatePerHourFloor, ServerHealthThresholds.DeadlockRatePerHourCeiling); break; + /* #6927 (V228): the blocking band, same named bounds BlockingBandThresholds clamps to. */ + case "blocking_warn_per_hour": + AddDouble("blocking_warn_per_hour", n, "health_bands.blocking_warn_per_hour", + ServerHealthThresholds.BlockingRatePerHourFloor, + ServerHealthThresholds.BlockingRatePerHourCeiling); + break; + case "blocking_critical_per_hour": + AddDouble("blocking_critical_per_hour", n, "health_bands.blocking_critical_per_hour", + ServerHealthThresholds.BlockingRatePerHourFloor, + ServerHealthThresholds.BlockingRatePerHourCeiling); + break; + case "blocking_warn_wait_seconds": + AddDouble("blocking_warn_wait_seconds", n, "health_bands.blocking_warn_wait_seconds", + ServerHealthThresholds.BlockingWaitSecondsFloor, + ServerHealthThresholds.BlockingWaitSecondsCeiling); + break; + case "blocking_critical_wait_seconds": + AddDouble("blocking_critical_wait_seconds", n, "health_bands.blocking_critical_wait_seconds", + ServerHealthThresholds.BlockingWaitSecondsFloor, + ServerHealthThresholds.BlockingWaitSecondsCeiling); + break; default: error = $"Unknown field 'health_bands.{k}'."; break; } }); @@ -3375,6 +3437,24 @@ is a Postgres error (multiple assignments to the same column) -- so refusing it error = RunnableQueueBarsOrderMessage; } + /* #6927: both bars of a blocking band pair in one body must already be in order; a lone bar is compared with the stored other one + inside UpdateAlertSettings' transaction (the builder has no store). */ + if (error == null + && updates.FirstOrDefault(u => u.Column == BlockingWarnPerHourColumn) is { Param: NpgsqlParameter blockWarnRate } + && updates.FirstOrDefault(u => u.Column == BlockingCriticalPerHourColumn) is { Param: NpgsqlParameter blockCriticalRate } + && blockCriticalRate.TypedValue < blockWarnRate.TypedValue) + { + error = BlockingRateBarsOrderMessage; + } + + if (error == null + && updates.FirstOrDefault(u => u.Column == BlockingWarnWaitColumn) is { Param: NpgsqlParameter blockWarnWait } + && updates.FirstOrDefault(u => u.Column == BlockingCriticalWaitColumn) is { Param: NpgsqlParameter blockCriticalWait } + && blockCriticalWait.TypedValue < blockWarnWait.TypedValue) + { + error = BlockingWaitBarsOrderMessage; + } + /* #5640: both Client Timeouts bars in one body must already be in order; a lone bar is compared with the stored other one. */ if (error == null && updates.FirstOrDefault(u => u.Column == ClientTimeoutsWarningColumn) is { Param: NpgsqlParameter timeoutWarningParam } diff --git a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs index 965aa93a3a..0c6883ad1f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs +++ b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs @@ -1431,11 +1431,12 @@ INSERT INTO config_alert_settings ( excluded_volumes, client_timeouts_warning_multiple, client_timeouts_critical_multiple, agent_alert_gaps_enabled, backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, - identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent) + identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, + blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds) VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46, $47, $48, $49, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, - $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86) + $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86, $87, $88, $89, $90) ON CONFLICT (id) DO NOTHING", connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; command.Parameters.AddWithValue(a.Enabled); command.Parameters.AddWithValue(a.CpuEnabled); @@ -1565,6 +1566,11 @@ apply and get_alert_settings reports back. */ command.Parameters.AddWithValue(a.IdentityCapacityEnabled); command.Parameters.AddWithValue(a.IdentityCapacityWarningPercent); command.Parameters.AddWithValue(a.IdentityCapacityCriticalPercent); + /* #6927, bound last in the order the V228 columns were appended. Seeded RAW like every sibling: the clamp lives on BlockingBandThresholds. */ + command.Parameters.AddWithValue(a.BlockingWarnPerHour); + command.Parameters.AddWithValue(a.BlockingCriticalPerHour); + command.Parameters.AddWithValue(a.BlockingWarnWaitSeconds); + command.Parameters.AddWithValue(a.BlockingCriticalWaitSeconds); await command.ExecuteNonQueryAsync(ct); } @@ -2204,7 +2210,8 @@ internal static string NormalizePlanXmlCompression(string? value) => excluded_volumes, client_timeouts_warning_multiple, client_timeouts_critical_multiple, agent_alert_gaps_enabled, backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, - identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent + identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, + blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1", connection) { CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds }; using var reader = await command.ExecuteReaderAsync(ct); if (!await reader.ReadAsync(ct)) @@ -2375,6 +2382,11 @@ selected but not read here -- or read but not selected -- would reset the list o IdentityCapacityEnabled = reader.GetBoolean(83), IdentityCapacityWarningPercent = reader.GetInt32(84), IdentityCapacityCriticalPercent = reader.GetInt32(85), + /* #6927 (V228): the four blocking band tiers at 86-89 (same reachability rule as every appended knob). */ + BlockingWarnPerHour = reader.GetDouble(86), + BlockingCriticalPerHour = reader.GetDouble(87), + BlockingWarnWaitSeconds = reader.GetDouble(88), + BlockingCriticalWaitSeconds = reader.GetDouble(89), }; var analysis = new AnalysisConfig { diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js index 0abc05db92..e2fe27cce3 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js @@ -196,7 +196,8 @@ export const GROUP_FIELDS = { cpu: ["enabled", "threshold_percent", "mode"], blocking: ["enabled", "count_threshold", "wait_threshold_seconds", "pg_count_threshold"], deadlocks: ["enabled", "count_threshold", "pg_count_threshold"], - health_bands: ["deadlock_warn_per_hour", "deadlock_critical_per_hour"], + health_bands: ["deadlock_warn_per_hour", "deadlock_critical_per_hour", "blocking_warn_per_hour", "blocking_critical_per_hour", + "blocking_warn_wait_seconds", "blocking_critical_wait_seconds"], poison_wait: ["enabled", "threshold_ms"], runnable_queue: ["enabled", "warning_per_scheduler", "critical_per_scheduler"], client_timeouts: ["enabled", "warning_per_minute", "critical_per_minute", "warning_multiple", "critical_multiple"], diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js index 1035e40e02..63fd351bcc 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js @@ -681,7 +681,9 @@ export function rollup(d) { not a confirmed incident either — warning is the "interrogate this number" class without claiming a deadlock happened. A real deadlock keeps critical whatever the coverage: a measured incident outranks an incomplete denominator, and the sub-line under it already says the count may be short. */ - const deadlockClass = d.total_deadlocks > 0 ? "critical" : deadlockSub && deadlockSub.partial ? "warning" : ""; + /* #6927: banded by the service through the same classifier and the store's tiers as the cards, not on any non-zero count. */ + const totalClass = (sev) => (sev === "Critical" ? "critical" : sev === "Warning" ? "warning" : ""); + const deadlockClass = totalClass(d.total_deadlock_severity) || (deadlockSub && deadlockSub.partial ? "warning" : ""); /* Two fixed groups (server-band counts | event counts) split by a divider; a non-zero blocking / deadlock total takes a severity color. */ @@ -695,7 +697,7 @@ export function rollup(d) { ]), el("div", { class: "rollup-divider" }), el("div", { class: "rollup-group" }, [ - tile(d.total_blocking_events, "Blocking (recent)", d.total_blocking_events > 0 ? "warning" : ""), + tile(d.total_blocking_events, "Blocking (recent)", totalClass(d.total_blocking_severity)), /* The API's own note rides along as the tile's title — the causes and the two windows, in the words the service already renders for get_fleet_overview, so the hover is DETAIL on a signal that is already visible rather than the only place the signal exists. */ diff --git a/Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs b/Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs new file mode 100644 index 0000000000..c854472ffc --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Storage; + +/// +/// The one read of the blocking health band's four tiers (#6927, V228) from the store's singleton settings row. It lives in Storage because +/// every Darling reader of the band needs the same values: the fleet cards and get_fleet_overview, the fleet sweep, the Daily Health +/// Calendar and get_daily_health in the service, and the Overview tab, the calendar and the fleet grid in the viewer. Values come back +/// RAW and clamps on read. A store with no row yet bands on today's constants, which is what such a +/// store would seed. +/// +public static class BlockingBandStore +{ + /// The singleton row's four tiers. No parameters: this is the current configuration, not history. + public const string ReadSql = @" +SELECT blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds +FROM config_alert_settings +WHERE id = 1"; + + /// Reads the tiers, or when the row is absent. + public static async Task ReadAsync( + NpgsqlDataSource postgres, int commandTimeoutSeconds, CancellationToken cancellationToken) + { + await using var command = postgres.CreateCommand(ReadSql); + command.CommandTimeout = commandTimeoutSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return new BlockingBandThresholds( + reader.GetDouble(0), reader.GetDouble(1), reader.GetDouble(2), reader.GetDouble(3)); + } + + return BlockingBandThresholds.Default; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index c65424e31c..ed1dd2faf9 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -2975,6 +2975,7 @@ holds the ordering. */ new Migration(224, "identity-capacity", V224Sql), new Migration(225, "server-aws-rds", V225Sql), new Migration(226, "server-aws-rds-web-or-express", V226Sql), + new Migration(228, "blocking-band-tiers", V228Sql), }; /// @@ -3002,6 +3003,24 @@ ALTER TABLE collect.servers ALTER TABLE collect.servers ADD COLUMN IF NOT EXISTS is_aws_rds_web_or_express boolean;"; + /// + /// V228 (#6927): the blocking health band's four tiers on the alert settings row, beside the V120 deadlock ones. They carry the + /// figures ServerHealthThresholds.BlockingWarnPerHour, BlockingCriticalPerHour, BlockingWarnWaitSeconds and + /// BlockingCriticalWaitSeconds have always been, so a store that never sets them bands exactly as before. They are literals here + /// because SQL text cannot name a C# constant. No CHECK ordering the two tiers of a pair, for the reason V120 gives: the writers refuse + /// a pair with critical below warn, and a hand-edited row reads as written. Idempotent: every column is added with IF NOT EXISTS, so a + /// store that already has them is left as it is. Table-level grants already cover new columns on this table, so no GRANT is added. + /// + private const string V228Sql = @" +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_warn_per_hour double precision NOT NULL DEFAULT 5.0; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_critical_per_hour double precision NOT NULL DEFAULT 20.0; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_warn_wait_seconds double precision NOT NULL DEFAULT 10.0; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_critical_wait_seconds double precision NOT NULL DEFAULT 60.0;"; + /// /// V189 (#5867): collect.query_store_compose_stamp_failures, the failure record of the Query Store compose builder: one row /// per hour whose last build threw, with its consecutive failures, last error and next attempt, so a build that cannot finish backs diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs index bfebc69b62..0db43132aa 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs @@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class StorageVersion { - public const int SchemaVersion = 226; + public const int SchemaVersion = 228; } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs index 98886ade29..46d0714e12 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs @@ -1669,6 +1669,17 @@ is exactly why DarlingFleetReader.GetFleetOverviewAsync hoists its own copy of t deadlockTiers = DeadlockRateThresholds.Default; } + /* #6927: the blocking band's tiers, hoisted like the deadlock ones above and for the same reasons. */ + BlockingBandThresholds blockingTiers; + try + { + blockingTiers = await service.GetBlockingBandThresholdsAsync(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + blockingTiers = BlockingBandThresholds.Default; + } + var perLane = await Task.WhenAll(lanes.Select(async lane => { var found = new List(lane.Count); @@ -1678,7 +1689,7 @@ is exactly why DarlingFleetReader.GetFleetOverviewAsync hoists its own copy of t try { var summary = await service.GetServerSummaryAsync( - server.ServerId, server.DisplayName, deadlockTiers); + server.ServerId, server.DisplayName, deadlockTiers, blockingTiers: blockingTiers); summary.ServerName = server.ServerName; /* #3029: the engine discriminator comes from the REGISTRY row, which already carries it (servers.engine_kind, via ManagedServersSql / ServersSql) — the per-server summary diff --git a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml index 0ca93eac7f..e05447a6b1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml @@ -445,6 +445,26 @@ + + + + + + + + + + + + + + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs index 025787b455..7b4fe4dd7d 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs @@ -923,6 +923,10 @@ and the alert text agree about what the number looks like. */ card agree about what the number looks like. */ BandDeadlockWarnPerHourBox.Text = r.DeadlockWarnPerHour.ToString("0.0", CultureInfo.InvariantCulture); BandDeadlockCriticalPerHourBox.Text = r.DeadlockCriticalPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnPerHourBox.Text = r.BlockingWarnPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalPerHourBox.Text = r.BlockingCriticalPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnWaitSecondsBox.Text = r.BlockingWarnWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalWaitSecondsBox.Text = r.BlockingCriticalWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); AlertPvsCheckBox.IsChecked = r.PvsEnabled; AlertPvsThresholdPercentBox.Text = r.PvsThresholdPercent.ToString(CultureInfo.InvariantCulture); AlertPvsFloorGbBox.Text = r.PvsFloorGb.ToString(CultureInfo.InvariantCulture); @@ -967,6 +971,23 @@ seeded from the row like every other alert-engine control. */ UpdateAlertControlStates(); } + /// Parses one blocking band box (#6927): InvariantCulture, within the named bounds the classifier clamps to and the MCP writer + /// accepts. A bad value adds an error naming the box and the range, and leaves null. + private static void BlockingBandBox( + System.Windows.Controls.TextBox box, double min, double max, string label, SettingsValidationErrors errors, out double? value) + { + if (double.TryParse(box.Text, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) && parsed >= min && parsed <= max) + { + value = parsed; + return; + } + + value = null; + errors.Add( + $"{label} must be a number between {min.ToString("0.0", CultureInfo.InvariantCulture)} and {max.ToString("0.0", CultureInfo.InvariantCulture)}.", + box); + } + /// Builds the store row from the alert + analysis controls, validating the numeric fields. Adds /// range errors to ; a bad field keeps its default rather than throwing. internal AlertSettingsRow BuildAlertRowFromControls(SettingsValidationErrors errors) @@ -1162,6 +1183,37 @@ not a pair to reject. */ && dlCritical >= ServerHealthThresholds.DeadlockRatePerHourFloor && dlCritical <= ServerHealthThresholds.DeadlockRatePerHourCeiling) row.DeadlockCriticalPerHour = dlCritical; + /* #6927: the blocking band, validated against the SAME named bounds BlockingBandThresholds clamps to and the MCP writer accepts, + InvariantCulture on the parse. Unlike the deadlock pair, a pair with critical below warn is refused, the rule update_alert_settings + applies, so the window cannot save a pair the tool would refuse. */ + BlockingBandBox(BandBlockingWarnPerHourBox, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling, + "Blocking warn per hour", errors, out var blWarnRate); + BlockingBandBox(BandBlockingCriticalPerHourBox, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling, + "Blocking critical per hour", errors, out var blCriticalRate); + BlockingBandBox(BandBlockingWarnWaitSecondsBox, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling, + "Blocking warn seconds", errors, out var blWarnWait); + BlockingBandBox(BandBlockingCriticalWaitSecondsBox, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling, + "Blocking critical seconds", errors, out var blCriticalWait); + if (blWarnRate.HasValue && blCriticalRate.HasValue) + { + if (blCriticalRate.Value < blWarnRate.Value) + errors.Add("Blocking critical per hour must not be lower than blocking warn per hour.", BandBlockingCriticalPerHourBox); + else + { + row.BlockingWarnPerHour = blWarnRate.Value; + row.BlockingCriticalPerHour = blCriticalRate.Value; + } + } + if (blWarnWait.HasValue && blCriticalWait.HasValue) + { + if (blCriticalWait.Value < blWarnWait.Value) + errors.Add("Blocking critical seconds must not be lower than blocking warn seconds.", BandBlockingCriticalWaitSecondsBox); + else + { + row.BlockingWarnWaitSeconds = blWarnWait.Value; + row.BlockingCriticalWaitSeconds = blCriticalWait.Value; + } + } if (int.TryParse(AlertPvsThresholdPercentBox.Text, out var pvsPct) && pvsPct is >= 0 and <= 100) row.PvsThresholdPercent = pvsPct; if (int.TryParse(AlertPvsFloorGbBox.Text, out var pvsFloor) && pvsFloor >= 0) @@ -1292,6 +1344,14 @@ the grid moved would arm the alert past the point it exists to precede. */ ServerHealthThresholds.DeadlockWarnPerHourDefault.ToString("0.0", CultureInfo.InvariantCulture); BandDeadlockCriticalPerHourBox.Text = ServerHealthThresholds.DeadlockCriticalPerHourDefault.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnPerHourBox.Text = + ServerHealthThresholds.BlockingWarnPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalPerHourBox.Text = + ServerHealthThresholds.BlockingCriticalPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnWaitSecondsBox.Text = + ServerHealthThresholds.BlockingWarnWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalWaitSecondsBox.Text = + ServerHealthThresholds.BlockingCriticalWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); AnalysisNotifyCooldownBox.Text = "360"; AlertPvsThresholdPercentBox.Text = "40"; AlertPvsFloorGbBox.Text = "1"; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs index f92c77fb31..f97ae27e73 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs @@ -99,7 +99,9 @@ the same reason and under the same connect-time store gate. */ /* #6873 (V223): the three database protection alert switches (backup, log backup, CHECKDB overdue), APPENDED, same gate. */ "backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, " + /* #6875 (V224): the Identity Capacity alert's switch (off by default) and its two bars, APPENDED for the same reason and under the same gate. */ - "identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent"; + "identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, " + + /* #6927 (V228): the blocking health band's four tiers, APPENDED for the same reason and under the same connect-time store gate. */ + "blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds"; /// The single global alert-settings row (id=1), for the Settings window prefill + the migrate-in /// defaults check. Column order matches . @@ -116,7 +118,7 @@ INSERT INTO config_alert_settings (id, " + AlertSettingsColumns + @", modified_a VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46, $47, $48, $49, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, - $63, $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86, + $63, $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86, $87, $88, $89, $90, (now() AT TIME ZONE 'UTC')) ON CONFLICT (id) DO UPDATE SET enabled = EXCLUDED.enabled, @@ -205,6 +207,10 @@ ON CONFLICT (id) DO UPDATE SET identity_capacity_enabled = EXCLUDED.identity_capacity_enabled, identity_capacity_warning_percent = EXCLUDED.identity_capacity_warning_percent, identity_capacity_critical_percent = EXCLUDED.identity_capacity_critical_percent, + blocking_warn_per_hour = EXCLUDED.blocking_warn_per_hour, + blocking_critical_per_hour = EXCLUDED.blocking_critical_per_hour, + blocking_warn_wait_seconds = EXCLUDED.blocking_warn_wait_seconds, + blocking_critical_wait_seconds = EXCLUDED.blocking_critical_wait_seconds, modified_at = (now() AT TIME ZONE 'UTC')"; /// The two cpu_mode values the service honors (it compares case-insensitively against @@ -344,6 +350,11 @@ parameter keeps its type on the INSERT arm. */ command.Parameters.Add(new NpgsqlParameter { TypedValue = r.IdentityCapacityEnabled }); // $84 (#6875, V224) command.Parameters.Add(new NpgsqlParameter { TypedValue = identityWarning }); // $85 (#6875, V224) command.Parameters.Add(new NpgsqlParameter { TypedValue = Math.Clamp(r.IdentityCapacityCriticalPercent, identityWarning, 100) }); // $86 (#6875, V224) + /* #6927 (V228): the blocking band tiers, stored raw like the deadlock pair above them. */ + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingWarnPerHour }); // $87 (#6927, V228) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingCriticalPerHour }); // $88 (#6927, V228) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingWarnWaitSeconds }); // $89 (#6927, V228) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingCriticalWaitSeconds }); // $90 (#6927, V228) } private static AlertSettingsRow ReadAlertSettingsRow(NpgsqlDataReader reader) => new() @@ -457,6 +468,11 @@ parameter keeps its type on the INSERT arm. */ IdentityCapacityEnabled = reader.GetBoolean(83), IdentityCapacityWarningPercent = reader.GetInt32(84), IdentityCapacityCriticalPercent = reader.GetInt32(85), + /* #6927 (V228) the four blocking band tiers appended at ordinals 86-89. */ + BlockingWarnPerHour = reader.GetDouble(86), + BlockingCriticalPerHour = reader.GetDouble(87), + BlockingWarnWaitSeconds = reader.GetDouble(88), + BlockingCriticalWaitSeconds = reader.GetDouble(89), }; /// Maps the Settings window's CPU-mode combo tag ("Total"/"SqlOnly") to the store value. @@ -614,6 +630,11 @@ is MB per HOUR averaged over the lookback (#3539 A8c) -- the Settings row's labe public int IdentityCapacityWarningPercent { get; set; } = 70; /// #6875 (V224): percent used at which an identity column is critical. public int IdentityCapacityCriticalPercent { get; set; } = 90; + /// #6927 (V228): the blocking health band's tiers, today's constants by default. + public double BlockingWarnPerHour { get; set; } = ServerHealthThresholds.BlockingWarnPerHour; + public double BlockingCriticalPerHour { get; set; } = ServerHealthThresholds.BlockingCriticalPerHour; + public double BlockingWarnWaitSeconds { get; set; } = ServerHealthThresholds.BlockingWarnWaitSeconds; + public double BlockingCriticalWaitSeconds { get; set; } = ServerHealthThresholds.BlockingCriticalWaitSeconds; public bool LongRunningQueryEnabled { get; set; } = true; public int LongRunningQueryThresholdMinutes { get; set; } = 30; public bool TempDbSpaceEnabled { get; set; } = true; @@ -749,6 +770,10 @@ public bool ValueEquals(AlertSettingsRow other) && IdentityCapacityEnabled == other.IdentityCapacityEnabled && IdentityCapacityWarningPercent == other.IdentityCapacityWarningPercent && IdentityCapacityCriticalPercent == other.IdentityCapacityCriticalPercent + && Math.Abs(BlockingWarnPerHour - other.BlockingWarnPerHour) < 0.0001 + && Math.Abs(BlockingCriticalPerHour - other.BlockingCriticalPerHour) < 0.0001 + && Math.Abs(BlockingWarnWaitSeconds - other.BlockingWarnWaitSeconds) < 0.0001 + && Math.Abs(BlockingCriticalWaitSeconds - other.BlockingCriticalWaitSeconds) < 0.0001 && LongRunningQueryEnabled == other.LongRunningQueryEnabled && LongRunningQueryThresholdMinutes == other.LongRunningQueryThresholdMinutes && TempDbSpaceEnabled == other.TempDbSpaceEnabled diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs index ccbebc3da0..e2381642ac 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs @@ -124,6 +124,7 @@ public async Task> GetDailySummaryRangeAsync( var banding = new DailyHealthThresholds { DeadlockRates = await GetDeadlockRateThresholdsAsync(cancellationToken), + BlockingBands = await GetBlockingBandThresholdsAsync(cancellationToken), }; /* #3653: the horizon, read once per range like the banding tiers above — a schedule save mid-read must diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs index 37453b9057..e5031f2bc6 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs @@ -290,7 +290,8 @@ public async Task GetServerSummaryAsync( int serverId, string displayName, DeadlockRateThresholds? deadlockTiers = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + BlockingBandThresholds? blockingTiers = null) { var nowUtc = DateTime.UtcNow; /* #3368: the window's LENGTH is named once and the start derived from it, so the card's deadlock @@ -586,6 +587,8 @@ know the engine (the loader stamps IsPostgres afterwards), and one arm is always DeadlockWindow = window, BlockingWindow = window, DeadlockRateThresholds = deadlockTiers, + /* #6927: the store's blocking tiers, or null for today's constants. */ + BlockingBandThresholds = blockingTiers, TotalThreads = totalThreads, CurrentWorkers = currentWorkers, ThreadsWaitingForCpu = threadsWaitingForCpu, @@ -622,6 +625,11 @@ FROM config_alert_settings /// Values come back RAW; clamps on read, so a hand-edited row /// cannot drive a nonsense threshold. /// + /// The store's blocking-band tiers (#6927), or today's constants when the singleton row is absent. The same read + /// the service uses (). + public Task GetBlockingBandThresholdsAsync(CancellationToken cancellationToken = default) => + BlockingBandStore.ReadAsync(_dataSource, ViewerCommandDeadlines.CurrentInteractiveReadSeconds, cancellationToken); + public async Task GetDeadlockRateThresholdsAsync(CancellationToken cancellationToken = default) { await using var command = _dataSource.CreateCommand(DeadlockRateThresholdSql); @@ -1328,10 +1336,10 @@ decision the service's fleet card uses so the two cannot disagree about whether : null; /// Blocking band (#3539 A3) — a 60 s block Critical whatever the rate; else blocking events per - /// hour over the window against the shared tiers (20/hr Critical, 5/hr Warning); a 10 s block Warning; - /// no source Unknown. + /// hour over the window against the store's tiers (#6927, 20/hr Critical, 5/hr Warning by default); a 10 s block + /// Warning by default; no source Unknown. public HealthSeverity BlockingSeverity => - ServerHealthClassifier.BlockingSeverity(BlockingCountForBand, MaxBlockedSeconds, BlockingWindow); + ServerHealthClassifier.BlockingSeverity(BlockingCountForBand, MaxBlockedSeconds, BlockingWindow, BlockingBandThresholds); /// /// The window and cover (#3368) — the @@ -1349,6 +1357,10 @@ decision the service's fleet card uses so the two cannot disagree about whether /// about which numbers are in force. public DeadlockRateThresholds? DeadlockRateThresholds { get; set; } + /// The store's blocking-band tiers (#6927), or null to band on today's constants, set by the read from + /// config_alert_settings so this card and the service's fleet card agree. + public BlockingBandThresholds? BlockingBandThresholds { get; set; } + /// Deadlocks per HOUR over — what the band evaluates (#3368), or /// null when the window is too short to normalise. Rendered beside the count so the dot's reason is /// legible. @@ -1424,6 +1436,7 @@ no difference (#3539). */ contradicting itself, which is the shape #3281 fixed on the CPU arm. */ DeadlockWindow = DeadlockWindow, DeadlockRateThresholds = DeadlockRateThresholds, + BlockingBandThresholds = BlockingBandThresholds, TotalThreads = TotalThreads, AvailableThreads = AvailableThreads, ThreadsWaitingForCpu = ThreadsWaitingForCpu, diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs index 7b42e0ab90..8436247a50 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs @@ -1377,7 +1377,10 @@ per the V71 finding. */ EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds' AND NOT attisdropped), /* V226 (#6893) adds the AWS RDS Web or Express fact to the server registry. Found by catalog like the V225 column above. Named only in this probe line, never in prose, per the V71 finding. */ - EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped)"; + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped), + /* V228 (#6927) adds the four settable blocking band columns to the alert settings row. Found by catalog like the V226 column above. The + warn per hour column is the sentinel, the first of the four. Named only in this probe line, never in prose, per the V71 finding. */ + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.config_alert_settings') AND attname = 'blocking_warn_per_hour' AND NOT attisdropped)"; /// The store schema version this viewer build requires — the highest migration it knows /// (). The connect-time gate blocks a store below this. @@ -1399,7 +1402,7 @@ per the V71 finding. */ await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201)); + return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201), reader.GetBoolean(202)); } return null; @@ -1424,7 +1427,7 @@ per the V71 finding. */ /// is unit-tested without a live store; any schema bump past the newest arm trips the pinning test that keeps /// this in step with . /// - internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false) + internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false, bool hasBlockingBands = false) { /* V71 (the PostgreSQL blocking-edges rung): a table-existence sentinel and now the newest-first arm. A collector table would ordinarily get no arm at all — see the V63-V69 note below — but the TOP @@ -1633,6 +1636,13 @@ store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through t /* V226 (#6893): the AWS RDS Web or Express fact on the server registry. Column-existence sentinel, newest-first, and the TOP rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 225 and showing a spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + /* V228 (#6927): the settable blocking band columns on the alert settings row. Column-existence sentinel, newest-first, and the TOP + rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 226 and showing a + spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + if (hasBlockingBands) + { + return 228; + } if (hasServerAwsRdsWebOrExpress) { return 226; diff --git a/PerformanceMonitor.Common/DailyHealthBand.cs b/PerformanceMonitor.Common/DailyHealthBand.cs index adb9176412..47a7dfbee0 100644 --- a/PerformanceMonitor.Common/DailyHealthBand.cs +++ b/PerformanceMonitor.Common/DailyHealthBand.cs @@ -211,6 +211,13 @@ public sealed record DailyHealthThresholds /// public DeadlockRateThresholds DeadlockRates { get; init; } = DeadlockRateThresholds.Default; + /// + /// The blocking band's tiers the day bands with (#6927) — the SAME store-backed values the Overview + /// card's blocking dot reads, so a day cell and the card cannot disagree about what a blocking rate or + /// a peak block means. Defaults to today's constants; Lite has no such knobs and bands on them. + /// + public BlockingBandThresholds BlockingBands { get; init; } = BlockingBandThresholds.Default; + /// The high-CPU sample count at or above which is Critical: the /// greater of and /// × the window's hours. An undeclared window @@ -275,7 +282,7 @@ arm are documented on ServerHealthThresholds. Delegating rather than restating i undeclared span) is the fallback this classifier wants: a non-zero count reads Warning, a zero count (Unknown) stays out of the blocking trigger. */ var blockingSeverity = ServerHealthClassifier.BlockingSeverity( - signals.BlockingReports ?? signals.BlockingEvents, signals.PeakBlockWaitMs / 1000.0, signals.Window); + signals.BlockingReports ?? signals.BlockingEvents, signals.PeakBlockWaitMs / 1000.0, signals.Window, t.BlockingBands); var cpuSeverity = HighCpuSeverity(signals.HighCpuEvents, signals.Window, t); var collectionSeverity = CollectionErrorSeverity(signals.CollectionErrors, signals.CollectionRuns); diff --git a/PerformanceMonitor.Common/ServerHealthBands.cs b/PerformanceMonitor.Common/ServerHealthBands.cs index c277dc33d3..7e53d036ff 100644 --- a/PerformanceMonitor.Common/ServerHealthBands.cs +++ b/PerformanceMonitor.Common/ServerHealthBands.cs @@ -448,6 +448,69 @@ public static class ServerHealthThresholds /// reach it first regardless. /// public const double DeadlockRatePerHourCeiling = 1000.0; + + /* ── the settable blocking band (#6927) ── */ + + /// + /// The FLOOR the settable blocking rate tiers clamp to (#6927): one event per hour, for the reason + /// gives. It keeps a tier a rate, and it keeps zero and + /// negative values, which would band every window Critical, out of the classifier. + /// + public const double BlockingRatePerHourFloor = 1.0; + + /// The CEILING the settable blocking rate tiers clamp to (#6927), so a tier stays a threshold and + /// not an undisclosed off switch. The worst measured server-hour held 232 reports, so 10,000 is far + /// above anything real. + public const double BlockingRatePerHourCeiling = 10000.0; + + /// The FLOOR the settable blocking wait tiers clamp to, seconds (#6927). A tier of zero would + /// band every block Critical. + public const double BlockingWaitSecondsFloor = 1.0; + + /// The CEILING the settable blocking wait tiers clamp to, seconds (#6927): one day. + public const double BlockingWaitSecondsCeiling = 86400.0; + } + + /// + /// The four settable tiers of the blocking band (#6927): events per hour and the longest single block in + /// seconds, each with a Warning and a Critical bar. The same shape as : + /// the store holds what the operator set, these accessors clamp on read, and a default struct + /// (every field zero) clamps to the floors rather than to zero. + /// + /// Defaults are today's constants ( and + /// its three siblings), so a store that never set them bands exactly as every build before this one + /// did. Lite has no store and keeps the constants. + /// + /// Like the deadlock pair, the critical bar is not floored at the warning bar here, so a hand-edited + /// pair reads as written. The writers refuse a pair with critical below warn. + /// + public readonly record struct BlockingBandThresholds( + double WarnPerHourRaw, + double CriticalPerHourRaw, + double WarnWaitSecondsRaw, + double CriticalWaitSecondsRaw) + { + /// Warning bar, events per hour, clamped. + public double WarnPerHour => Clamp(WarnPerHourRaw, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling); + + /// Critical bar, events per hour, clamped independently of the warning bar. + public double CriticalPerHour => Clamp(CriticalPerHourRaw, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling); + + /// Warning bar, longest block in seconds, clamped. + public double WarnWaitSeconds => Clamp(WarnWaitSecondsRaw, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling); + + /// Critical bar, longest block in seconds, clamped independently of the warning bar. + public double CriticalWaitSeconds => Clamp(CriticalWaitSecondsRaw, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling); + + /// Today's constants: what a store at its V228 column defaults hands the band. + public static BlockingBandThresholds Default => new( + ServerHealthThresholds.BlockingWarnPerHour, + ServerHealthThresholds.BlockingCriticalPerHour, + ServerHealthThresholds.BlockingWarnWaitSeconds, + ServerHealthThresholds.BlockingCriticalWaitSeconds); + + private static double Clamp(double value, double floor, double ceiling) => + double.IsNaN(value) ? floor : Math.Clamp(value, floor, ceiling); } /// @@ -582,6 +645,12 @@ public readonly record struct ServerHealthMetrics /// public DeadlockRateThresholds? DeadlockRateThresholds { get; init; } + /// The blocking band's four tiers (#6927), or null for today's constants + /// (). A Darling path with a store hands the + /// config_alert_settings values in so the card, the Overview tab, the calendar and the fleet + /// sweep band on what the operator set. Lite has no such knobs and leaves it null. + public BlockingBandThresholds? BlockingBandThresholds { get; init; } + /// Worker-thread ceiling (max_workers_count), or null with no scheduler snapshot (e.g. Azure SQL DB). public int? TotalThreads { get; init; } @@ -887,7 +956,8 @@ public static HealthSeverity MemorySeverity(bool? hasMemoryPressure) public static HealthSeverity BlockingSeverity( long? blockingCountOrNullWhenUnmeasured, double maxBlockedSeconds, - TimeSpan window) + TimeSpan window, + BlockingBandThresholds? thresholds = null) { if (!blockingCountOrNullWhenUnmeasured.HasValue) { @@ -895,19 +965,20 @@ public static HealthSeverity BlockingSeverity( } var blockingCount = blockingCountOrNullWhenUnmeasured.Value; + var tiers = thresholds ?? BlockingBandThresholds.Default; - if (maxBlockedSeconds >= ServerHealthThresholds.BlockingCriticalWaitSeconds) + if (maxBlockedSeconds >= tiers.CriticalWaitSeconds) { return HealthSeverity.Critical; } var ratePerHour = BlockingRatePerHour(blockingCount, window); - if (ratePerHour.HasValue && ratePerHour.Value >= ServerHealthThresholds.BlockingCriticalPerHour) + if (ratePerHour.HasValue && ratePerHour.Value >= tiers.CriticalPerHour) { return HealthSeverity.Critical; } - if (maxBlockedSeconds >= ServerHealthThresholds.BlockingWarnWaitSeconds) + if (maxBlockedSeconds >= tiers.WarnWaitSeconds) { return HealthSeverity.Warning; } @@ -917,7 +988,7 @@ public static HealthSeverity BlockingSeverity( return blockingCount > 0 ? HealthSeverity.Warning : HealthSeverity.Unknown; } - return ratePerHour.Value >= ServerHealthThresholds.BlockingWarnPerHour + return ratePerHour.Value >= tiers.WarnPerHour ? HealthSeverity.Warning : HealthSeverity.Healthy; } @@ -1140,7 +1211,7 @@ public static IEnumerable MetricSeverities(ServerHealthMetrics m yield return CpuSeverity(m.CpuPercentForAlert, m.CapacityUtilizationPercent, m.CpuSource); yield return ThreadsSeverity(m.TotalThreads, m.AvailableThreads, m.ThreadsWaitingForCpu, m.RequestsWaitingForThreads); yield return MemorySeverity(m.HasMemoryPressure); - yield return BlockingSeverity(m.BlockingCount, m.MaxBlockedSeconds, m.BlockingWindow); + yield return BlockingSeverity(m.BlockingCount, m.MaxBlockedSeconds, m.BlockingWindow, m.BlockingBandThresholds); yield return DeadlockSeverity( m.DeadlockCount, m.DeadlockWindow, From 5ec3b92772f51171f397fb894512c0945117765e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:47:20 -0400 Subject: [PATCH 04/22] Carry the blocking tiers on the Overview card item, and fix the settings row sample (#6927) --- Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs | 3 ++- Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs | 4 +++- .../ViewerDataService.Overview.cs | 5 +---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs b/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs index 2b0ee84ee0..92ff8853e8 100644 --- a/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs @@ -1333,7 +1333,8 @@ dropped the columns and fell back to the engine defaults still match. Warning is ClientTimeoutsWarningMultiple: 2, ClientTimeoutsCriticalMultiple: 7, AgentAlertGapsEnabled: true, BackupOverdueEnabled: true, LogBackupOverdueEnabled: true, CheckDbOverdueEnabled: true, - IdentityCapacityEnabled: true, IdentityCapacityWarningPercent: 72, IdentityCapacityCriticalPercent: 92); + IdentityCapacityEnabled: true, IdentityCapacityWarningPercent: 72, IdentityCapacityCriticalPercent: 92, + BlockingWarnPerHour: 6.5, BlockingCriticalPerHour: 22.5, BlockingWarnWaitSeconds: 11.5, BlockingCriticalWaitSeconds: 62.5); [Fact] public void AlertSettingsSql_ReadsSingleGlobalRow() diff --git a/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs index 46d0714e12..7dd680f253 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs @@ -1689,7 +1689,9 @@ is exactly why DarlingFleetReader.GetFleetOverviewAsync hoists its own copy of t try { var summary = await service.GetServerSummaryAsync( - server.ServerId, server.DisplayName, deadlockTiers, blockingTiers: blockingTiers); + server.ServerId, server.DisplayName, deadlockTiers); + /* #6927: the store's blocking tiers; the card's blocking band and its re-band read them from the item. */ + summary.BlockingBandThresholds = blockingTiers; summary.ServerName = server.ServerName; /* #3029: the engine discriminator comes from the REGISTRY row, which already carries it (servers.engine_kind, via ManagedServersSql / ServersSql) — the per-server summary diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs index e5031f2bc6..297ff2b1b0 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs @@ -290,8 +290,7 @@ public async Task GetServerSummaryAsync( int serverId, string displayName, DeadlockRateThresholds? deadlockTiers = null, - CancellationToken cancellationToken = default, - BlockingBandThresholds? blockingTiers = null) + CancellationToken cancellationToken = default) { var nowUtc = DateTime.UtcNow; /* #3368: the window's LENGTH is named once and the start derived from it, so the card's deadlock @@ -587,8 +586,6 @@ know the engine (the loader stamps IsPostgres afterwards), and one arm is always DeadlockWindow = window, BlockingWindow = window, DeadlockRateThresholds = deadlockTiers, - /* #6927: the store's blocking tiers, or null for today's constants. */ - BlockingBandThresholds = blockingTiers, TotalThreads = totalThreads, CurrentWorkers = currentWorkers, ThreadsWaitingForCpu = threadsWaitingForCpu, From 208e3fc0dff1e6662e1c641e2883a8587fe56474 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:52:02 -0400 Subject: [PATCH 05/22] Mark the daily classifier test day as having data (#6927) --- Darling/Darling.Tests/BlockingBandTiersTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Darling/Darling.Tests/BlockingBandTiersTests.cs b/Darling/Darling.Tests/BlockingBandTiersTests.cs index 7bd72a3314..e8d98cf0d0 100644 --- a/Darling/Darling.Tests/BlockingBandTiersTests.cs +++ b/Darling/Darling.Tests/BlockingBandTiersTests.cs @@ -44,6 +44,7 @@ public void TheDailyClassifier_UsesTheTunedTiers() { var signals = new DailyHealthSignals { + HasData = true, Window = Day, BlockingReports = 1764, BlockingEvents = 1764, From dc8a5ab824695f3866464c7c59b105dedf98e1f4 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:07:26 -0400 Subject: [PATCH 06/22] Band each fleet totals tile by its worst card, and test the V228 upgrade and the settings round trip (#6927) The fleet total was banded as one rate against the per-server tiers, so ten servers at 3 events an hour (none even Warning) made the tile Critical. The roll-up now takes the worst card band per signal, in the same fields. Adds live tests for the V228 upgrade on a store with a settings row and for update_alert_settings and get_alert_settings with the critical-below-warn refusal, and moves the settings column pins from 86 to 90. --- .../BlockingBandSettingsLiveTests.cs | 152 ++++++++++++++++++ .../Darling.Tests/BlockingBandTiersTests.cs | 34 +++- .../ClientTimeoutsSettingsSurfaceTests.cs | 14 +- .../Mcp/DarlingFleetReader.cs | 31 ++-- 4 files changed, 199 insertions(+), 32 deletions(-) create mode 100644 Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs diff --git a/Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs b/Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs new file mode 100644 index 0000000000..6f835bcf51 --- /dev/null +++ b/Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// The blocking band tiers (#6927, V228) on a real PostgreSQL store: an upgraded store whose settings row already exists keeps +/// today's defaults and its other values, and the tiers round trip through update_alert_settings and get_alert_settings +/// with a critical bar below its warn bar refused and nothing written. Each fact mints its own scratch database. +/// +/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Each fact mints its own scratch database through + ScratchPostgres and never touches the shared store's tables. */ +public sealed class BlockingBandSettingsLiveTests +{ + private const string TiersSql = "SELECT blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1"; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task AnUpgradedStoreWithASettingsRow_KeepsTodaysDefaults_AndItsOtherValues() + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the V228 upgrade test."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + + await PgMigrations.MigrateAsync(connection, ct); + await SeedAsync(scratch.ConnectionString, ct); + + /* Roll the store back to look like V227: the settings row exists and carries an operator's own deadlock tier, the four + columns do not. */ + await ExecAsync(connection, ct, "UPDATE config.config_alert_settings SET deadlock_warn_per_hour = 7.0, deadlock_critical_per_hour = 30.0 WHERE id = 1"); + foreach (var column in new[] { "blocking_warn_per_hour", "blocking_critical_per_hour", "blocking_warn_wait_seconds", "blocking_critical_wait_seconds" }) + { + await ExecAsync(connection, ct, "ALTER TABLE config.config_alert_settings DROP COLUMN " + column); + } + + await ExecAsync(connection, ct, "DELETE FROM darling_schema_version WHERE version >= 228"); + + var applied = await PgMigrations.MigrateAsync(connection, ct); + Assert.Equal(PgMigrations.Scripts.Count(m => m.Version >= 228), applied); + + Assert.Equal((5.0, 20.0, 10.0, 60.0), await ReadTiersAsync(connection, ct)); + + /* The band reads today's constants from the upgraded row, and the operator's deadlock tier did not move. */ + await using var dataSource = NpgsqlDataSource.Create(scratch.ConnectionString); + Assert.Equal(BlockingBandThresholds.Default, await BlockingBandStore.ReadAsync(dataSource, 10, ct)); + using var deadlock = new NpgsqlCommand("SELECT deadlock_warn_per_hour, deadlock_critical_per_hour FROM config_alert_settings WHERE id = 1", connection); + using var reader = await deadlock.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct)); + Assert.Equal(7.0, reader.GetDouble(0)); + Assert.Equal(30.0, reader.GetDouble(1)); + } + + [Fact] + public async Task UpdateAlertSettings_SavesTheFourTiers_GetAlertSettingsReadsThemBack_AndACriticalBelowWarnIsRefused() + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the blocking band settings test."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await SeedAsync(scratch.ConnectionString, ct); + + await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString); + + /* The seeded row reads today's constants. */ + AssertBands(await DarlingMcpAlertTools.GetAlertSettings(postgres), 5, 20, 10, 60); + + /* A valid body saves, and a fresh read carries it. */ + var saved = await DarlingMcpAlertTools.UpdateAlertSettings(postgres, + "{\"health_bands\":{\"blocking_warn_per_hour\":50,\"blocking_critical_per_hour\":100,\"blocking_warn_wait_seconds\":60,\"blocking_critical_wait_seconds\":180}}"); + Assert.Equal("updated", DarlingMcpTestData.StatusOf(saved)); + Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct)); + AssertBands(await DarlingMcpAlertTools.GetAlertSettings(postgres), 50, 100, 60, 180); + Assert.Equal(new BlockingBandThresholds(50, 100, 60, 180), await BlockingBandStore.ReadAsync(postgres, 10, ct)); + + /* A critical bar below its warn bar is refused with nothing written: in one body, and as a lone value against the stored other one. */ + foreach (var body in new[] + { + "{\"health_bands\":{\"blocking_warn_per_hour\":80,\"blocking_critical_per_hour\":30}}", + "{\"health_bands\":{\"blocking_critical_per_hour\":40}}", + "{\"health_bands\":{\"blocking_warn_wait_seconds\":200}}", + "{\"health_bands\":{\"blocking_warn_wait_seconds\":90,\"blocking_critical_wait_seconds\":30}}", + }) + { + var refused = await DarlingMcpAlertTools.UpdateAlertSettings(postgres, body); + Assert.Equal("invalid", DarlingMcpTestData.StatusOf(refused)); + Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct)); + } + + /* Out of range is refused too. */ + Assert.Equal("invalid", DarlingMcpTestData.StatusOf(await DarlingMcpAlertTools.UpdateAlertSettings(postgres, + "{\"health_bands\":{\"blocking_warn_per_hour\":0}}"))); + Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct)); + } + + private static void AssertBands(string settings, double warnPerHour, double criticalPerHour, double warnWait, double criticalWait) + { + Assert.False(McpHelpers.IsErrorEnvelope(settings), settings); + using var doc = JsonDocument.Parse(settings); + var bands = doc.RootElement.GetProperty("health_bands"); + Assert.Equal(warnPerHour, bands.GetProperty("blocking_warn_per_hour").GetDouble()); + Assert.Equal(criticalPerHour, bands.GetProperty("blocking_critical_per_hour").GetDouble()); + Assert.Equal(warnWait, bands.GetProperty("blocking_warn_wait_seconds").GetDouble()); + Assert.Equal(criticalWait, bands.GetProperty("blocking_critical_wait_seconds").GetDouble()); + } + + private static async Task<(double WarnPerHour, double CriticalPerHour, double WarnWait, double CriticalWait)> ReadTiersAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var command = new NpgsqlCommand(TiersSql, connection); + using var reader = await command.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct), "config_alert_settings has no singleton row"); + return (reader.GetDouble(0), reader.GetDouble(1), reader.GetDouble(2), reader.GetDouble(3)); + } + + private static async Task ExecAsync(NpgsqlConnection connection, CancellationToken ct, string sql) + { + using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(ct); + } + + /// Seeds the singleton rows through the product's own path, so this test need not track their other NOT NULL columns. + private static async Task SeedAsync(string connectionString, CancellationToken ct) + { + await using var dataSource = NpgsqlDataSource.Create(connectionString); + await new StoreConfigProvider(dataSource).SeedIfEmptyAsync(new DarlingConfig(), ct); + } +} diff --git a/Darling/Darling.Tests/BlockingBandTiersTests.cs b/Darling/Darling.Tests/BlockingBandTiersTests.cs index e8d98cf0d0..7f0e52cfe1 100644 --- a/Darling/Darling.Tests/BlockingBandTiersTests.cs +++ b/Darling/Darling.Tests/BlockingBandTiersTests.cs @@ -67,16 +67,40 @@ public void RawValuesClampToTheFloorsAndCeilings_AndADefaultStructClampsToTheFlo Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, default(BlockingBandThresholds).WarnPerHour); } + private static FleetServerCard CardAt(int id, HealthSeverity blocking, HealthSeverity deadlock) => new() + { + ServerId = id, + DisplayName = "s" + id, + ServerName = "s" + id, + BlockingCount = 3, + BlockingSeverity = blocking, + DeadlockSeverity = deadlock, + }; + [Fact] - public void TheFleetTotalsTiles_AreBandedByTheClassifier_NotOnAnyNonZeroCount() + public void TheFleetTotalsTiles_TakeTheWorstCardBand_SoTenServersAtThreeAnHourAreNotCritical() { var now = new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc); var empty = DarlingFleetReader.BuildRollup(Array.Empty(), now, now.AddHours(-1), now); + Assert.Equal(HealthSeverity.Healthy, empty.TotalBlockingSeverity); Assert.Equal(HealthSeverity.Healthy, empty.TotalDeadlockSeverity); - var tuned = DarlingFleetReader.BuildRollup( - new List(), now, now.AddHours(-1), now, - deadlockTiers: new DeadlockRateThresholds(50.0, 100.0), blockingTiers: new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0)); - Assert.Equal(HealthSeverity.Healthy, tuned.TotalDeadlockSeverity); + /* Ten servers each at 3 reports an hour: no card is even Warning at the default 5 an hour, and the fleet sum of 30 an hour + must not band as one server's rate. */ + var window = TimeSpan.FromHours(1); + var cards = new List(); + for (var i = 1; i <= 10; i++) + { + var sev = ServerHealthClassifier.BlockingSeverity(3, 0.0, window, BlockingBandThresholds.Default); + Assert.Equal(HealthSeverity.Healthy, sev); + cards.Add(CardAt(i, sev, HealthSeverity.Healthy)); + } + Assert.Equal(30, DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now).TotalBlockingEvents); + Assert.Equal(HealthSeverity.Healthy, DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now).TotalBlockingSeverity); + + cards[4] = CardAt(5, HealthSeverity.Critical, HealthSeverity.Warning); + var one = DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now); + Assert.Equal(HealthSeverity.Critical, one.TotalBlockingSeverity); + Assert.Equal(HealthSeverity.Warning, one.TotalDeadlockSeverity); } } diff --git a/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs b/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs index cfd8abc4ef..325bb6f96b 100644 --- a/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs +++ b/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs @@ -23,7 +23,7 @@ namespace Darling.Tests; /// #5640 (Lane E1): the Darling Viewer's Client Timeouts settings, the V179 twin of /// RunnableQueueSettingsSurfaceTests. The row carries the engine's defaults (ON, 10, 60), compares and saves /// all three columns through the engine's clamp, and the Settings window names the alert. V179 is the newest append, so -/// this class owns the settings row's total width (86). The viewer needs no per-version gate for the columns: it refuses a +/// this class owns the settings row's total width (90). The viewer needs no per-version gate for the columns: it refuses a /// store below StorageVersion.SchemaVersion at connect, as it did for V177's. /// public sealed class ClientTimeoutsSettingsSurfaceTests @@ -72,7 +72,7 @@ public void TheUpsertAndTheSelect_CarryTheThreeColumns_AtTheEnd() /// /// V179's three columns sit at the FIXED ordinals 73, 74 and 75 (parameters $74 to $76) on every positional - /// settings-row surface, and the row is 86 columns wide (V200 appended the excluded volumes at 76, V211 the two multiples at 77 and 78, V222 the Agent alert gaps switch at 79, V223 the three database protection switches at 80 to 82, V224 the three identity capacity columns at 83 to 85): the viewer's select and upsert, the service's own read and + /// settings-row surface, and the row is 90 columns wide (V228 appended the four blocking band tiers at 86 to 89, V200 appended the excluded volumes at 76, V211 the two multiples at 77 and 78, V222 the Agent alert gaps switch at 79, V223 the three database protection switches at 80 to 82, V224 the three identity capacity columns at 83 to 85): the viewer's select and upsert, the service's own read and /// its seed, and the MCP read with its positional record. This is the test that was red while the seed had 76 columns /// and the viewer 73. /// @@ -82,7 +82,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co var viewerColumns = ViewerDataService.AlertSettingsSelectSql .Replace("SELECT ", "", StringComparison.Ordinal).Split(" FROM ")[0] .Split(',').Select(c => c.Trim()).ToList(); - Assert.Equal(86, viewerColumns.Count); + Assert.Equal(90, viewerColumns.Count); Assert.Equal(Expected, viewerColumns.Skip(73).Take(3).ToArray()); var service = File.ReadAllText(Path.Combine(RepoRoot(), "Darling", "PerformanceMonitor.Darling.Service", "StoreConfigProvider.cs")); @@ -91,7 +91,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co var select = service[start..]; select = select[(select.IndexOf("SELECT", StringComparison.Ordinal) + 6)..select.IndexOf("FROM config_alert_settings WHERE id = 1", StringComparison.Ordinal)]; var serviceColumns = select.Split(',').Select(c => c.Trim()).Where(c => c.Length > 0).ToList(); - Assert.Equal(86, serviceColumns.Count); + Assert.Equal(90, serviceColumns.Count); Assert.Equal(Expected, serviceColumns.Skip(73).Take(3).ToArray()); Assert.Contains("ClientTimeoutsEnabled = reader.GetBoolean(73),", service, StringComparison.Ordinal); Assert.Contains("ClientTimeoutsWarningPerMinute = reader.GetInt32(74),", service, StringComparison.Ordinal); @@ -107,10 +107,10 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co var mcpColumns = DarlingAlertReader.AlertSettingsSelectSql .Split("FROM config_alert_settings")[0].Replace("SELECT", "", StringComparison.Ordinal) .Split(',').Select(c => c.Trim()).Where(c => c.Length > 0).ToList(); - Assert.Equal(86, mcpColumns.Count); + Assert.Equal(90, mcpColumns.Count); Assert.Equal(Expected, mcpColumns.Skip(73).Take(3).ToArray()); var parameters = typeof(DarlingAlertReader.AlertSettingsReadRow).GetConstructors().Single().GetParameters(); - Assert.Equal(86, parameters.Length); + Assert.Equal(90, parameters.Length); Assert.Equal(new[] { "ClientTimeoutsEnabled", "ClientTimeoutsWarningPerMinute", "ClientTimeoutsCriticalPerMinute" }, parameters.Skip(73).Take(3).Select(p => p.Name!).ToArray()); Assert.Equal(typeof(bool), parameters[73].ParameterType); @@ -120,7 +120,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co /* The viewer's bind supplies $74 to $76 at indexes 73 to 75, in column order (V200's excluded_volumes follows at 76). */ var row = new AlertSettingsRow { ClientTimeoutsEnabled = true, ClientTimeoutsWarningPerMinute = 5, ClientTimeoutsCriticalPerMinute = 11 }; var command = Bind(row); - Assert.Equal(86, command.Parameters.Count); + Assert.Equal(90, command.Parameters.Count); Assert.Equal(true, command.Parameters[73].Value); Assert.Equal(5, command.Parameters[74].Value); Assert.Equal(11, command.Parameters[75].Value); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index 5fcb31de44..fb4609d8be 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -973,7 +973,7 @@ server the read reported with an empty window reads Offline or never-collected b } return BuildRollup(cards, now, windowStartUtc, windowEndUtc, worstCount, tagForest, collectionHealthAgeSeconds, - await ReadCollectionStateAsync(postgres, logger, cancellationToken), deadlockTiers, blockingTiers); + await ReadCollectionStateAsync(postgres, logger, cancellationToken)); } /// Builds one pre-banded card from a server's raw cross-server reads (pure — the reduction the WPF @@ -1329,9 +1329,7 @@ public static FleetOverviewResult BuildRollup( int worstCount = DefaultWorstCount, IReadOnlyList? tags = null, int collectionHealthAgeSeconds = 0, - FleetCollectionState? collection = null, - DeadlockRateThresholds? deadlockTiers = null, - BlockingBandThresholds? blockingTiers = null) + FleetCollectionState? collection = null) { var healthy = 0; var warning = 0; @@ -1340,8 +1338,11 @@ public static FleetOverviewResult BuildRollup( var failures = 0; long totalBlocking = 0; long totalDeadlocks = 0; - long? totalBlockingReports = null; - var maxBlockedSeconds = 0.0; + /* #6927: each totals tile takes the worst band among the cards for that signal, the classification the cards already show. + A fleet total is not a per-server rate: ten servers at 3 an hour each would otherwise band the sum of 30 against the + per-server tiers and turn Critical with no card even Warning. */ + var blockingTileSeverity = HealthSeverity.Healthy; + var deadlockTileSeverity = HealthSeverity.Healthy; var deadlockSourcesRead = 0; var deadlockPostgresTargets = 0; var deadlockCollectorsSilent = 0; @@ -1385,14 +1386,8 @@ attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on totalBlocking += card.BlockingCount; totalDeadlocks += card.DeadlockCount; - /* #6927: the blocking tile bands on the reports the cards band on, and the longest block anywhere. A PostgreSQL card - has no blocking source, so it adds nothing and does not make the tile measured. */ - var cardMetrics = card.ToHealthMetrics(); - if (cardMetrics.BlockingCount.HasValue) - { - totalBlockingReports = (totalBlockingReports ?? 0) + cardMetrics.BlockingCount.Value; - maxBlockedSeconds = Math.Max(maxBlockedSeconds, cardMetrics.MaxBlockedSeconds); - } + if (card.BlockingSeverity > blockingTileSeverity) blockingTileSeverity = card.BlockingSeverity; + if (card.DeadlockSeverity > deadlockTileSeverity) deadlockTileSeverity = card.DeadlockSeverity; } var problems = cards @@ -1428,12 +1423,8 @@ attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on ServersWithCollectionFailures = failures, TotalBlockingEvents = totalBlocking, TotalDeadlocks = totalDeadlocks, - /* #6927: the two totals tiles band through the same classifier and the store's tiers as the cards, over the window the totals - were read in, instead of the page colouring them on any non-zero count. */ - TotalBlockingSeverity = ServerHealthClassifier.BlockingSeverity( - totalBlockingReports, maxBlockedSeconds, windowEndUtc - windowStartUtc, blockingTiers), - TotalDeadlockSeverity = ServerHealthClassifier.DeadlockSeverity( - totalDeadlocks, windowEndUtc - windowStartUtc, deadlockTiers ?? DeadlockRateThresholds.Default), + TotalBlockingSeverity = blockingTileSeverity, + TotalDeadlockSeverity = deadlockTileSeverity, DeadlockCoverage = new FleetDeadlockCoverage { ServersRead = deadlockSourcesRead, From dbceee6f1f126d0920b5e0e92bfb64741c8c88b3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:42:48 -0400 Subject: [PATCH 07/22] Accept a blank severities word on a route as both and clear the CA1862 warning (part of #6928) --- .../NotificationRouteWriter.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs b/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs index 27433670af..3981a55ebc 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs @@ -403,9 +403,10 @@ public static (int Count, string[] Domains) DescribeRecipients(string? recipient /// The sentence for a severity filter that is not both, critical or warning (case-insensitive), else null (#6928). public static string? SeveritiesRefusal(string? severities) => - severities is not null && NotificationRouteKinds.NormalizeSeverities(severities) != severities.Trim().ToLowerInvariant() - ? "severities is critical, warning or both." - : null; + severities?.Trim().ToLowerInvariant() is null or "" or NotificationRouteKinds.SeveritiesBoth + or NotificationRouteKinds.SeveritiesCritical or NotificationRouteKinds.SeveritiesWarning + ? null + : "severities is critical, warning or both."; /// Binds the agent columns, the resolution override and the fallback, in the order every insert and update takes them. public static void AddAgentColumns(NpgsqlCommand command, NotificationRouteDraft row) From 32b09cc93c38b24becb4a28dfa29c6a8d37a0224 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:44:03 -0400 Subject: [PATCH 08/22] Show and edit the route severity filter in the viewer and the web Routes list (part of #6928) --- .../Darling.Tests/AdminPendingNotesTests.cs | 32 +++++ .../Darling.Tests/AgentSetupWebLiveTests.cs | 13 ++ .../ViewerNotificationRouteSeveritiesTests.cs | 133 ++++++++++++++++++ .../DarlingWebAgentSetup.cs | 11 ++ .../wwwroot/js/pages/admin-agent-setup.js | 8 +- .../wwwroot/js/pages/admin.js | 9 ++ .../NotificationRouteEditDialog.xaml | 14 +- .../NotificationRouteEditDialog.xaml.cs | 12 ++ .../NotificationRoutesWindow.xaml | 2 + .../ViewerDataService.NotificationRoutes.cs | 65 +++++++-- .../ViewerDataService.cs | 16 ++- 11 files changed, 294 insertions(+), 21 deletions(-) create mode 100644 Darling/Darling.Tests/ViewerNotificationRouteSeveritiesTests.cs diff --git a/Darling/Darling.Tests/AdminPendingNotesTests.cs b/Darling/Darling.Tests/AdminPendingNotesTests.cs index 4a2016a730..e3a6755624 100644 --- a/Darling/Darling.Tests/AdminPendingNotesTests.cs +++ b/Darling/Darling.Tests/AdminPendingNotesTests.cs @@ -120,6 +120,38 @@ public void AddingARouteWithNoMatch_IsRefusedByTheFormBeforeARequest_Pure() Assert.Equal("blocking", named.GetProperty("payload").GetProperty("metric_match").GetString()); } + /* #6928: the Routes list names the severity filter in the desktop window's words (Both when the field is missing or unknown), the editor + reads it back from a route and the save sends it, and a word the page does not know is sent as both. */ + [Fact] + public void TheRoutesList_ShowsTheSeverityBesideTheScope_AndTheEditorRoundTripsIt_Pure() + { + var data = JsonNode.Parse(""" + {"routes":[ + {"route_id":1,"metric_match":"blocking","severities":"critical","enabled":true}, + {"route_id":2,"metric_match":"performance","severities":"warning","enabled":true}, + {"route_id":3,"metric_match":"reports","severities":"both","enabled":true}, + {"route_id":4,"metric_match":"agent-jobs","enabled":true}, + {"route_id":5,"metric_match":"waits","severities":"someday","enabled":true} + ]} + """); + var rows = Pure("routeRows", data).EnumerateArray().ToArray(); + Assert.Equal(new[] { "Critical only", "Warning only", "Both", "Both", "Both" }, rows.Select(r => r.GetProperty("severities").GetString())); + + var columns = Pure("ROUTE_COLUMNS").EnumerateArray().Select(c => c.GetProperty("key").GetString()).ToList(); + Assert.Equal(columns.IndexOf("applies_to") + 1, columns.IndexOf("severities")); + + Assert.Equal("critical", Pure("entriesFromRoute", new { route_id = 1, metric_match = "blocking", severities = "critical" }).GetProperty("severities").GetString()); + Assert.Equal("both", Pure("entriesFromRoute", new { route_id = 1, metric_match = "blocking" }).GetProperty("severities").GetString()); + Assert.Equal("both", Pure("entriesFromRoute", (object?)null).GetProperty("severities").GetString()); + + string Sent(string severities) => Pure("routePayload", new { id = (int?)null, match = "blocking", perHour = "10", perDay = "50", lease = "30", expiry = "120", fallback = "", scope = "", level = "", resolutions = "", enabled = true, severities }) + .GetProperty("payload").GetProperty("severities").GetString()!; + Assert.Equal("warning", Sent("warning")); + Assert.Equal("critical", Sent("critical")); + Assert.Equal("both", Sent("both")); + Assert.Equal("both", Sent("someday")); + } + [Fact] public void OnlyASeatThatCanEdit_GetsTheDecideColumn_Pure() { diff --git a/Darling/Darling.Tests/AgentSetupWebLiveTests.cs b/Darling/Darling.Tests/AgentSetupWebLiveTests.cs index 37341f26a0..5f226c8a87 100644 --- a/Darling/Darling.Tests/AgentSetupWebLiveTests.cs +++ b/Darling/Darling.Tests/AgentSetupWebLiveTests.cs @@ -246,6 +246,17 @@ await WithWebRoutesAsync("admin", async (owner, _, Send, ct) => Assert.Equal("Analysis: Waits|ops@example.com|3|60|500|false", await ScalarAsync(owner, $"SELECT metric_match || '|' || smtp_recipients || '|' || agent_level || '|' || agent_max_per_hour || '|' || agent_max_per_day || '|' || enabled::text FROM config_notification_routes WHERE route_id = {seeded}", ct)); + /* #6928: the severity filter. A new route is Both, an edit sets it (any case), an edit that leaves it out keeps it. */ + Assert.Equal("both", await ScalarAsync(owner, $"SELECT severities FROM config_notification_routes WHERE route_id = {seeded}", ct)); + Assert.Equal(200, (await Send("PUT", $"/api/notification-routes/{seeded}", Json, "{\"severities\":\"Critical\"}")).Status); + Assert.Equal("critical", await ScalarAsync(owner, $"SELECT severities FROM config_notification_routes WHERE route_id = {seeded}", ct)); + Assert.Equal(200, (await Send("PUT", $"/api/notification-routes/{seeded}", Json, "{\"enabled\":true}")).Status); + Assert.Equal("critical", await ScalarAsync(owner, $"SELECT severities FROM config_notification_routes WHERE route_id = {seeded}", ct)); + var warningOnly = await Send("POST", "/api/notification-routes", Json, "{\"metric_match\":\"blocking\",\"agent_level\":1,\"severities\":\"warning\"}"); + Assert.Equal(200, warningOnly.Status); + Assert.Equal("warning", await ScalarAsync(owner, $"SELECT severities FROM config_notification_routes WHERE route_id = {(int)warningOnly.Body!["route_id"]!}", ct)); + Assert.Equal(200, (await Send("PUT", $"/api/notification-routes/{seeded}", Json, "{\"severities\":\"both\"}")).Status); + /* Level 0 (Off) and Inherit are both settable. */ Assert.Equal(200, (await Send("PUT", $"/api/notification-routes/{seeded}", Json, "{\"agent_level\":0}")).Status); Assert.Equal("0", await ScalarAsync(owner, $"SELECT agent_level FROM config_notification_routes WHERE route_id = {seeded}", ct)); @@ -401,6 +412,8 @@ async Task Refused(string method, string path, string json, string contains) await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\"}", "at least one destination or the agent channel"); await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\",\"agent_level\":1,\"fallback_route_id\":99999}", "Fallback route 99999 does not exist"); await Refused("PUT", $"/api/notification-routes/{seeded}", $"{{\"fallback_route_id\":{seeded}}}", "its own fallback"); + await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\",\"agent_level\":1,\"severities\":\"major\"}", "severities is critical, warning or both"); + await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\",\"agent_level\":1,\"severities\":3}", "severities must be text"); await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\",\"agent_level\":\"one\"}", "agent_level must be a whole number"); await Refused("POST", "/api/notification-routes", "{\"metric_match\":\"blocking\",\"agent_level\":1,\"agent_max_per_hour\":1.5}", "agent_max_per_hour must be a whole number"); diff --git a/Darling/Darling.Tests/ViewerNotificationRouteSeveritiesTests.cs b/Darling/Darling.Tests/ViewerNotificationRouteSeveritiesTests.cs new file mode 100644 index 0000000000..bcf9205021 --- /dev/null +++ b/Darling/Darling.Tests/ViewerNotificationRouteSeveritiesTests.cs @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using PerformanceMonitor.Notifications; +using Xunit; + +namespace Darling.Tests; + +/// +/// #6928 (V229): the viewer's part of the route severity filter. The store-version probe maps the new column to 229, the list names +/// the filter in words, and a route's choice round-trips through the viewer's insert, update and sealed save. A store below V229 has no +/// column, so the list still opens and every route reads as Both. +/// +public sealed class ViewerNotificationRouteSeveritiesTests +{ + /// The first sixteen sentinels, all present (the V16 to V31 floor), so only the one named after them decides the answer. + private static int Map(bool hasRouteSeverities) => ViewerDataService.MapProbedSchemaVersion( + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + hasRouteSeverities: hasRouteSeverities); + + [Fact] + public void TheProbeSentinel_MapsTheRouteSeveritiesColumn_ToV229_AndItsAbsence_ToAnOlderVersion() + { + Assert.Equal(229, Map(true)); + Assert.True(Map(false) < 229); + Assert.Equal(229, ViewerDataService.RouteSeveritiesStoreSchemaVersion); + } + + [Theory] + [InlineData("both", "Both")] + [InlineData("critical", "Critical only")] + [InlineData("warning", "Warning only")] + [InlineData("CRITICAL", "Critical only")] + [InlineData("", "Both")] + [InlineData("something-new", "Both")] + public void TheGridWords_NameTheFilter_AndAnUnknownWordReadsAsBoth(string stored, string expected) => + Assert.Equal(expected, new NotificationRouteRow { Severities = stored }.SeveritiesDisplay); + + [Fact] + public void ANewRoute_DefaultsToBoth_AndTheCloneAndTheDraftKeepTheChoice() + { + Assert.Equal(NotificationRouteKinds.SeveritiesBoth, new NotificationRouteRow().Severities); + var row = new NotificationRouteRow { MetricMatch = "performance", SmtpRecipients = "a@example.invalid", Severities = "critical" }; + Assert.Equal("critical", row.Clone().Severities); + Assert.Equal("critical", row.ToDraft().Severities); + Assert.Null(ViewerDataService.ValidateNotificationRoute(row)); + Assert.NotNull(ViewerDataService.ValidateNotificationRoute(new NotificationRouteRow { MetricMatch = "performance", SmtpRecipients = "a@example.invalid", Severities = "major" })); + } + + private static string? BaseConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private static async Task ExecAsync(NpgsqlConnection connection, string sql, System.Threading.CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(ct); + } + + /* #1776 own-store: this class mints its own scratch database, so it is not [Collection("live-postgres")]. */ + [Fact] + public async Task Live_TheSeverityRoundTripsThroughTheViewer_AndAStoreBelowV229StillOpensTheList() + { + var baseConnectionString = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to run the viewer's route severity round trip."); + + var ct = TestContext.Current.CancellationToken; + var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + using var key = PasswordPrivateKey.Generate(); + var bodySucceeded = false; + try + { + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await ExecAsync(connection, "INSERT INTO config.config_service (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + await ExecAsync(connection, "INSERT INTO config.config_notification (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + + await using (var viewer = new ViewerDataService(scratch.ConnectionString)) + { + Assert.True(await viewer.GetStoreSchemaVersionAsync(ct) >= 229); + + var inserted = await viewer.InsertNotificationRouteAsync( + new NotificationRouteRow { MetricMatch = "performance", SmtpRecipients = "a@example.invalid", Severities = "critical" }, ct); + var plain = await viewer.InsertNotificationRouteAsync( + new NotificationRouteRow { MetricMatch = "reports", SmtpRecipients = "b@example.invalid" }, ct); + var sealedId = await viewer.SaveNotificationRouteSealedAsync( + new NotificationRouteRow { MetricMatch = "agent-jobs", SlackUrl = "https://hooks.example.invalid/w", Severities = "warning" }, + null, new NotificationRow(), new ViewerPasswordSealer(key.PublicKey), null, ct); + + var read = await viewer.GetNotificationRoutesAsync(ct); + Assert.Equal("critical", read.Single(r => r.RouteId == inserted).Severities); + Assert.Equal("both", read.Single(r => r.RouteId == plain).Severities); + Assert.Equal("warning", read.Single(r => r.RouteId == sealedId).Severities); + Assert.Equal("Critical only", read.Single(r => r.RouteId == inserted).SeveritiesDisplay); + + var edited = read.Single(r => r.RouteId == inserted).Clone(); + edited.Severities = "warning"; + await viewer.UpdateNotificationRouteAsync(edited, ct); + Assert.Equal("warning", (await viewer.GetNotificationRoutesAsync(ct)).Single(r => r.RouteId == inserted).Severities); + + await Assert.ThrowsAsync(() => viewer.InsertNotificationRouteAsync( + new NotificationRouteRow { MetricMatch = "performance", SmtpRecipients = "c@example.invalid", Severities = "major" }, ct)); + } + + /* A store below V229 has no column: the list still reads, and every route is Both. */ + await ExecAsync(connection, "ALTER TABLE config.config_notification_routes DROP COLUMN severities", ct); + await using (var older = new ViewerDataService(scratch.ConnectionString)) + { + Assert.True(await older.GetStoreSchemaVersionAsync(ct) < 229); + var rows = await older.GetNotificationRoutesAsync(ct); + Assert.Equal(3, rows.Count); + Assert.All(rows, r => Assert.Equal("both", r.Severities)); + } + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async (_, _) => await Task.CompletedTask); + await scratch.DisposeAsync(); + } + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebAgentSetup.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebAgentSetup.cs index b08c41f5d4..d47ae06406 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebAgentSetup.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebAgentSetup.cs @@ -320,6 +320,17 @@ internal static async Task ProfilesBodyAsync(NpgsqlDataSource postgr draft.SendResolutions = kind == JsonValueKind.Null ? null : body["send_resolutions"]!.GetValue(); } + /* #6928: the severity filter. A word other than both, critical or warning is refused by the writer's validation with its sentence. */ + if (body.ContainsKey("severities")) + { + if (body["severities"] is not JsonValue severities || severities.GetValueKind() != JsonValueKind.String) + { + return "severities must be text."; + } + + draft.Severities = severities.GetValue(); + } + if (!TryInt(body, "scope_server_id", out var hasServer, out var server, out var error) || !TryInt(body, "scope_tag_id", out var hasTag, out var tag, out error) || !TryInt(body, "agent_level", out var hasLevel, out var level, out error) diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin-agent-setup.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin-agent-setup.js index 15169de552..469793c803 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin-agent-setup.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin-agent-setup.js @@ -58,7 +58,7 @@ export function entriesFromRoute(r) { if (!r) { return { id: null, match: "", scope: "", enabled: true, level: "", perHour: AGENT_LIMIT_DEFAULTS.perHour, perDay: AGENT_LIMIT_DEFAULTS.perDay, - lease: AGENT_LIMIT_DEFAULTS.lease, expiry: AGENT_LIMIT_DEFAULTS.expiry, resolutions: "", fallback: "", + lease: AGENT_LIMIT_DEFAULTS.lease, expiry: AGENT_LIMIT_DEFAULTS.expiry, resolutions: "", fallback: "", severities: "both", saved: [], hooks: {}, removes: {}, recipients: "", recipientsWas: "", modifiedAt: null, }; } @@ -76,6 +76,8 @@ export function entriesFromRoute(r) { expiry: String(r.agent_expiry_minutes ?? AGENT_LIMIT_DEFAULTS.expiry), resolutions: r.send_resolutions === true ? "true" : r.send_resolutions === false ? "false" : "", fallback: r.fallback_route_id != null ? String(r.fallback_route_id) : "", + /* #6928: a route read without the field, or with a word this page does not know, edits as Both. */ + severities: r.severities === "critical" || r.severities === "warning" ? r.severities : "both", saved: Array.isArray(r.configured_channels) ? r.configured_channels : [], hooks: {}, removes: {}, @@ -138,6 +140,7 @@ export function routePayload(entries) { agent_expiry_minutes: nums[3], send_resolutions: entries.resolutions === "" ? null : entries.resolutions === "true", fallback_route_id: fallback, + severities: entries.severities === "critical" || entries.severities === "warning" ? entries.severities : "both", ...(entries.id != null ? { expected_modified_at: entries.modifiedAt } : {}), }, }; @@ -206,6 +209,7 @@ function routeForm(options) { const controls = { match: text("match", entries.match, { placeholder: "family, 'Analysis: ', exact metric or *" }), scope: select("scope", scopeChoices, entries.scope), + severities: select("severities", [{ value: "both", label: "Both (Warning and Critical)" }, { value: "critical", label: "Critical only" }, { value: "warning", label: "Warning only" }], entries.severities || "both"), level: select("level", AGENT_LEVEL_CHOICES, entries.level), perHour: text("perHour", entries.perHour, { inputmode: "numeric" }), perDay: text("perDay", entries.perDay, { inputmode: "numeric" }), @@ -256,7 +260,7 @@ function routeForm(options) { onClick: () => sendRoute("save", read(), options.repaint), }); const parts = [ - field("Match:", controls.match), field("Applies to:", controls.scope), field("Agent:", controls.level), + field("Match:", controls.match), field("Applies to:", controls.scope), field("Severity:", controls.severities), field("Agent:", controls.level), field("Requests per hour:", controls.perHour), field("Requests per day:", controls.perDay), field("Lease minutes:", controls.lease), field("Expiry minutes:", controls.expiry), field("Send resolutions:", controls.resolutions), field("Fallback route:", controls.fallback), diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js index 0abc05db92..e6a2c7df4d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js @@ -92,6 +92,7 @@ export const ROUTE_COLUMNS = [ { key: "match_kind", label: "Kind" }, { key: "family", label: "Family" }, { key: "applies_to", label: "Applies to" }, + { key: "severities", label: "Severity" }, { key: "status", label: "Status" }, { key: "channels", label: "Channels set", wrap: true }, { key: "agent", label: "Agent" }, @@ -307,6 +308,13 @@ export function routeAgentText(r) { return "Off"; } +/** The Severity column (#6928), in the desktop routes window's words: which alerts the route takes by severity. A route read without + the field, or with a word this page does not know, reads as Both, the same as the service. */ +export function routeSeveritiesText(r) { + const word = r && typeof r.severities === "string" ? r.severities.trim().toLowerCase() : ""; + return word === "critical" ? "Critical only" : word === "warning" ? "Warning only" : "Both"; +} + /** The Send resolutions column (#5825), in the route dialog's words: send_resolutions null follows each channel's own "Send resolutions" setting, true always sends the cleared message to this route's destinations, false never does. A route read without the field reads as the channel setting, the same as null. */ @@ -325,6 +333,7 @@ export function routeRows(data) { match_kind: r.match_kind, family: r.family, applies_to: routeScopeText(r), + severities: routeSeveritiesText(r), status: routeStatusText(r), channels: Array.isArray(r.configured_channels) ? r.configured_channels.filter((c) => !isSecretKey(c)).join(", ") : null, agent: routeAgentText(r), diff --git a/Darling/PerformanceMonitor.Darling.Viewer/NotificationRouteEditDialog.xaml b/Darling/PerformanceMonitor.Darling.Viewer/NotificationRouteEditDialog.xaml index 02430e69e8..f18797d186 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/NotificationRouteEditDialog.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/NotificationRouteEditDialog.xaml @@ -46,9 +46,17 @@ - + + + + + + + + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/NotificationRouteEditDialog.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/NotificationRouteEditDialog.xaml.cs index b8b0ad1de2..65ccf6535a 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/NotificationRouteEditDialog.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/NotificationRouteEditDialog.xaml.cs @@ -33,6 +33,11 @@ public partial class NotificationRouteEditDialog : Window internal const string ResolutionsChannelChoice = "Use channel setting"; internal const string ResolutionsAlwaysChoice = "Always"; internal const string ResolutionsNeverChoice = "Never"; + + /* #6928: the severity filter's three choices, in the order they are offered. */ + internal const string SeveritiesBothChoice = "Both (Warning and Critical)"; + internal const string SeveritiesCriticalChoice = "Critical only"; + internal const string SeveritiesWarningChoice = "Warning only"; private const string AgentOffChoice = "Off"; /* #5912: shown only on a route already stored at Level 2, and never chosen from here (the write path for Act is a later slice). */ private const string AgentActChoice = "Act (Level 2: investigate and change alert settings)"; @@ -94,6 +99,12 @@ public NotificationRouteEditDialog(ViewerRouteScopeInputs? scopeInputs, Notifica var storedResolutions = existing?.SendResolutions; SendResolutionsCombo.SelectedItem = SendResolutionsCombo.Items.Cast().First(i => Equals(i.Tag, storedResolutions)); + SeveritiesCombo.Items.Add(new ComboBoxItem { Content = SeveritiesBothChoice, Tag = NotificationRouteKinds.SeveritiesBoth }); + SeveritiesCombo.Items.Add(new ComboBoxItem { Content = SeveritiesCriticalChoice, Tag = NotificationRouteKinds.SeveritiesCritical }); + SeveritiesCombo.Items.Add(new ComboBoxItem { Content = SeveritiesWarningChoice, Tag = NotificationRouteKinds.SeveritiesWarning }); + var storedSeverities = NotificationRouteKinds.NormalizeSeverities(existing?.Severities); + SeveritiesCombo.SelectedItem = SeveritiesCombo.Items.Cast().First(i => Equals(i.Tag, storedSeverities)); + /* #6036: the fallback picker lists every OTHER route, with a none choice; the route being edited is never offered. */ FallbackRouteCombo.Items.Add(new ComboBoxItem { Content = FallbackNoneChoice, Tag = null }); foreach (var other in (allRoutes ?? System.Linq.Enumerable.Empty()).Where(r => r.RouteId != (existing?.RouteId ?? 0))) @@ -226,6 +237,7 @@ private void Save_Click(object sender, RoutedEventArgs e) Route.PagerDutyRoutingKey = ViewerWebhookSealing.CarryKept(PagerDutyKeyBox.Text, _stored?.PagerDutyRoutingKey); Route.SmtpRecipients = SmtpRecipientsBox.Text.Trim(); Route.Enabled = EnabledCheckBox.IsChecked == true; + Route.Severities = NotificationRouteKinds.NormalizeSeverities((SeveritiesCombo.SelectedItem as ComboBoxItem)?.Tag as string); var error = ReadAgent(); error ??= ViewerDataService.ValidateNotificationRoute(Route); diff --git a/Darling/PerformanceMonitor.Darling.Viewer/NotificationRoutesWindow.xaml b/Darling/PerformanceMonitor.Darling.Viewer/NotificationRoutesWindow.xaml index f1f61049ac..11580d8760 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/NotificationRoutesWindow.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/NotificationRoutesWindow.xaml @@ -52,6 +52,8 @@ + + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.NotificationRoutes.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.NotificationRoutes.cs index 1c339dc20d..a070c3bdd0 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.NotificationRoutes.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.NotificationRoutes.cs @@ -43,21 +43,38 @@ public sealed partial class ViewerDataService projections carry them, so a read-only seat sees which routes send alerts to agents. */ private const string NotificationRouteAgentColumns = NotificationRouteWriter.AgentColumns; - /// The full row, for an admin seat, with the two scope columns (V181, #5750). Ordered by - /// route_id; the window re-sorts the grid newest first. The resolver's order is the scope tier, then the - /// metric level, then the route id, so the id order is no longer the order routes are consulted in. + /* The columns both projections share, so the V229 variants below differ only by the one appended column. */ + private const string NotificationRoutesAdminColumns = + "route_id, metric_match, " + NotificationRouteDestinationColumns + ", enabled, modified_at, configured_channels, scope_server_id, scope_tag_id, " + + NotificationRouteAgentColumns + ", send_resolutions, fallback_route_id"; + + private const string NotificationRoutesNoSecretColumns = + "route_id, metric_match, smtp_recipients, enabled, modified_at, configured_channels, scope_server_id, scope_tag_id, " + + NotificationRouteAgentColumns + ", send_resolutions, fallback_route_id"; + + /// The store version that added config_notification_routes.severities (V229, #6928). A store below it has no column, so + /// the list reads the statements without it and every route shows as Both. + internal const int RouteSeveritiesStoreSchemaVersion = 229; + + /// The full row, for an admin seat, with the two scope columns (V181, #5750), for a store below V229 (no + /// severities column). Ordered by route_id; the window re-sorts the grid newest first. The resolver's order is the + /// scope tier, then the metric level, then the route id, so the id order is no longer the order routes are consulted in. public const string NotificationRoutesSelectSql = - "SELECT route_id, metric_match, " + NotificationRouteDestinationColumns + ", enabled, modified_at, configured_channels, scope_server_id, scope_tag_id, " - + NotificationRouteAgentColumns + ", send_resolutions, fallback_route_id " - + "FROM config_notification_routes ORDER BY route_id"; + "SELECT " + NotificationRoutesAdminColumns + " FROM config_notification_routes ORDER BY route_id"; /// The secret-free projection a read-only viewer seat reads: the four webhook-class /// destinations are omitted (they 42501 for that role), smtp_recipients stays (an address list is /// not a secret, on the parent either), and configured_channels carries presence. public const string NotificationRoutesSelectNoSecretSql = - "SELECT route_id, metric_match, smtp_recipients, enabled, modified_at, configured_channels, scope_server_id, scope_tag_id, " - + NotificationRouteAgentColumns + ", send_resolutions, fallback_route_id " - + "FROM config_notification_routes ORDER BY route_id"; + "SELECT " + NotificationRoutesNoSecretColumns + " FROM config_notification_routes ORDER BY route_id"; + + /// The full row with the severity filter as the last column (V229, #6928), for a store that has it. + public const string NotificationRoutesSelectWithSeveritiesSql = + "SELECT " + NotificationRoutesAdminColumns + ", severities FROM config_notification_routes ORDER BY route_id"; + + /// The secret-free projection with the severity filter as the last column (V229, #6928). The column is not a secret. + public const string NotificationRoutesSelectNoSecretWithSeveritiesSql = + "SELECT " + NotificationRoutesNoSecretColumns + ", severities FROM config_notification_routes ORDER BY route_id"; /// Inserts one route (#6231: the one copy is , which the MCP tools share). public const string NotificationRouteInsertSql = NotificationRouteWriter.InsertSql; @@ -75,12 +92,18 @@ public sealed partial class ViewerDataService public async Task> GetNotificationRoutesAsync(CancellationToken cancellationToken = default) { var rows = new List(); - await using var command = _dataSource.CreateCommand(IsReadOnly ? NotificationRoutesSelectNoSecretSql : NotificationRoutesSelectSql); + /* V229 (#6928): a store below it has no severities column, so the list reads the older statements and every route shows as Both. */ + var version = _cachedStoreSchemaVersion ??= await GetStoreSchemaVersionAsync(cancellationToken); + var hasSeverities = version is int known && known >= RouteSeveritiesStoreSchemaVersion; + var sql = IsReadOnly + ? (hasSeverities ? NotificationRoutesSelectNoSecretWithSeveritiesSql : NotificationRoutesSelectNoSecretSql) + : (hasSeverities ? NotificationRoutesSelectWithSeveritiesSql : NotificationRoutesSelectSql); + await using var command = _dataSource.CreateCommand(sql); command.CommandTimeout = ViewerCommandDeadlines.CurrentInteractiveReadSeconds; await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { - rows.Add(IsReadOnly ? ReadRouteRowNoSecret(reader) : ReadRouteRow(reader)); + rows.Add(IsReadOnly ? ReadRouteRowNoSecret(reader, hasSeverities) : ReadRouteRow(reader, hasSeverities)); } foreach (var row in rows.Where(r => r.FallbackRouteId.HasValue)) @@ -301,7 +324,7 @@ private static void ValidateRoute(NotificationRouteRow row) } } - private static NotificationRouteRow ReadRouteRow(NpgsqlDataReader reader) => new() + private static NotificationRouteRow ReadRouteRow(NpgsqlDataReader reader, bool hasSeverities) => new() { RouteId = reader.GetInt32(0), MetricMatch = reader.GetString(1), @@ -322,11 +345,12 @@ private static void ValidateRoute(NotificationRouteRow row) AgentExpiryMinutes = reader.GetInt32(16), SendResolutions = reader.IsDBNull(17) ? null : reader.GetBoolean(17), FallbackRouteId = reader.IsDBNull(18) ? null : reader.GetInt32(18), + Severities = hasSeverities ? NotificationRouteKinds.NormalizeSeverities(reader.GetString(19)) : NotificationRouteKinds.SeveritiesBoth, }; /// The secret-free reader: the four carved destinations stay empty (the seat cannot author them /// anyway) and every later column shifts ordinal, the ReadNotificationRowNoSecret idiom. - private static NotificationRouteRow ReadRouteRowNoSecret(NpgsqlDataReader reader) => new() + private static NotificationRouteRow ReadRouteRowNoSecret(NpgsqlDataReader reader, bool hasSeverities) => new() { RouteId = reader.GetInt32(0), MetricMatch = reader.GetString(1), @@ -347,6 +371,7 @@ private static void ValidateRoute(NotificationRouteRow row) AgentExpiryMinutes = reader.GetInt32(12), SendResolutions = reader.IsDBNull(13) ? null : reader.GetBoolean(13), FallbackRouteId = reader.IsDBNull(14) ? null : reader.GetInt32(14), + Severities = hasSeverities ? NotificationRouteKinds.NormalizeSeverities(reader.GetString(15)) : NotificationRouteKinds.SeveritiesBoth, }; } @@ -396,6 +421,18 @@ public sealed class NotificationRouteRow /// true always sends it, false never does. public bool? SendResolutions { get; set; } + /// The severity filter (V229, #6928): , + /// or . A route the filter excludes an alert from is skipped and matching goes on. + public string Severities { get; set; } = NotificationRouteKinds.SeveritiesBoth; + + /// The Severity column: Both, Critical only or Warning only (#6928). + public string SeveritiesDisplay => NotificationRouteKinds.NormalizeSeverities(Severities) switch + { + NotificationRouteKinds.SeveritiesCritical => "Critical only", + NotificationRouteKinds.SeveritiesWarning => "Warning only", + _ => "Both", + }; + /// The route an expired investigation request goes to first (#6036, D8 of #5632 Phase C), or null for none. /// Never the route itself; the delete clears it when that route is deleted. public int? FallbackRouteId { get; set; } @@ -487,6 +524,7 @@ public string ChannelsDisplay AgentExpiryMinutes = AgentExpiryMinutes, SendResolutions = SendResolutions, FallbackRouteId = FallbackRouteId, + Severities = NotificationRouteKinds.NormalizeSeverities(Severities), ModifiedAtUtc = ModifiedAtUtc == default ? null : ModifiedAtUtc, }; @@ -512,6 +550,7 @@ public string ChannelsDisplay SendResolutions = SendResolutions, FallbackRouteId = FallbackRouteId, FallbackRouteName = FallbackRouteName, + Severities = Severities, ScopeDisplay = ScopeDisplay, StatusDisplay = StatusDisplay, }; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs index 7b42e0ab90..857e98ffb8 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs @@ -1377,7 +1377,10 @@ per the V71 finding. */ EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds' AND NOT attisdropped), /* V226 (#6893) adds the AWS RDS Web or Express fact to the server registry. Found by catalog like the V225 column above. Named only in this probe line, never in prose, per the V71 finding. */ - EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped)"; + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped), + /* V229 (#6928) adds the severity filter to the notification routes. Found by catalog like the V225 column above. Named only in + this probe line, never in prose, per the V71 finding. */ + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.config_notification_routes') AND attname = 'severities' AND NOT attisdropped)"; /// The store schema version this viewer build requires — the highest migration it knows /// (). The connect-time gate blocks a store below this. @@ -1399,7 +1402,7 @@ per the V71 finding. */ await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201)); + return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201), reader.GetBoolean(202)); } return null; @@ -1424,7 +1427,7 @@ per the V71 finding. */ /// is unit-tested without a live store; any schema bump past the newest arm trips the pinning test that keeps /// this in step with . /// - internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false) + internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false, bool hasRouteSeverities = false) { /* V71 (the PostgreSQL blocking-edges rung): a table-existence sentinel and now the newest-first arm. A collector table would ordinarily get no arm at all — see the V63-V69 note below — but the TOP @@ -1633,6 +1636,13 @@ store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through t /* V226 (#6893): the AWS RDS Web or Express fact on the server registry. Column-existence sentinel, newest-first, and the TOP rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 225 and showing a spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + /* V229 (#6928): the severity filter on the notification routes. Column-existence sentinel, newest-first, and the TOP rung, so a + fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 226 and showing a spurious upgrade + banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + if (hasRouteSeverities) + { + return 229; + } if (hasServerAwsRdsWebOrExpress) { return 226; From 12316d3642f3135d928d2e73c6560e7db8b472b6 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:10:06 -0400 Subject: [PATCH 09/22] Accept severities on the web route write and keep a bad word visible to validation (part of #6928) --- .../PerformanceMonitor.Darling.Service/DarlingWebAgentSetup.cs | 2 +- .../ViewerDataService.NotificationRoutes.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebAgentSetup.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebAgentSetup.cs index d47ae06406..2251913a69 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebAgentSetup.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebAgentSetup.cs @@ -56,7 +56,7 @@ internal static string RefusedMessage(bool managedStore, bool adminWritesOff = f private static readonly string[] s_routeKeys = [ "metric_match", "enabled", "scope_server_id", "scope_tag_id", "agent_level", "agent_max_per_hour", "agent_max_per_day", - "agent_lease_minutes", "agent_expiry_minutes", "send_resolutions", "fallback_route_id", + "agent_lease_minutes", "agent_expiry_minutes", "send_resolutions", "fallback_route_id", "severities", "teams_webhook_url", "slack_webhook_url", "generic_webhook_url", "pagerduty_routing_key", "email_recipients", "expected_modified_at", ]; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.NotificationRoutes.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.NotificationRoutes.cs index a070c3bdd0..6b64f53101 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.NotificationRoutes.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.NotificationRoutes.cs @@ -524,7 +524,7 @@ public string ChannelsDisplay AgentExpiryMinutes = AgentExpiryMinutes, SendResolutions = SendResolutions, FallbackRouteId = FallbackRouteId, - Severities = NotificationRouteKinds.NormalizeSeverities(Severities), + Severities = Severities, ModifiedAtUtc = ModifiedAtUtc == default ? null : ModifiedAtUtc, }; From ce38ff8ee05e21173b63a015c87e6acb65741a57 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:02:38 -0400 Subject: [PATCH 10/22] Test and document the route severity filter (part of #6928) Router, tier rule, the three-channel example through the deliverer, resolution routing, the PagerDuty close severity, MCP round trip and V229 on a store with routes. Docs: alerts and notifications, MCP configuration, rollback line for V229. --- .../AlertResolutionDeliveryTests.cs | 29 +++ .../ExternalAlertDrainLiveTests.cs | 30 ++- .../NotificationRouteSeverityLiveTests.cs | 222 ++++++++++++++++++ .../NotificationRouteSeverityTests.cs | 193 +++++++++++++++ docs/darling/admin.md | 1 + docs/darling/alerts-and-notifications.md | 13 + docs/darling/configuration-mcp.md | 4 +- 7 files changed, 487 insertions(+), 5 deletions(-) create mode 100644 Darling/Darling.Tests/NotificationRouteSeverityLiveTests.cs create mode 100644 Darling/Darling.Tests/NotificationRouteSeverityTests.cs diff --git a/Darling/Darling.Tests/AlertResolutionDeliveryTests.cs b/Darling/Darling.Tests/AlertResolutionDeliveryTests.cs index 3b906a8ebf..93028e2317 100644 --- a/Darling/Darling.Tests/AlertResolutionDeliveryTests.cs +++ b/Darling/Darling.Tests/AlertResolutionDeliveryTests.cs @@ -365,6 +365,35 @@ public async Task ASecondClearWithNoFireBetween_SendsNothing() Assert.Equal(AlertDelivery.ChannelNotApplicable, rig.History.Records[^1].NotificationType); } + /// #6928: a clear goes to the channels the fire reached, not to wherever the severity filter would send it now. The fire + /// was Critical and went to the Critical-only route's endpoint. The routes' filters are then swapped (the other endpoint's route has the lower id, so a re-match at the clear would pick it), and the clear still goes to + /// that endpoint and never to the other. + [Fact] + public async Task AClear_GoesToTheChannelTheFireReached_WhateverTheSeverityFiltersSayAtTheClear() + { + using var criticalEndpoint = new CapturingWebhookEndpoint(); + using var warningEndpoint = new CapturingWebhookEndpoint(); + var config = new DarlingConfig(); + config.Webhooks.SlackSendResolutions = true; + NotificationRoute[] RoutesWith(string first, string second) => new[] + { + new NotificationRoute(2, Metric, "", criticalEndpoint.Url, "", "", "", true) { Severities = first }, + new NotificationRoute(1, Metric, "", warningEndpoint.Url, "", "", "", true) { Severities = second }, + }; + config.NotificationRoutes = RoutesWith("critical", "warning"); + var rig = Rig.Build(config); + + await rig.Deliverer.DeliverAsync(Fire() with { Severity = AlertSeverityLevel.Critical }, TestContext.Current.CancellationToken); + Assert.Single(criticalEndpoint.Bodies); + Assert.Empty(warningEndpoint.Bodies); + + config.NotificationRoutes = RoutesWith("warning", "critical"); + await rig.Deliverer.DeliverResolutionAsync(CpuResolution(), cancellationToken: TestContext.Current.CancellationToken); + + Assert.Equal(2, criticalEndpoint.Bodies.Count); + Assert.Empty(warningEndpoint.Bodies); + } + private static AlertOutcome Fire() => new(ServerKey, Server, Metric, "97%", "90%", Context: null, DetailText: null, NumericCurrentValue: 97, NumericThresholdValue: 90, Muted: false, Severity: null); diff --git a/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs b/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs index f235c3f532..89fb1ae2e4 100644 --- a/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs +++ b/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs @@ -218,6 +218,25 @@ await WithDrainAsync(async rig => }); } + /// #6928: the PagerDuty close is routed at the severity the fire went out at, so a severity filter on a route sends the close + /// to the route the fire used. A critical fire closes as Critical and a warning fire as Warning. + [Theory] + [InlineData("critical", AlertSeverityLevel.Critical)] + [InlineData("warning", AlertSeverityLevel.Warning)] + public async Task TheClear_AsksForThePagerDutyCloseAtTheSeverityTheFireWentOutAt(string fireSeverity, AlertSeverityLevel expected) + { + await WithDrainAsync(async rig => + { + await rig.SubmitAsync("fire", "disk", server: "SQL01", severity: fireSeverity); + await rig.Drain.RunPassAsync(rig.Ct); + var clear = await rig.SubmitAsync("clear", "disk", server: "SQL01"); + await rig.Drain.RunPassAsync(rig.Ct); + + Assert.Equal("delivered", await rig.StatusAsync(clear)); + Assert.Equal(new AlertSeverityLevel?[] { expected }, rig.PagerDutySeverities.ToArray()); + }); + } + [Fact] public async Task AClearWithNoFire_IsIgnored_AndAsksForNoResolve() { @@ -577,9 +596,10 @@ await History.RecordAlertAsync(new AlertHistoryRecord( AlertContextSerializer.Serialize(outcome.Context!))); return delivery; }, - (metric, serverName, serverKey, _, _, _) => + (metric, serverName, serverKey, _, firedSeverity, _) => { PagerDutyResolves.Add(metric.Split(':')[2]); + PagerDutySeverities.Add(firedSeverity); PagerDutyIdentities.Add(serverName + "|" + serverKey); return Task.FromResult(AlertChannelOutcome.Delivered); }, @@ -598,6 +618,9 @@ await History.RecordAlertAsync(new AlertHistoryRecord( public List Sent { get; } = new(); public List PagerDutyResolves { get; } = new(); + /// The severity each PagerDuty resolve was asked to route at (#6928): the tier the fire went out at. + public List PagerDutySeverities { get; } = new(); + /// The server name and server key each PagerDuty resolve was asked to close, as name|key. public List PagerDutyIdentities { get; } = new(); @@ -618,16 +641,17 @@ public async Task AddSourceAsync(string name) => await ScalarAsync($"INSERT INTO config.external_alert_source (source_name, role_name) VALUES ('{name}', 'role_{name}') RETURNING source_id"), CultureInfo.InvariantCulture); - public async Task SubmitAsync(string kind, string key, string? server = null, double? value = null, string source = "nagios") + public async Task SubmitAsync(string kind, string key, string? server = null, double? value = null, string source = "nagios", string severity = "warning") { await using var command = new NpgsqlCommand( "INSERT INTO config.external_alert_pending (source_id, kind, alert_key, severity, title, server_name, value, expires_at)" - + " VALUES ($1, $2, $3, 'warning', 'a title', $4, $5, now() + interval '15 minutes') RETURNING pending_id", _seed); + + " VALUES ($1, $2, $3, $6, 'a title', $4, $5, now() + interval '15 minutes') RETURNING pending_id", _seed); command.Parameters.Add(new NpgsqlParameter { TypedValue = _sourceIds[source] }); command.Parameters.Add(new NpgsqlParameter { TypedValue = kind }); command.Parameters.Add(new NpgsqlParameter { TypedValue = key }); command.Parameters.Add(new NpgsqlParameter { Value = (object?)server ?? DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Text }); command.Parameters.Add(new NpgsqlParameter { Value = (object?)value ?? DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Double }); + command.Parameters.Add(new NpgsqlParameter { TypedValue = severity }); return Convert.ToInt64(await command.ExecuteScalarAsync(Ct), CultureInfo.InvariantCulture); } diff --git a/Darling/Darling.Tests/NotificationRouteSeverityLiveTests.cs b/Darling/Darling.Tests/NotificationRouteSeverityLiveTests.cs new file mode 100644 index 0000000000..948d25601e --- /dev/null +++ b/Darling/Darling.Tests/NotificationRouteSeverityLiveTests.cs @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/* #1776 own-store: every fact mints its own scratch database (and the MCP facts a role with a per-run name), so the class is deliberately + NOT [Collection("live-postgres")]. */ + +/// +/// #6928: the route severity filter against a live store. V229 on a store that already holds routes (they read both, the +/// constraint holds the three words, and applying the rung again changes nothing), and the MCP create, update and get round trip +/// AS the mcp role with the grants taken from the managed batch, including the invalid answer for a word that is +/// not critical, warning or both. +/// +public sealed class NotificationRouteSeverityLiveTests +{ + private static readonly string RolePassword = Convert.ToHexString(RandomNumberGenerator.GetBytes(16)); + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private static DarlingMcpServedPolicy OperatorPolicy() => new(new DarlingMcpWritePolicy.Effective(true, false, false, "ops-lead")); + + private static string Status(string answer) => JsonDocument.Parse(answer).RootElement.GetProperty("status").GetString()!; + + private static int RouteIdOf(string answer) => + JsonDocument.Parse(answer).RootElement.GetProperty("route").GetProperty("route_id").GetInt32(); + + [Fact] + public async Task V229_OnAStoreWithExistingRoutes_ReadsBoth_HoldsTheThreeWords_AndAppliesTwiceWithNoChange() + { + Assert.SkipWhen(string.IsNullOrEmpty(ConnectionString), "Set DARLING_TEST_PG to run the V229 live pin (it mints its own scratch database)."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(ConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + var bodySucceeded = false; + try + { + /* A store one rung behind: V229's constraint and column gone, two routes stored (one scoped, one not). */ + await ExecAsync(connection, + "ALTER TABLE config.config_notification_routes DROP CONSTRAINT ck_config_notification_routes_severities, DROP COLUMN severities", ct); + await ExecAsync(connection, "DELETE FROM darling_schema_version WHERE version >= 229", ct); + await ExecAsync(connection, + "INSERT INTO config.config_notification_routes (metric_match, slack_url) VALUES ('Deadlocks Detected', 'sealed-a')", ct); + await ExecAsync(connection, + "INSERT INTO config.config_notification_routes (metric_match, slack_url, scope_server_id) VALUES ('High CPU', 'sealed-b', 11)", ct); + + await PgMigrations.MigrateAsync(connection, ct); + + Assert.Equal("both|both", await ScalarAsync(connection, + "SELECT string_agg(severities, '|' ORDER BY route_id) FROM config.config_notification_routes", ct)); + + /* The three words are accepted and anything else is refused by the CHECK. */ + foreach (var word in new[] { "critical", "warning", "both" }) + { + await ExecAsync(connection, $"UPDATE config.config_notification_routes SET severities = '{word}' WHERE metric_match = 'Deadlocks Detected'", ct); + } + + var refused = await Assert.ThrowsAsync(() => ExecAsync(connection, + "UPDATE config.config_notification_routes SET severities = 'urgent' WHERE metric_match = 'Deadlocks Detected'", ct)); + Assert.Equal("23514", refused.SqlState); + var nullRefused = await Assert.ThrowsAsync(() => ExecAsync(connection, + "UPDATE config.config_notification_routes SET severities = NULL WHERE metric_match = 'Deadlocks Detected'", ct)); + Assert.Equal("23502", nullRefused.SqlState); + + /* The rung applied again changes nothing: the stored word survives and the constraint is not added twice. */ + await ExecAsync(connection, "UPDATE config.config_notification_routes SET severities = 'critical' WHERE metric_match = 'High CPU'", ct); + await ExecAsync(connection, "DELETE FROM darling_schema_version WHERE version >= 229", ct); + await PgMigrations.MigrateAsync(connection, ct); + Assert.Equal("both|critical", await ScalarAsync(connection, + "SELECT string_agg(severities, '|' ORDER BY route_id) FROM config.config_notification_routes", ct)); + Assert.Equal("1", await ScalarAsync(connection, + "SELECT count(*) FROM pg_constraint WHERE conname = 'ck_config_notification_routes_severities'", ct)); + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, static (_, _) => Task.CompletedTask); + } + } + + [Fact] + public async Task Mcp_CreateUpdateAndGet_RoundTripTheSeverityFilter_AndABadWordIsInvalid() + { + await WithMcpRoleAsync(async (owner, asMcp, ct) => + { + /* Left out, a new route takes both. */ + var plain = await DarlingMcpAlertTools.CreateNotificationRoute(asMcp, "Blocking", served: OperatorPolicy(), cancellationToken: ct); + Assert.Equal("created", Status(plain)); + Assert.Equal("both", await ScalarAsync(owner, $"SELECT severities FROM config.config_notification_routes WHERE route_id = {RouteIdOf(plain)}", ct)); + + /* Any case is accepted and stored lower. */ + var created = await DarlingMcpAlertTools.CreateNotificationRoute( + asMcp, "Deadlocks", severities: "Critical", served: OperatorPolicy(), cancellationToken: ct); + Assert.Equal("created", Status(created)); + var routeId = RouteIdOf(created); + Assert.Equal("critical", await ScalarAsync(owner, $"SELECT severities FROM config.config_notification_routes WHERE route_id = {routeId}", ct)); + Assert.Equal("critical", await ReadSeveritiesAsync(asMcp, routeId, ct)); + + /* An update changes it, and a field left out keeps it. */ + Assert.Equal("updated", Status(await DarlingMcpAlertTools.UpdateNotificationRoute( + asMcp, routeId, severities: "warning", served: OperatorPolicy(), cancellationToken: ct))); + Assert.Equal("warning", await ReadSeveritiesAsync(asMcp, routeId, ct)); + Assert.Equal("updated", Status(await DarlingMcpAlertTools.UpdateNotificationRoute( + asMcp, routeId, enabled: false, served: OperatorPolicy(), cancellationToken: ct))); + Assert.Equal("warning", await ReadSeveritiesAsync(asMcp, routeId, ct)); + + /* A bad word is invalid on both tools, and nothing changes or is stored. */ + var badCreate = await DarlingMcpAlertTools.CreateNotificationRoute( + asMcp, "Blocking Wait", severities: "urgent", served: OperatorPolicy(), cancellationToken: ct); + Assert.Equal("invalid", Status(badCreate)); + Assert.Contains("severities is critical, warning or both.", badCreate, StringComparison.Ordinal); + Assert.Equal("2", await ScalarAsync(owner, "SELECT count(*) FROM config.config_notification_routes", ct)); + Assert.Equal("invalid", Status(await DarlingMcpAlertTools.UpdateNotificationRoute( + asMcp, routeId, severities: "urgent", served: OperatorPolicy(), cancellationToken: ct))); + Assert.Equal("warning", await ScalarAsync(owner, $"SELECT severities FROM config.config_notification_routes WHERE route_id = {routeId}", ct)); + }); + } + + private static async Task ReadSeveritiesAsync(NpgsqlDataSource source, int routeId, CancellationToken ct) => + JsonDocument.Parse(await DarlingMcpAlertTools.GetNotificationRoutes(source, ct)).RootElement + .GetProperty("routes").EnumerateArray().Single(r => r.GetProperty("route_id").GetInt32() == routeId) + .GetProperty("severities").GetString()!; + + private static async Task WithMcpRoleAsync(Func body) + { + Assert.SkipWhen(string.IsNullOrEmpty(ConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the route severity live tests."); + + var ct = TestContext.Current.CancellationToken; + var roleName = "route_sev_" + Guid.NewGuid().ToString("N")[..8]; + await using var scratch = await ScratchPostgres.CreateAsync(ConnectionString!, ct); + await using (var connection = new NpgsqlConnection(scratch.ConnectionString)) + { + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + } + + await using var owner = NpgsqlDataSource.Create(scratch.ConnectionString); + var bodySucceeded = false; + try + { + /* The functions and the journal trigger are created by the owner, as the managed batch does; the batch's own grants, retargeted. */ + await ExecAsync(owner, DarlingManagedRoles.BuildNotificationRouteFunctionsSql("config"), ct); + await ExecAsync(owner, DarlingManagedRoles.BuildJournalEventRulesSql("config"), ct); + await ExecAsync(owner, $"CREATE ROLE {roleName} LOGIN NOSUPERUSER PASSWORD '{RolePassword}'", ct); + foreach (var statement in ViewerGrantReplay.StatementsFor(roleName, scratch.DatabaseName, "mcp")) + { + await ExecAsync(owner, statement, ct); + } + + var executes = ViewerGrantReplay.BatchStatements().Where(s => + s.StartsWith("GRANT EXECUTE ON FUNCTION config.create_notification_route", StringComparison.Ordinal) + || s.StartsWith("GRANT EXECUTE ON FUNCTION config.update_notification_route", StringComparison.Ordinal) + || s.StartsWith("GRANT EXECUTE ON FUNCTION config.set_notification_route_agent", StringComparison.Ordinal)).ToList(); + Assert.Equal(3, executes.Count); + foreach (var execute in executes) + { + var at = execute.LastIndexOf(" TO ", StringComparison.Ordinal); + Assert.Equal("mcp", execute[(at + 4)..].Trim()); + await ExecAsync(owner, execute[..at] + " TO " + roleName, ct); + } + + await using var asMcp = NpgsqlDataSource.Create(new NpgsqlConnectionStringBuilder(scratch.ConnectionString) + { + Username = roleName, + Password = RolePassword, + Pooling = false, + }.ConnectionString); + + await body(owner, asMcp, ct); + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunOwnedAsync(bodySucceeded, () => ExecAsync(owner, $"DROP OWNED BY {roleName}; DROP ROLE IF EXISTS {roleName};", CancellationToken.None)); + } + } + + private static async Task ExecAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task ExecAsync(NpgsqlDataSource source, string sql, CancellationToken ct) + { + await using var command = source.CreateCommand(sql); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + return Convert.ToString(await command.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); + } + + private static async Task ScalarAsync(NpgsqlDataSource source, string sql, CancellationToken ct) + { + await using var command = source.CreateCommand(sql); + return Convert.ToString(await command.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); + } +} diff --git a/Darling/Darling.Tests/NotificationRouteSeverityTests.cs b/Darling/Darling.Tests/NotificationRouteSeverityTests.cs new file mode 100644 index 0000000000..043dea4ca6 --- /dev/null +++ b/Darling/Darling.Tests/NotificationRouteSeverityTests.cs @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Notifications; +using Xunit; + +namespace Darling.Tests; + +/// +/// #6928: a notification route can be limited by severity (Critical only, Warning only, or both). These pin the router's +/// filter (default both, an excluded severity falls through to the next route and then the defaults), the tier rule the send +/// uses, the issue's three-channel example driven through the deliverer to loopback endpoints, and the PagerDuty close +/// riding the severity the fire went out at. +/// +public sealed class NotificationRouteSeverityTests +{ + private const string Metric = "High CPU"; + + private static NotificationRoute Route(int id, string match, string severities = "both", string slack = "", string teams = "", + string generic = "", string pagerDuty = "") => + new(id, match, teams, slack, generic, pagerDuty, "", true) { Severities = severities }; + + private static DarlingAlertSettings Defaults(string pagerDutyKey = "") + { + var config = new DarlingConfig(); + config.Webhooks.PagerDutyRoutingKey = pagerDutyKey; + return new DarlingAlertSettings(config); + } + + /// A route nobody gave a filter to reads both, so every route stored before the filter keeps its effect. + [Fact] + public void ARouteWithNoSeverityFilter_ReadsBoth_AndTakesEveryTier() + { + var route = new NotificationRoute(1, Metric, "", "https://slack.example.invalid/a", "", "", "", true); + + Assert.Equal("both", route.Severities); + Assert.True(route.AcceptsSeverity(AlertSeverityLevel.Critical)); + Assert.True(route.AcceptsSeverity(AlertSeverityLevel.Warning)); + Assert.True(route.AcceptsSeverity(null)); + + var decision = NotificationRouter.Resolve(Metric, new[] { route }, Defaults(), severity: AlertSeverityLevel.Warning); + Assert.Equal("https://slack.example.invalid/a", decision.Slack.Destination); + Assert.Equal("both", NotificationRouteKinds.NormalizeSeverities(null)); + Assert.Equal("both", NotificationRouteKinds.NormalizeSeverities("urgent")); + Assert.Equal("critical", NotificationRouteKinds.NormalizeSeverities(" Critical ")); + } + + /// The issue's example: a Critical-only channel, a Warning-only channel and a both channel. Each alert lands on its own. + [Fact] + public void TheThreeChannelExample_EachAlertLandsWhereItShould() + { + var routes = new[] + { + Route(1, Metric, "critical", slack: "https://slack.example.invalid/critical"), + Route(2, Metric, "warning", teams: "https://teams.example.invalid/warning"), + Route(3, Metric, "both", generic: "https://generic.example.invalid/both"), + }; + + var critical = NotificationRouter.Resolve(Metric, routes, Defaults(), severity: AlertSeverityLevel.Critical); + Assert.Equal("https://slack.example.invalid/critical", critical.Slack.Destination); + Assert.Null(critical.Teams.Destination); + Assert.Equal("https://generic.example.invalid/both", critical.Generic.Destination); + + var warning = NotificationRouter.Resolve(Metric, routes, Defaults(), severity: AlertSeverityLevel.Warning); + Assert.Null(warning.Slack.Destination); + Assert.Equal("https://teams.example.invalid/warning", warning.Teams.Destination); + Assert.Equal("https://generic.example.invalid/both", warning.Generic.Destination); + } + + /// A route whose filter excludes the alert is skipped: the next matching route answers, and when none does the defaults do. + [Fact] + public void AnExcludedSeverity_FallsThroughToTheNextRoute_ThenToTheDefaults() + { + var exactCritical = Route(1, Metric, "critical", pagerDuty: "rk-exact-critical"); + var familyBoth = Route(2, AlertFamily.Of(Metric), "both", pagerDuty: "rk-family"); + var settings = Defaults("rk-default"); + + Assert.Equal("rk-exact-critical", + NotificationRouter.Resolve(Metric, new[] { exactCritical, familyBoth }, settings, severity: AlertSeverityLevel.Critical).PagerDuty.Destination); + + var next = NotificationRouter.Resolve(Metric, new[] { exactCritical, familyBoth }, settings, severity: AlertSeverityLevel.Warning); + Assert.Equal("rk-family", next.PagerDuty.Destination); + Assert.Equal(RouteSource.Family, next.PagerDuty.Source); + + var defaulted = NotificationRouter.Resolve(Metric, new[] { exactCritical }, settings, severity: AlertSeverityLevel.Warning); + Assert.Equal("rk-default", defaulted.PagerDuty.Destination); + Assert.Equal(RouteSource.Default, defaulted.PagerDuty.Source); + Assert.True(defaulted.IsAllDefaults); + + /* A caller that does not know the tier is taken by every route, which is how Lite and the agent gate behave. */ + Assert.Equal("rk-exact-critical", NotificationRouter.Resolve(Metric, new[] { exactCritical, familyBoth }, settings).PagerDuty.Destination); + } + + /// The agent route search drops an excluded route too, so a Warning alert does not open an investigation under a Critical-only route. + [Fact] + public void ResolveAgent_SkipsARouteWhoseFilterExcludesTheAlert() + { + var criticalAgent = Route(1, Metric, "critical") with { AgentLevel = NotificationRoute.AgentLevelInvestigate }; + + Assert.Same(criticalAgent, NotificationRouter.ResolveAgent(Metric, new[] { criticalAgent }, severity: AlertSeverityLevel.Critical)); + Assert.Null(NotificationRouter.ResolveAgent(Metric, new[] { criticalAgent }, severity: AlertSeverityLevel.Warning)); + } + + /// The tier the send gives a route filter: the fire site's tier when it carries one, else the badge (red is Critical, every other badge Warning). + [Theory] + [InlineData("Capture Down", null, AlertSeverityLevel.Critical)] // red badge + [InlineData("High CPU", null, AlertSeverityLevel.Warning)] // amber badge + [InlineData("Blocking Detected", null, AlertSeverityLevel.Warning)] // ALERT badge + [InlineData("A Metric Nobody Mapped", null, AlertSeverityLevel.Warning)] // INFO badge + [InlineData("Capture Down", AlertSeverityLevel.Warning, AlertSeverityLevel.Warning)] // fire site beats a red badge + [InlineData("High CPU", AlertSeverityLevel.Critical, AlertSeverityLevel.Critical)] // fire site beats an amber badge + public void TheTier_IsTheFireSitesElseTheBadge(string metric, AlertSeverityLevel? fireSite, AlertSeverityLevel expected) => + Assert.Equal(expected, AlertSeverity.RouteLevelOf(metric, fireSite)); + + /// The issue's three channels driven through the deliverer: a Critical alert and a Warning alert reach the loopback endpoints their routes name. + [Fact] + public async Task TheThreeChannelExample_ThroughTheDeliverer_ReachesTheEndpointsEachAlertBelongsTo() + { + using var criticalSlack = new CapturingWebhookEndpoint(); + using var warningTeams = new CapturingWebhookEndpoint(); + using var bothGeneric = new CapturingWebhookEndpoint(); + var config = new DarlingConfig(); + config.NotificationRoutes = new[] + { + Route(1, Metric, "critical", slack: criticalSlack.Url), + Route(2, Metric, "warning", teams: warningTeams.Url), + Route(3, Metric, "both", generic: bothGeneric.Url), + }; + var settings = new DarlingAlertSettings(config); + var history = new DiscardingHistoryStore(); + var webhooks = new WebhookAlertService(settings, DarlingAlertDeliverer.Branding, NullLogger.Instance, history); + var deliverer = new DarlingAlertDeliverer(settings, history, webhooks, NullLogger.Instance); + + /* Two servers, so the two fires are not folded by the per-server cooldown. */ + await deliverer.DeliverAsync(Fire("7", "CRIT-SRV", AlertSeverityLevel.Critical), TestContext.Current.CancellationToken); + await deliverer.DeliverAsync(Fire("8", "WARN-SRV", AlertSeverityLevel.Warning), TestContext.Current.CancellationToken); + + Assert.Contains("CRIT-SRV", Assert.Single(criticalSlack.Bodies), StringComparison.Ordinal); + Assert.Contains("WARN-SRV", Assert.Single(warningTeams.Bodies), StringComparison.Ordinal); + Assert.Equal(2, bothGeneric.Bodies.Count); + Assert.Contains(bothGeneric.Bodies, b => b.Contains("CRIT-SRV", StringComparison.Ordinal)); + Assert.Contains(bothGeneric.Bodies, b => b.Contains("WARN-SRV", StringComparison.Ordinal)); + } + + /// The PagerDuty close routes at the severity the fire went out at: each tier's close reaches the key its fire's route carried. + [Theory] + [InlineData(AlertSeverityLevel.Critical, "rk-critical")] + [InlineData(AlertSeverityLevel.Warning, "rk-warning")] + public async Task ThePagerDutyClose_UsesTheFiresSeverity_ToPickTheKey(AlertSeverityLevel firedSeverity, string expectedKey) + { + var config = new DarlingConfig(); + config.Webhooks.PagerDutyAutoResolve = true; + config.NotificationRoutes = new[] + { + Route(1, Metric, "critical", pagerDuty: "rk-critical"), + Route(2, Metric, "warning", pagerDuty: "rk-warning"), + }; + var settings = new DarlingAlertSettings(config); + var posted = new List(); + var webhooks = new WebhookAlertService(settings, DarlingAlertDeliverer.Branding, NullLogger.Instance, new DiscardingHistoryStore()) + { + PostPagerDutyAsyncOverride = (_, payload) => + { + posted.Add(payload); + return Task.FromResult(null); + }, + }; + + var outcome = await webhooks.TrySendPagerDutyResolveAsync( + Metric, "SQL01", "7", "back to normal", firedSeverity, TestContext.Current.CancellationToken); + + Assert.Equal(AlertChannelOutcome.Delivered, outcome); + var body = JsonDocument.Parse(Assert.Single(posted)).RootElement; + Assert.Equal(expectedKey, body.GetProperty("routing_key").GetString()); + Assert.Equal("resolve", body.GetProperty("event_action").GetString()); + } + + private static AlertOutcome Fire(string serverKey, string server, AlertSeverityLevel severity) => + new(serverKey, server, Metric, "97%", "90%", Context: null, DetailText: null, + NumericCurrentValue: 97, NumericThresholdValue: 90, Muted: false, Severity: severity); +} diff --git a/docs/darling/admin.md b/docs/darling/admin.md index df819a8722..09dd9eefbb 100644 --- a/docs/darling/admin.md +++ b/docs/darling/admin.md @@ -189,6 +189,7 @@ An older build ignores what those migrations added, so it can read some newer da - **V219**: an older build does not know the Query Store self-statement keys ([#6750](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6750)). Rolling back leaves `collect.query_store_self_keys`, `collect.query_store_self_keys_state` and the two triggers on `collect.query_store_text` in place. The older build ignores the tables, and the triggers keep the keys current whenever it writes statement text, so when you roll forward again the Query Store regressions read still hides the monitor's own statement. Nothing needs to be done before you roll back. - **V220**: an older build does not know the Job History statistics index ([#6798](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6798)). Rolling back leaves `idx_job_history_job_success` on `collect.job_history` in place. The older build's Job History read can still use it, and the collector keeps it current on every insert. Nothing needs doing before you roll back. Leave the index in place: the viewer recognizes a V220 store by it, so a store whose index was dropped answers 219 to the viewer while its schema version says 220, the viewer refuses it, and the migrator does not build the index again. - **V221**: an older build does not know the Query Store hash-hourly table ([#6800](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6800)). Rolling back leaves `collect.query_store_hash_hourly`, the day tables `collect.query_store_hash_daily`, `collect.query_store_hash_daily_built` and `collect.query_store_hash_daily_failures`, and the `hash_built_seq` column of `collect.query_store_compose_stamp_built` in place. The older build neither reads nor fills them, so the tables stop growing and their hours and days go stale. Nothing to do before rolling back. If you roll forward again, the next hourly passes rebuild the stale hours, newest first, and the next cleanup removes the hash-hourly rows of any hour the older build retired without deleting them (it deletes by the retention floor, not by the hours it just retired) +- **V229**: an older build does not know the severity filter on a notification route ([#6928](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6928)). Rolling back leaves `config.config_notification_routes.severities` in place. The older build ignores it. A route set to `critical` or `warning` takes alerts of both severities again. A Warning alert then reaches the channels of a route that was meant for Critical alerts only. Before you roll back, run `get_notification_routes` and look at every route whose `severities` is not `both`. If its channels must not see the other severity, change the route's destinations or disable it. A route set to `both` is unchanged. If you roll forward again, the filters you set are still there. ### Upgrading from a build that predates the script diff --git a/docs/darling/alerts-and-notifications.md b/docs/darling/alerts-and-notifications.md index 7e4e11ef67..087c30b7eb 100644 --- a/docs/darling/alerts-and-notifications.md +++ b/docs/darling/alerts-and-notifications.md @@ -277,6 +277,19 @@ A bring-your-own store that was provisioned before 3.12 needs the same step for Destinations are bearer secrets, so the `viewer` and `mcp` roles never read a webhook URL or key: the table's generated `configured_channels` column says *which* channels a route sets, and the MCP surface administers routes in full: `create_notification_route` makes a route (with no destination of its own, each channel goes where it went before, to the next matching route and then to the channels configured in Settings), `update_notification_route` changes a route's match, scope, enabled flag, resolution override and fallback, and both set a route's own destinations (a Teams, Slack or generic webhook URL, a PagerDuty routing key, email recipients; `-` removes one). The service seals each webhook value for the route's own row, in the transaction that creates or edits the route, exactly as the Viewer does, so a value copied to another route does not open there; an answer never shows a webhook URL or key: it says whether one is saved (with the host for Teams and Slack only), it shows the recipient list in full, and the undo of a destination change carries the earlier recipient list and says to pass an earlier webhook URL or key again. `set_notification_route_enabled`, `set_notification_route_fallback` and `delete_notification_route` change the rest. A route with an explicit agent setting (Off included) keeps its match and scope in the Viewer, and an agent route is changed only in the Viewer. The `mcp` role has no INSERT on the table, no write on the destination columns and no read of the webhook and key columns (it reads the recipient list and `configured_channels`); the route tools write through `config.create_notification_route`, `config.update_notification_route` and `config.set_notification_route_destinations`, functions that store only a sealed value or an empty one for a webhook and refuse an agent route. `config.set_notification_route_destinations` hands back only whether a webhook or key was saved before, for the columns the call names, and never a stored value; for a changed webhook value it accepts only the sealed form (`sealed:v1:`, the key id and base64, at most 16384 characters), and for a changed recipient list it refuses control characters, angle brackets, double quotes and more than 50 entries. Every writer (the Viewer's settings and route windows, the MCP route tools) refuses a Teams, Slack or generic webhook URL at save unless it is an absolute http or https URL with a host and no user name or password and whose host is not a link-local or cloud instance address written out, and refuses an email recipient list that is not comma separated plain addresses (the function the `mcp` role writes through checks only the shapes listed above, and the service checks the rest before it calls it), and a direct webhook post never connects to a link-local address (169.254.0.0/16, fe80::/10), whether it came from the URL, a DNS answer or a redirect (a channel that posts through a proxy is only checked against the URL's own address at save, because Darling cannot see what the proxy connects to); loopback and private addresses stay allowed for internal receivers. Routes hot-reload through the same `config_version` beacon as the parent row (the V131 trigger), so a change lands on the next firing with no restart. +#### Limiting a route to Critical or Warning alerts (V229, #6928) + +A route takes alerts of both severities unless you limit it. The **severities** setting of a route is `both` (the default, and what every route you made earlier has), `critical` or `warning`. A route set to `critical` takes Critical alerts only. A route set to `warning` takes Warning alerts only. With one route per severity you can send production warnings to a team channel and keep the on-call channel for Critical alerts. + +A route whose setting leaves an alert out does not match it. Matching goes on to the next route in the order above, and to the default channels when no route is left. It is the same fall-through as a route that is disabled or that names a different server. + +- **Which severity an alert has.** The alert's own severity decides when it carries one. An alert that is graded when it fires, such as an external alert or a custom alert rule, carries the severity it fired at. Otherwise the alert's badge decides: a red badge is Critical, and every other badge (Warning, Alert, Info) counts as Warning. So a Warning-only route also takes alerts that are not Critical. +- **A Warning alert that becomes Critical** while it is open is delivered again as a new Critical alert. The Critical route takes it from then on. +- **A resolution** goes to the channels that the alert's fire reached. The severity setting is not read again when the alert clears. Editing a route between the fire and the clear does not move the clear. The PagerDuty close for an external alert uses the severity that the fire went out at. It reaches the routing key of the route that opened the incident. +- **Agent requests** follow the same filter: a Warning alert does not open an investigation under a route limited to `critical`. +- **Set it** in the route dialog of the Viewer or in the Routes list of the web dashboard. In MCP, use the `severities` argument of `create_notification_route` and `update_notification_route`. The value is `critical`, `warning` or `both`, in any case. Any other value is refused as `invalid`. `get_notification_routes` returns `severities` for every route. +- **Lite** does not have this setting. A route you make in Lite always takes both severities. + #### Send resolutions (#5825) A resolution is the message that an alert has cleared: the condition that fired it is no longer there. Each channel (Slack, Teams, the generic webhook and email) has a **Send resolutions** box in the Viewer's Settings > Notifications, off by default. When it is on, a resolution goes only to the destinations of that channel that the alert's fire reached; an alert whose fire was not sent gets no resolution. A service restart between the fire and the clear does not lose this: the clear reads the fire's alert-history row, which names each channel that delivered and the route that supplied it, and sends to that route's destination as it is now (or the channel's own destination when no route supplied it). A route deleted or disabled since the fire gets nothing, with or without a restart between the fire and the clear: nothing is sent to that channel, and the history row says so. A mute rule stops a resolution when it matches the alert's fire, including a rule scoped to a database, wait type, job or query. Each resolution carries the server, the alert, when it fired and cleared, how long it lasted, the last value its fire sent and the fire's triage link. In alert history, the resolution row records what its send did: `get_alert_history` reports `resolution_sent_at` (UTC, set when a channel delivered), `resolution_channels` and `resolution_outcome` (`sent`, `failed: `, `muted` or `route deleted or disabled: `), null on every other row; the Viewer's Alert History grid has a **Resolution** column and Alert Details shows the three fields; the web Alerts page shows them in the row's detail. A route can override the channel setting for its own destinations: in the route dialog, **Send resolutions** is `Use channel setting` (the default), `Always` or `Never`. PagerDuty is not one of the four: its incidents close through **Auto-resolve the incident when the alert's condition clears**. Over MCP, `get_alert_settings` reports the four settings as `delivery.send_resolutions`, `update_alert_settings` changes them, and `get_notification_routes` reports each route's `send_resolutions` (null means the channel decides). A bring-your-own store needs `tools/provision-roles.sql` run again after the service upgrades the store to V187, so the `viewer` and `mcp` roles can read the new columns. diff --git a/docs/darling/configuration-mcp.md b/docs/darling/configuration-mcp.md index 93fa74f9cf..9d361029b5 100644 --- a/docs/darling/configuration-mcp.md +++ b/docs/darling/configuration-mcp.md @@ -59,7 +59,7 @@ The embedded MCP server, over Streamable HTTP bound to `localhost` by default (s - **Tool reading guides** - `get_tool_guide(tools, topics)` (#3898, both editions). A converted tool's description is split at a `<>` marker: `tools/list` serves the short head, which keeps every fact needed to read the answer correctly and ends "Reading guide: get_tool_guide.", and this tool returns the long-form guide by tool name, plus named cross-tool topics such as `system_health_empty_windows`. Unknown names are reported, not errors; called with no arguments it lists the tools that have guides and the topics. The `get_health_parser_*` family is converted first. - **Alert + health-overview tools** — the fleet-triage reads the fleet edition previously lacked, each a stored read over the monitoring store (no live hit): - - *Alerts* — `get_alert_history` (what fired, value vs threshold, delivery success/failure, muted, and since V131 `route` — where the post went — fleet-wide by default, or scoped to a server; `external_source` names the external alert source an alert came from and is null on every built-in and custom alert), `get_alert_settings` (the current alert config the service is using — per-alert enable/thresholds, cooldown, excluded databases, delivery mode, analysis cadence), `get_mute_rules` (the alert mute rules in force, so a suppressed server is distinguishable from a healthy-quiet one; each rule carries `external_source`, null unless the rule is limited to one external alert source), `get_notification_routes` (the alert-family taxonomy and the [notification routes](alerts-and-notifications.md#notification-routes--sending-alert-families-to-different-channels) — which channels each route sets, never the destination values). + - *Alerts* — `get_alert_history` (what fired, value vs threshold, delivery success/failure, muted, and since V131 `route` — where the post went — fleet-wide by default, or scoped to a server; `external_source` names the external alert source an alert came from and is null on every built-in and custom alert), `get_alert_settings` (the current alert config the service is using — per-alert enable/thresholds, cooldown, excluded databases, delivery mode, analysis cadence), `get_mute_rules` (the alert mute rules in force, so a suppressed server is distinguishable from a healthy-quiet one; each rule carries `external_source`, null unless the rule is limited to one external alert source), `get_notification_routes` (the alert-family taxonomy and the [notification routes](alerts-and-notifications.md#notification-routes--sending-alert-families-to-different-channels) — which channels each route sets and its `severities`, never the destination values). - *Health overview* — `get_server_summary` (one-shot per-server CPU / memory / recent blocking / recent deadlocks), `get_daily_summary` (a day's composite health band — Healthy / Warning / Critical — folded through the shared `DailyHealthBandCalculator`, plus the signals behind it). - **Fleet sweep reports (Darling-only)** — `get_sweep_reports` reads the scheduled [Fleet Sweep Reports](https://github.com/erikdarlingdata/PerformanceMonitor/issues/3466) — the stateful whole-fleet summaries the service persists at the configured cadence (`get_alert_settings`' `fleet_sweep` group; default hourly): the sweep timeline over a caller-set window with each sweep's document embedded, the newest sweep in full (per-server verdicts with band transitions pre-computed, and the would-have-paged ledger whenever a sweep ran under the alert master switch — present-and-empty on a muted sweep with nothing to report, ABSENT on an alerts-on sweep where no such check was made), and the watch-item worklist with its hysteresis position. One `sweep_id` (as a STRING — the ids are too large for a JSON number to round-trip) fetches one sweep in full. The same `FleetSweepPresentation` builders and store reads the web feed's `/api/sweeps` routes serve, so an agent and a browser cannot disagree about what a sweep looks like. The description carries the two field contracts an agent must read: `alerts_enabled: false` on a sweep means DELIVERY WAS OFF while it ran, and `instruments_alive: false` means the sweep could not prove its own data sources — quiet is not clean, and the `instrument_liveness` block says which instrument went unproven. A stored read; no monitored server is touched. @@ -72,7 +72,7 @@ The embedded MCP server, over Streamable HTTP bound to `localhost` by default (s The create/update/delete tools are the one view-authoring **write** surface; create/update run the SAME `ValidateDefinition` authority as `validate_custom_view`, so an invalid definition is rejected before it stores; every tool routes through the SAME store + validator + compile-and-run + catalog the web viewer's editor uses (no divergent second implementation). This write surface is part of what the MCP token gates — see [What a token can reach](network-endpoints.md) below. -- **Alert-tuning write tools (Darling-only)** — `update_alert_settings`, `create_mute_rule`, `update_mute_rule`, `delete_mute_rule`, `purge_expired_mute_rules` (deletes every expired rule, as the Viewer's Purge Expired does, and returns each one it deleted with `undo: null` because an expired rule had no effect), `set_mute_rule_enabled`, and the route writes `create_notification_route` / `update_notification_route` / `set_notification_route_enabled` / `set_notification_route_fallback` / `delete_notification_route`, and `update_notification_channels` (the default channels the Viewer sets in Settings > Notifications: SMTP host, port, SSL, user, password, from and recipients; the Teams, Slack and generic webhook URLs, the generic headers, body template and proxies; the PagerDuty routing key, region, proxy and auto-resolve. Only the fields passed change and `-` clears a text field. Each secret is sealed by the service the way the Viewer seals it and is shown in an answer only as set or not set; the `mcp` role holds no read or write on a secret column and writes them through the store's `config.set_notification_channels` function, which makes the Viewer's refusals on the stored row (a change to the host, port, SSL flag, user or a proxy while a secret tied to it is saved needs the secret again; a saved signing key blocks a generic URL or proxy change). The undo restores only the non-secret settings.) (both route tools set a route's own destinations: Teams, Slack and generic webhook URLs, a PagerDuty routing key and email recipients, sealed for the route's own row and shown in an answer only as saved or none (with the host for Teams and Slack), apart from the recipient list, which is shown in full; `create_notification_route` with no destination makes a route with every channel empty, so each channel goes where it went before (the next matching route, else the channels configured in Settings); `update_notification_route` changes the match, scope, enabled flag, resolution override, fallback and destinations of any route that is not an agent route, and the match or scope only on a route with no explicit agent setting (a route with an agent setting of Off keeps its match and scope in the Viewer, and the store function refuses the change for any login); both refuse an agent route, and the answer of an edit gives the previous values and the undo call, which for a destination says to pass the earlier value again. Each write is logged with the tool, the identity the service knows for the token (its first 8 SHA-256 hex characters, never the token), the client's own name labelled `client says` (any client can send any name) and the names of the fields changed, never a value), plus `dismiss_alert_history` / `restore_alert_history` (the flag the web dashboard's Dismiss sets, and the call that undoes it; nothing is deleted). None of these is callable by an agent Darling sends to investigate an alert let an MCP client TUNE the alert engine the fleet shares (`create_mute_rule` / `update_mute_rule` also take an optional lower-case `external_source` that limits a rule to the alerts one external source raised; a name that is not a valid source name is refused, and `update_mute_rule` clears it with an explicit null) — the SAME config `get_alert_settings` / `get_mute_rules` read and the Viewer's Settings window writes. `update_alert_settings` is a PARTIAL update of the alert configuration: read via `get_alert_settings`, change fields, and send only those back in the same nested shape; every field is validated against the SAME ranges/enums the Settings window enforces BEFORE any write, an out-of-range or unknown field returns `{status:"invalid"}` and writes nothing, and the write self-bumps `config_version` so the running service hot-reloads within one collection sweep. `create_mute_rule` / `update_mute_rule` / `delete_mute_rule` / `set_mute_rule_enabled` reuse the SAME `PgMuteRuleStore` `get_mute_rules` reads through (and the same GUID id-generation the Viewer's mute-create path uses). `set_mute_rule_enabled` is the headless twin of the Viewer's mute-rule checkbox: it writes `enabled` and nothing else, so a rule can leave force and return keeping its id, its reason and the `created_at_utc` the `Stale Mute Rules` self-alert ages it from — where a delete-and-re-create resets that clock and delivers the suppressed alerts in between. `update_mute_rule` is the headless twin of the Viewer's Edit dialog on the same terms: a PARTIAL update (send only the fields to change, in the shape `get_mute_rules` returns; an explicit JSON `null` clears a field, so a bounded rule can be made permanent again) written through the store's own full-row `UpdateAsync` — whose SET list does not name `created_at_utc`, so no edit, however sweeping, resets the staleness clock the way the delete/recreate workaround did for every field the enable flag cannot express. `update_alert_settings` spans the two singleton config rows the alert engine reads: every knob is on `config_alert_settings` except the per-fingerprint DELIVERY cooldown, which is stored as `config_notification.email_cooldown_minutes` and reported and accepted as `delivery.cooldown_minutes` — a channel-neutral name, because that one number is the only throttle on a Slack / Teams / PagerDuty / generic-webhook post as well as on email, so a headless deployment with no SMTP configured at all is still governed by it (the stored spelling is accepted as a write-only alias). Note the two cooldowns are different stages: top-level `cooldown_minutes` gates whether the engine FIRES, `delivery.cooldown_minutes` bounds the resulting post — twice over, once per alert FINGERPRINT and once per METRIC across the whole fleet for a RE-notification, so one fault on forty servers does not cost forty posts an hour and the servers that second bound holds back are named on the post that does go out. A first notice is never held back by either bound, and `delivery.mode` `PerEvent` opts out of the per-metric one. None touches a monitored SQL Server or the collected data — only the shared alert configuration; the one tool that sets delivery credentials is `update_notification_channels`, which sets the SMTP password, the webhook URLs and the keys sealed and never shows them back (the `mcp` role itself can read and write exactly the one non-secret cooldown column of the notification table and cannot read or write any secret column; the sealed values reach the store through the `config.set_notification_channels` function). It is part of what the MCP token gates — see [What a token can reach](network-endpoints.md) below. +- **Alert-tuning write tools (Darling-only)** — `update_alert_settings`, `create_mute_rule`, `update_mute_rule`, `delete_mute_rule`, `purge_expired_mute_rules` (deletes every expired rule, as the Viewer's Purge Expired does, and returns each one it deleted with `undo: null` because an expired rule had no effect), `set_mute_rule_enabled`, and the route writes `create_notification_route` / `update_notification_route` / `set_notification_route_enabled` / `set_notification_route_fallback` / `delete_notification_route`, and `update_notification_channels` (the default channels the Viewer sets in Settings > Notifications: SMTP host, port, SSL, user, password, from and recipients; the Teams, Slack and generic webhook URLs, the generic headers, body template and proxies; the PagerDuty routing key, region, proxy and auto-resolve. Only the fields passed change and `-` clears a text field. Each secret is sealed by the service the way the Viewer seals it and is shown in an answer only as set or not set; the `mcp` role holds no read or write on a secret column and writes them through the store's `config.set_notification_channels` function, which makes the Viewer's refusals on the stored row (a change to the host, port, SSL flag, user or a proxy while a secret tied to it is saved needs the secret again; a saved signing key blocks a generic URL or proxy change). The undo restores only the non-secret settings.) (both route tools set a route's own destinations: Teams, Slack and generic webhook URLs, a PagerDuty routing key and email recipients, sealed for the route's own row and shown in an answer only as saved or none (with the host for Teams and Slack), apart from the recipient list, which is shown in full; `create_notification_route` takes the same `severities` word (default `both`, so a route takes alerts of both severities unless you limit it, and any other value is refused as `invalid`), and with no destination makes a route with every channel empty, so each channel goes where it went before (the next matching route, else the channels configured in Settings); `update_notification_route` changes the match, scope, enabled flag, resolution override, severity filter (`severities`: `critical`, `warning` or `both`), fallback and destinations of any route that is not an agent route, and the match or scope only on a route with no explicit agent setting (a route with an agent setting of Off keeps its match and scope in the Viewer, and the store function refuses the change for any login); both refuse an agent route, and the answer of an edit gives the previous values and the undo call, which for a destination says to pass the earlier value again. Each write is logged with the tool, the identity the service knows for the token (its first 8 SHA-256 hex characters, never the token), the client's own name labelled `client says` (any client can send any name) and the names of the fields changed, never a value), plus `dismiss_alert_history` / `restore_alert_history` (the flag the web dashboard's Dismiss sets, and the call that undoes it; nothing is deleted). None of these is callable by an agent Darling sends to investigate an alert let an MCP client TUNE the alert engine the fleet shares (`create_mute_rule` / `update_mute_rule` also take an optional lower-case `external_source` that limits a rule to the alerts one external source raised; a name that is not a valid source name is refused, and `update_mute_rule` clears it with an explicit null) — the SAME config `get_alert_settings` / `get_mute_rules` read and the Viewer's Settings window writes. `update_alert_settings` is a PARTIAL update of the alert configuration: read via `get_alert_settings`, change fields, and send only those back in the same nested shape; every field is validated against the SAME ranges/enums the Settings window enforces BEFORE any write, an out-of-range or unknown field returns `{status:"invalid"}` and writes nothing, and the write self-bumps `config_version` so the running service hot-reloads within one collection sweep. `create_mute_rule` / `update_mute_rule` / `delete_mute_rule` / `set_mute_rule_enabled` reuse the SAME `PgMuteRuleStore` `get_mute_rules` reads through (and the same GUID id-generation the Viewer's mute-create path uses). `set_mute_rule_enabled` is the headless twin of the Viewer's mute-rule checkbox: it writes `enabled` and nothing else, so a rule can leave force and return keeping its id, its reason and the `created_at_utc` the `Stale Mute Rules` self-alert ages it from — where a delete-and-re-create resets that clock and delivers the suppressed alerts in between. `update_mute_rule` is the headless twin of the Viewer's Edit dialog on the same terms: a PARTIAL update (send only the fields to change, in the shape `get_mute_rules` returns; an explicit JSON `null` clears a field, so a bounded rule can be made permanent again) written through the store's own full-row `UpdateAsync` — whose SET list does not name `created_at_utc`, so no edit, however sweeping, resets the staleness clock the way the delete/recreate workaround did for every field the enable flag cannot express. `update_alert_settings` spans the two singleton config rows the alert engine reads: every knob is on `config_alert_settings` except the per-fingerprint DELIVERY cooldown, which is stored as `config_notification.email_cooldown_minutes` and reported and accepted as `delivery.cooldown_minutes` — a channel-neutral name, because that one number is the only throttle on a Slack / Teams / PagerDuty / generic-webhook post as well as on email, so a headless deployment with no SMTP configured at all is still governed by it (the stored spelling is accepted as a write-only alias). Note the two cooldowns are different stages: top-level `cooldown_minutes` gates whether the engine FIRES, `delivery.cooldown_minutes` bounds the resulting post — twice over, once per alert FINGERPRINT and once per METRIC across the whole fleet for a RE-notification, so one fault on forty servers does not cost forty posts an hour and the servers that second bound holds back are named on the post that does go out. A first notice is never held back by either bound, and `delivery.mode` `PerEvent` opts out of the per-metric one. None touches a monitored SQL Server or the collected data — only the shared alert configuration; the one tool that sets delivery credentials is `update_notification_channels`, which sets the SMTP password, the webhook URLs and the keys sealed and never shows them back (the `mcp` role itself can read and write exactly the one non-secret cooldown column of the notification table and cannot read or write any secret column; the sealed values reach the store through the `config.set_notification_channels` function). It is part of what the MCP token gates — see [What a token can reach](network-endpoints.md) below. - **Webhook signing key tools (Darling-only, #6394)** — `generate_webhook_signing_key` and `remove_webhook_signing_key`, each with an optional `route_id` (none is the default destination's key in Settings > Notifications, a number is that route's own generic webhook). `generate_webhook_signing_key` makes the key in the service, seals it for the destination's saved generic URL and proxy as the Viewer does, and stores it through the store's `config.set_webhook_signing_key` function; the `mcp` role has no read or write on a signing key column, and the service reads the saved URL and proxy on its own store connection. When a key is already stored the call rotates it: the current key moves to the previous slot and keeps signing for 7 days. The answer shows the new key once, and no answer ever shows a stored or previous key; the key goes only over TLS or to a caller on the service's machine, and a second generate inside the 7 day changeover of the first is refused unless the call passes `replace_pending_rotation: true`. A destination with no saved generic webhook URL, and a `route_id` that does not exist, are refused and write nothing. `remove_webhook_signing_key` clears the key, the previous key and the window, and says whether a key was there. Both are write tools, never offered to an agent token. `update_notification_route` refuses to change the generic URL of a route that has a key until the key is removed. - **Server-onboarding write tools (Darling-only)** — `add_servers` (BULK) and `remove_server` (which drops this install's own Extended Events sessions on the server and leaves the shared ones: `PerformanceMonitor_Deadlock`, `PerformanceMonitor_BlockedProcess`, `PerformanceMonitor_Attention` and the old `PerformanceMonitor_LongQueryCompletions`; once the server is removed, [`--drop-xe-sessions --print-sql`](collector-settings.md#drop-the-extended-events-sessions-a-removed-server-left-behind---drop-xe-sessions) prints the statements that drop what is left, because the named form `--drop-xe-sessions ` works only before the removal) let an MCP client stand up or tear down FLEET monitoring conversationally ("monitor these twenty servers with this login"), the service-side twin of the Viewer's Add / Manage Servers dialogs. `edit_server` changes one saved server in place and is served under the same `mcp.allowServerAdmin` setting as `add_servers` and `remove_server`; an edit that points Windows authentication at a connection whose server identity is not verified is refused as `add_servers` refuses it, before any connection is made. `add_servers` takes a JSON **array** of server objects (`host` required; optional `display_name` / `database` / `read_only_intent` / `multi_subnet_failover`; `auth` `Windows`/`SQL` with `username`+`password` for SQL; and the exposed TLS options `encrypt_mode` `Optional`/`Mandatory`/`Strict` + `trust_server_certificate`) and processes them **in order**: it validates each entry, PROBES the connection in-process (reusing the same `DarlingServerConnector.ProbeAsync` the `--test-connection` verb runs — the service holds the network path + credentials, so no `test_connect` command plane is needed), skips a case-folded duplicate (`duplicate`) of an already-monitored server or an earlier entry, DPAPI-encrypts the SQL password (the service identity, so it round-trips at collection time), and INSERTs the row mirroring the service's own seed shape. A server that fails to connect is `connection_failed` and the batch continues. `auth` also takes the two non-interactive Microsoft Entra modes (#3484). With `ServicePrincipal`, `username` is the app's client id and `password` is its client secret. The other mode is `ManagedIdentity`. Only the interactive Entra modes (MFA, device code, default credential) are `invalid`, because they need a signed-in user and cannot run headless. Off Windows, a literal SQL password or client secret is refused as `invalid` too, because encryption at rest is Windows DPAPI. Give an `env:NAME` or `file:/run/secrets/` reference instead, the same two forms that `darling.json` takes for every secret. A server is refused as `collides` when its probed connection lands in a database that another monitored server already covers. That happens when it names one database but connects to a different one, such as a wrong Initial Catalog. Adding it gives one database two identities, so its history is stored twice and its alerts fire twice. A server whose id is already held by a different server is refused as `collides` as well, because two identities can hash to one id. Nothing is written for a `collides` entry. If saving an entry fails, that entry and every entry after it are `not_saved` and are not probed. The entries before it stay `added`. `remove_server` DELETEs a monitored server by name (resolved the same way every `server_name` is) — already-collected history is kept. Both write only the monitoring store's `config.config_monitored_servers` registry; neither runs anything on a monitored server beyond the one-time probe. **The SQL password travels to the endpoint inside `add_servers`' request** and is DPAPI-encrypted at rest (never returned) — it is part of what the MCP token gates, and it puts a credential on the wire; see [What a token can reach](network-endpoints.md) below. From 579d338b67a1dc91953674f7051bc9d6da759dd0 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:11:14 -0400 Subject: [PATCH 11/22] Move the route tests and the schema version to V229 and declare the rung in the data-moving census (part of #6928) --- .../Darling.Tests/MigrationDataMovingRungCensusPins.cs | 8 ++++++++ Darling/Darling.Tests/NotificationRoutesRungTests.cs | 7 ++++--- .../Darling.Tests/ViewerNotificationRouteScopeTests.cs | 2 +- .../PerformanceMonitor.Darling.Storage/StorageVersion.cs | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs b/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs index 2c222b64ce..9139bdbced 100644 --- a/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs +++ b/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs @@ -359,6 +359,14 @@ public sealed class MigrationDataMovingRungCensusPins + "singleton (id = 1): turns the Client Timeouts alert on by default (#5736). V147's shape on the same kind of table. The " + "UPDATE's WHERE client_timeouts_enabled = FALSE touches at most one row, the SET DEFAULT is catalog-only, and neither reads " + "any collected data, so the rung spends none of the lock-wait budget and the multiple stays 5."), + new( + 229, + SetsTheFloor: false, + "ADD COLUMN IF NOT EXISTS severities text NOT NULL DEFAULT 'both' plus ADD CONSTRAINT ... CHECK on config.config_notification_routes " + + "(created V131), the notification routes registry (#6928). The ADD COLUMN with a constant default is catalog-only. The CHECK " + + "reads the table once to validate it, and the table is a hand-edited registry of at most a few hundred rows (one per route " + + "an operator made), so the scan takes milliseconds. It reads no collected data and spends none of the lock-wait budget, so " + + "the multiple stays 5."), ]; /// diff --git a/Darling/Darling.Tests/NotificationRoutesRungTests.cs b/Darling/Darling.Tests/NotificationRoutesRungTests.cs index c2c0686767..329fe9801c 100644 --- a/Darling/Darling.Tests/NotificationRoutesRungTests.cs +++ b/Darling/Darling.Tests/NotificationRoutesRungTests.cs @@ -159,7 +159,7 @@ public void TheViewerCarve_ClassifiesEveryColumn_MatchingTheParentForSharedNames var parent = DarlingManagedRoles.ViewerRestrictedConfigTables.Single(a => a.Table == "config_notification"); Assert.Equal(new[] { "teams_url", "slack_url", "generic_url", "pagerduty_routing_key", "signing_key", "previous_signing_key" }, acl.SecretColumns); - Assert.Equal(new[] { "route_id", "metric_match", "smtp_recipients", "configured_channels", "enabled", "modified_at", "scope_server_id", "scope_tag_id" }.Concat(AgentColumns).Concat(new[] { "fallback_route_id", "previous_signing_key_until" }), acl.NonSecretColumns); + Assert.Equal(new[] { "route_id", "metric_match", "smtp_recipients", "configured_channels", "enabled", "modified_at", "scope_server_id", "scope_tag_id" }.Concat(AgentColumns).Concat(new[] { "fallback_route_id", "previous_signing_key_until", "severities" }), acl.NonSecretColumns); foreach (var column in DestinationColumns) { @@ -243,7 +243,7 @@ public void TheServiceReader_TheViewerWriter_AndTheDdl_NameTheSameColumnsInTheSa below is unchanged. */ /* V193 (#5998): the signing key columns come last on the service read; the viewer reads them by their own statements. */ /* #6003 (D8): fallback_route_id follows them, set by init like the agent columns; the viewer reads and writes it since #6036. */ - Assert.Equal(new[] { "route_id", "metric_match" }.Concat(DestinationColumns).Concat(new[] { "enabled", "scope_server_id", "scope_tag_id" }).Concat(AgentColumns).Concat(new[] { "signing_key", "previous_signing_key", "previous_signing_key_until", "fallback_route_id" }), serviceSelect); + Assert.Equal(new[] { "route_id", "metric_match" }.Concat(DestinationColumns).Concat(new[] { "enabled", "scope_server_id", "scope_tag_id" }).Concat(AgentColumns).Concat(new[] { "signing_key", "previous_signing_key", "previous_signing_key_until", "fallback_route_id", "severities" }), serviceSelect); Assert.EndsWith("ORDER BY route_id", StoreConfigProvider.NotificationRoutesSelectSql, StringComparison.Ordinal); /* The constructor's parameter order IS the SELECT's order, so a reader ordinal cannot re-map a column. */ @@ -257,7 +257,7 @@ below is unchanged. */ ColumnsOf(ViewerDataService.NotificationRoutesSelectNoSecretSql, "SELECT", "FROM")); var insert = ColumnsOf(ViewerDataService.NotificationRouteInsertSql, "(", ")"); - Assert.Equal(new[] { "metric_match" }.Concat(DestinationColumns).Concat(new[] { "enabled", "scope_server_id", "scope_tag_id" }).Concat(AgentColumns).Concat(new[] { "fallback_route_id", "modified_at" }), insert); + Assert.Equal(new[] { "metric_match" }.Concat(DestinationColumns).Concat(new[] { "enabled", "scope_server_id", "scope_tag_id" }).Concat(AgentColumns).Concat(new[] { "fallback_route_id", "severities", "modified_at" }), insert); Assert.DoesNotContain("route_id", insert); Assert.Contains("RETURNING route_id", ViewerDataService.NotificationRouteInsertSql, StringComparison.Ordinal); @@ -471,6 +471,7 @@ public async Task InsertRoute_BumpsTheBeacon_ReloadsThroughTheProvider_AndResolv insert.Parameters.Add(new NpgsqlParameter { TypedValue = NotificationRoute.DefaultAgentExpiryMinutes }); insert.Parameters.Add(new NpgsqlParameter { Value = DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Boolean }); insert.Parameters.Add(new NpgsqlParameter { Value = DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Integer }); /* fallback_route_id (#6036) */ + insert.Parameters.Add(new NpgsqlParameter { TypedValue = "both" }); /* severities (V229, #6928) */ routeId = Convert.ToInt32(await insert.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); } diff --git a/Darling/Darling.Tests/ViewerNotificationRouteScopeTests.cs b/Darling/Darling.Tests/ViewerNotificationRouteScopeTests.cs index 746a93034f..b950a7e4b2 100644 --- a/Darling/Darling.Tests/ViewerNotificationRouteScopeTests.cs +++ b/Darling/Darling.Tests/ViewerNotificationRouteScopeTests.cs @@ -78,7 +78,7 @@ public void TheSql_ReadsAndWritesBothScopeColumns_InEveryStatement() Assert.Contains("scope_server_id = $9", ViewerDataService.NotificationRouteUpdateSql, StringComparison.Ordinal); Assert.Contains("scope_tag_id = $10", ViewerDataService.NotificationRouteUpdateSql, StringComparison.Ordinal); /* #5632: the agent columns, $10-$14, follow the scope pair. */ - Assert.Contains("$8, $9, $10, $11, $12, $13, $14, $15, $16, (now()", ViewerDataService.NotificationRouteInsertSql, StringComparison.Ordinal); + Assert.Contains("$8, $9, $10, $11, $12, $13, $14, $15, $16, $17, (now()", ViewerDataService.NotificationRouteInsertSql, StringComparison.Ordinal); } /* ---- the picker ---------------------------------------------------------------------------------------- */ diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs index bfebc69b62..e6b397cbec 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs @@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class StorageVersion { - public const int SchemaVersion = 226; + public const int SchemaVersion = 229; } From 9e189f44982566e54d6598049c0ca59db6647e38 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:21:31 -0400 Subject: [PATCH 12/22] Add the V229 rewind artifact and the severities column to the carve pin (part of #6928) --- Darling/Darling.Tests/NotificationRoutesRungTests.cs | 2 +- Darling/Darling.Tests/RungRewindArtifacts.cs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/NotificationRoutesRungTests.cs b/Darling/Darling.Tests/NotificationRoutesRungTests.cs index 329fe9801c..d8614304e2 100644 --- a/Darling/Darling.Tests/NotificationRoutesRungTests.cs +++ b/Darling/Darling.Tests/NotificationRoutesRungTests.cs @@ -169,7 +169,7 @@ public void TheViewerCarve_ClassifiesEveryColumn_MatchingTheParentForSharedNames } /* Every column in the DDL is in exactly one list. */ - var ddlColumns = new[] { "route_id", "metric_match" }.Concat(DestinationColumns).Concat(new[] { "configured_channels", "enabled", "modified_at", "scope_server_id", "scope_tag_id" }).Concat(AgentColumns).Concat(new[] { "signing_key", "previous_signing_key", "previous_signing_key_until", "fallback_route_id" }).ToList(); + var ddlColumns = new[] { "route_id", "metric_match" }.Concat(DestinationColumns).Concat(new[] { "configured_channels", "enabled", "modified_at", "scope_server_id", "scope_tag_id" }).Concat(AgentColumns).Concat(new[] { "signing_key", "previous_signing_key", "previous_signing_key_until", "fallback_route_id", "severities" }).ToList(); var classified = acl.SecretColumns.Concat(acl.NonSecretColumns).ToList(); Assert.Equal(ddlColumns.OrderBy(c => c, StringComparer.Ordinal), classified.OrderBy(c => c, StringComparer.Ordinal)); Assert.Equal(classified.Count, classified.Distinct(StringComparer.Ordinal).Count()); diff --git a/Darling/Darling.Tests/RungRewindArtifacts.cs b/Darling/Darling.Tests/RungRewindArtifacts.cs index 375b56efdd..741c07c359 100644 --- a/Darling/Darling.Tests/RungRewindArtifacts.cs +++ b/Darling/Darling.Tests/RungRewindArtifacts.cs @@ -577,6 +577,10 @@ ALTER TABLE config.config_alert_settings // so a re-run of V226 does everything again. Safe on a store that never had it. new(226, "hasServerAwsRdsWebOrExpress", @"ALTER TABLE collect.servers DROP COLUMN IF EXISTS is_aws_rds_web_or_express;"), + // V229 (#6928): the severity filter on the notification routes. The column is the probe's newest sentinel. The rewind drops the column + // (its CHECK goes with it), so a re-run of V229 does everything again. Safe on a store that never had it. + new(229, "hasRouteSeverities", + @"ALTER TABLE config.config_notification_routes DROP COLUMN IF EXISTS severities;"), ]; /// From 735da3965ef91c348886fa88a5671d18b3c61cb3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:50:01 -0400 Subject: [PATCH 13/22] Put the cancellation token last on GetSweepPressureAsync (CA1068) and use Assert.Single in the tag scope test --- Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs | 2 +- Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs | 4 ++-- .../ViewerDataService.CollectionHealth.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs index 1c5c98400b..b97cef96a9 100644 --- a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -165,7 +165,7 @@ await RunAsync(async (ds, connection, store, ct) => Assert.Equal("Production", cpu["tag"]!.GetValue()); Assert.Equal(92, cpu["effective_value"]!.GetValue()); Assert.Equal("global", Row(eight, "blocking.count_threshold")!["source"]!.GetValue()); - Assert.Equal(1, eight["tag_overrides"]!.AsArray().Count); + Assert.Single(eight["tag_overrides"]!.AsArray()); var nine = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-9", ct))!["server_view"]!; Assert.Equal("global", Row(nine, "cpu.threshold_percent")!["source"]!.GetValue()); diff --git a/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs b/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs index 59fec5ed1f..bbdb559253 100644 --- a/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs +++ b/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs @@ -77,7 +77,7 @@ public async Task TheViewersSweepPressureAndHeaviestCollectors_AreTheFiguresTheW var web = doc.RootElement.GetProperty("sweep_pressure"); await using var viewer = new ViewerDataService(scratch.ConnectionString); - var report = await viewer.GetSweepPressureAsync(ServerId, ct); + var report = await viewer.GetSweepPressureAsync(ServerId, cancellationToken: ct); Assert.NotNull(report); var pressure = report.Pressure; @@ -113,7 +113,7 @@ public async Task TheViewersSweepPressureAndHeaviestCollectors_AreTheFiguresTheW SweepPressurePanel.StatRows(pressure).Select(r => r.Label)); /* A server with no collection log rows has no report, as the tool answers it "unavailable". */ - Assert.Null(await viewer.GetSweepPressureAsync(EmptyServerId, ct)); + Assert.Null(await viewer.GetSweepPressureAsync(EmptyServerId, cancellationToken: ct)); bodySucceeded = true; } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs index 97cb7b2228..f0752874d1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs @@ -518,7 +518,7 @@ public async Task> GetCollectionHealthAsync(int serverI /// pre-gate runs are left out of both panels, as get_collection_health computes its figures from the running rows only, so the /// panel and the tool name the same peak collector for the same server. Null leaves every row in. public async Task GetSweepPressureAsync( - int serverId, CancellationToken cancellationToken = default, IReadOnlySet? engineGated = null) + int serverId, IReadOnlySet? engineGated = null, CancellationToken cancellationToken = default) { var scheduleOverrides = await ReadScheduleOverridesForHealthAsync(cancellationToken); var costs = new List<(CollectorHealthRow Row, double Avg, double P95, double Max)>(); From 464b75ea75f2c941abea5b73459100704df18fd0 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:53:13 -0400 Subject: [PATCH 14/22] Close every destination a channel was fired to, give the agent gate the alert tier, and journal severity filter changes (part of #6928) An alert that fired Warning and climbed to Critical now sends its clear to each destination its fires reached, on the in-process path and the external alert path. The agent-action gate re-resolves each alert at the tier it arrived at. A change to the severity filter of an agent route is journaled from the writer and the MCP tool. --- .../AgentActionGateSeverityLiveTests.cs | 89 +++++++++++++ Darling/Darling.Tests/AgentActionGateTests.cs | 10 +- .../Darling.Tests/AgentChangeUndoLiveTests.cs | 2 +- .../Darling.Tests/AgentLevel2StoreTests.cs | 2 +- .../ExternalAlertDrainLiveTests.cs | 27 +++- .../Darling.Tests/ExternalAlertRungTests.cs | 2 +- .../InvestigationPracticeTests.cs | 2 +- .../NotificationRouteSeverityLiveTests.cs | 78 +++++++++++ .../NotificationRouteSeverityTests.cs | 126 ++++++++++++++++++ .../Darling.Tests/PlanForceCheckLiveTests.cs | 2 +- Darling/Darling.Tests/RungRewindArtifacts.cs | 4 +- .../Darling.Tests/ServerActionGateTests.cs | 14 +- .../ServerActionSwitchWriteLiveTests.cs | 2 +- .../DarlingAlertDeliverer.cs | 112 +++++++++------- .../DarlingManagedRoles.cs | 13 +- .../ExternalAlertDrain.cs | 25 +++- .../Mcp/DarlingAgentActionGate.cs | 4 +- .../Mcp/DarlingMcpAlertTools.RouteAdmin.cs | 2 +- .../ExternalAlertStore.cs | 17 ++- .../InvestigationPracticeStore.cs | 4 +- .../InvestigationRequestStore.AgentAction.cs | 30 +++-- .../InvestigationRequestStore.ServerAction.cs | 5 +- .../InvestigationRequestStore.cs | 12 +- .../NotificationRouteWriter.cs | 7 + .../PgMigrations.cs | 9 ++ Darling/tools/provision-roles.sql | 13 +- .../WebhookAlertService.cs | 57 +++++--- 27 files changed, 553 insertions(+), 117 deletions(-) create mode 100644 Darling/Darling.Tests/AgentActionGateSeverityLiveTests.cs diff --git a/Darling/Darling.Tests/AgentActionGateSeverityLiveTests.cs b/Darling/Darling.Tests/AgentActionGateSeverityLiveTests.cs new file mode 100644 index 0000000000..686b8148ba --- /dev/null +++ b/Darling/Darling.Tests/AgentActionGateSeverityLiveTests.cs @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Npgsql; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/* #1776 own-store: the fact mints its own scratch database through ScratchPostgres, so the class is deliberately NOT [Collection("live-postgres")]. */ + +/// +/// #6928: the agent-action gate re-resolves a request's routes at the tier each alert arrived at, so a Critical alert dispatched under a +/// fleet route is not refused because a server route that takes only Warning (agent Off) would have won the lookup with no tier. +/// +public sealed class AgentActionGateSeverityLiveTests +{ + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task ACriticalAlertDispatchedUnderTheFleetRoute_IsNotRefusedByAWarningOnlyServerRouteThatIsOff() + { + Assert.SkipWhen(string.IsNullOrEmpty(ConnectionString), "Set DARLING_TEST_PG to run the agent gate severity pin (it mints its own scratch database)."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(ConnectionString!, ct); + await using (var migrate = new NpgsqlConnection(scratch.ConnectionString)) + { + await migrate.OpenAsync(ct); + await PgMigrations.MigrateAsync(migrate, ct); + } + + await using var owner = NpgsqlDataSource.Create(scratch.ConnectionString); + var bodySucceeded = false; + try + { + var proxies = new WebhookProxies(null, null, null, null); + /* A server-scoped Blocking route that takes Warning only, with the agent Off, and a fleet-wide Blocking route with the agent at Act. */ + await using (var seed = owner.CreateCommand("INSERT INTO config.config_monitored_servers (server_id, name, host) VALUES (7, 'srv-7', 'host-a') ON CONFLICT DO NOTHING")) + { + await seed.ExecuteNonQueryAsync(ct); + } + + await NotificationRouteWriter.SaveAsync( + owner, new NotificationRouteDraft { MetricMatch = "Blocking", ScopeServerId = 7, Severities = "warning", AgentLevel = 0 }, + null, proxies, null, null, 30, ct); + await NotificationRouteWriter.SaveAsync( + owner, new NotificationRouteDraft { MetricMatch = "Blocking", AgentLevel = 2 }, null, proxies, null, null, 30, ct); + + var store = new InvestigationRequestStore(owner, 30); + /* A Critical alert dispatches under the fleet route: request level 2. */ + var opened = await store.OpenOrJoinAsync( + new InvestigationAlert(1, 7, "srv-7", "blocking", "Blocking", DateTime.UtcNow, Array.Empty(), null, 20, 100, LeaseMinutes: 15, ExpiryMinutes: 240) + { AutonomyLevel = 2, Severity = "critical" }, ct); + var request = opened.RequestId!.Value; + var claim = (await store.ClaimAsync(request, "agent-a", ct)).ClaimId!.Value; + + var writes = 0; + var hooks = new AgentActionHooks + { + LoadLevels = (connection, _, token) => DarlingAgentActionGate.LoadLevelsAsync(connection, NullLogger.Instance, token), + ReadBefore = (_, _, _) => Task.FromResult(null), + Write = (_, _, _) => { writes++; return Task.FromResult("{}"); }, + }; + var call = new AgentActionCall("create_mute_rule", "mute", new AgentAction(request, claim, "agent-a", "known batch", + "[{\"tool\":\"get_blocking\",\"arguments\":{},\"key_numbers\":\"12\"}]"), new AgentActionTarget(7, "srv-7", null, null)); + + var done = await store.RunAgentActionAsync(call, hooks, ct); + + Assert.True(done.Done, done.Refusal?.Message); + Assert.Equal(1, writes); + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, static (_, _) => Task.CompletedTask); + } + } +} diff --git a/Darling/Darling.Tests/AgentActionGateTests.cs b/Darling/Darling.Tests/AgentActionGateTests.cs index 5c6fcf5e47..bd1be2deb4 100644 --- a/Darling/Darling.Tests/AgentActionGateTests.cs +++ b/Darling/Darling.Tests/AgentActionGateTests.cs @@ -218,7 +218,7 @@ private static InvestigationAlert Alert(DateTime at, int level = 2, string metri private AgentActionHooks Hooks(int? resolved = 2, AgentActionRefusal? kind = null) => new() { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => resolved), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => resolved), KindRule = kind is null ? null : (_, _, _, _) => Task.FromResult(kind), ReadBefore = (_, _, _) => Task.FromResult("{\"reason\":\"old\"}"), Write = (_, _, _) => { _writes++; return Task.FromResult("{\"reason\":\"new\"}"); }, @@ -363,7 +363,7 @@ await RunAsync(async (store, connection, ct) => LoadLevels = (_, _, _) => { var level = ++calls == 1 ? 2 : 1; - return Task.FromResult>((_, _) => level); + return Task.FromResult>((_, _, _) => level); }, ReadBefore = (_, _, _) => Task.FromResult(null), Write = (_, _, _) => { _writes++; return Task.FromResult(null); }, @@ -387,7 +387,7 @@ await RunAsync(async (store, connection, ct) => await ExecAsync(connection, "UPDATE config.config_service SET agent_actions_enabled = true", ct); var hooks = new AgentActionHooks { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => 2), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => 2), /* Runs in the gate's transaction after the switch check passed and before action_called commits: a person turns the switch off now, so only the write's re-check can see it. */ ReadBefore = async (_, _, token) => @@ -486,7 +486,7 @@ await RunAsync(async (store, connection, ct) => LoadLevels = (_, _, _) => { loads++; - return Task.FromResult>((metric, _) => { seen.Add(metric); return 2; }); + return Task.FromResult>((metric, _, _) => { seen.Add(metric); return 2; }); }, ReadBefore = (_, _, _) => Task.FromResult(null), Write = (_, _, _) => { _writes++; return Task.FromResult(null); }, @@ -553,7 +553,7 @@ CREATE TRIGGER test_fail_every_done BEFORE INSERT ON config.investigation_journa private AgentActionHooks WriteHooks(ServerChangeSignal? signal, bool mark, Func? then = null) => new() { ServerChange = signal, - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => 2), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => 2), ReadBefore = (_, _, _) => Task.FromResult("{\"state\":\"old\"}"), Write = async (_, _, token) => { diff --git a/Darling/Darling.Tests/AgentChangeUndoLiveTests.cs b/Darling/Darling.Tests/AgentChangeUndoLiveTests.cs index abd6aff841..05d2f8a014 100644 --- a/Darling/Darling.Tests/AgentChangeUndoLiveTests.cs +++ b/Darling/Darling.Tests/AgentChangeUndoLiveTests.cs @@ -277,7 +277,7 @@ await RunAsync(async (undo, connection, ct) => var writes = 0; var hooks = new AgentActionHooks { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => 2), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => 2), ReadBefore = (_, _, _) => Task.FromResult(null), Write = (_, _, _) => { writes++; return Task.FromResult("{}"); }, }; diff --git a/Darling/Darling.Tests/AgentLevel2StoreTests.cs b/Darling/Darling.Tests/AgentLevel2StoreTests.cs index fa1c989235..b9b9f249f0 100644 --- a/Darling/Darling.Tests/AgentLevel2StoreTests.cs +++ b/Darling/Darling.Tests/AgentLevel2StoreTests.cs @@ -527,7 +527,7 @@ private static async Task RunTheGateAsMcpAsync(NpgsqlConnection owner, ScratchPo "[{\"tool\":\"get_blocking\",\"arguments\":{},\"key_numbers\":\"12\"}]"), new AgentActionTarget(7, "srv-7", null, null)); AgentActionHooks Hooks(int level) => new() { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => level), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => level), ReadBefore = (_, _, _) => Task.FromResult(null), Write = (_, _, _) => { writes++; return Task.FromResult("{}"); }, }; diff --git a/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs b/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs index 89fb1ae2e4..1603a2082b 100644 --- a/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs +++ b/Darling/Darling.Tests/ExternalAlertDrainLiveTests.cs @@ -506,6 +506,29 @@ await WithDrainAsync(async rig => }); } + /// #6928: an external alert that fired Warning and then climbed to Critical closes the PagerDuty incident at both tiers, so a + /// Warning-only route's key is resolved as well as the Critical-only route's. + [Fact] + public async Task TheClear_AfterAWarningThatEscalatedToCritical_AsksForTheCloseAtBothTiers() + { + await WithDrainAsync(async rig => + { + await rig.SubmitAsync("fire", "disk", server: "SQL01", severity: "warning"); + await rig.Drain.RunPassAsync(rig.Ct); + await rig.SubmitAsync("fire", "disk", server: "SQL01", severity: "critical"); + await rig.Drain.RunPassAsync(rig.Ct); + Assert.Equal("critical", await rig.ScalarAsync("SELECT fired_severity FROM config.external_alert_state WHERE alert_key = 'disk'")); + Assert.Equal("warning,critical", await rig.ScalarAsync("SELECT fired_severities FROM config.external_alert_state WHERE alert_key = 'disk'")); + + var clear = await rig.SubmitAsync("clear", "disk", server: "SQL01"); + await rig.Drain.RunPassAsync(rig.Ct); + + Assert.Equal("delivered", await rig.StatusAsync(clear)); + Assert.Equal( + new AlertSeverityLevel?[] { AlertSeverityLevel.Warning, AlertSeverityLevel.Critical }, rig.PagerDutySeverities.ToArray()); + }); + } + [Fact] public async Task AFireOpenedByARetryThatSucceeds_IsConfirmed_AndItsClearResolvesIt() { @@ -596,10 +619,10 @@ await History.RecordAlertAsync(new AlertHistoryRecord( AlertContextSerializer.Serialize(outcome.Context!))); return delivery; }, - (metric, serverName, serverKey, _, firedSeverity, _) => + (metric, serverName, serverKey, _, firedSeverities, _) => { PagerDutyResolves.Add(metric.Split(':')[2]); - PagerDutySeverities.Add(firedSeverity); + PagerDutySeverities.AddRange(firedSeverities); PagerDutyIdentities.Add(serverName + "|" + serverKey); return Task.FromResult(AlertChannelOutcome.Delivered); }, diff --git a/Darling/Darling.Tests/ExternalAlertRungTests.cs b/Darling/Darling.Tests/ExternalAlertRungTests.cs index 04537ac643..8692c3ad7f 100644 --- a/Darling/Darling.Tests/ExternalAlertRungTests.cs +++ b/Darling/Darling.Tests/ExternalAlertRungTests.cs @@ -127,7 +127,7 @@ public async Task MigratingAFreshStore_CreatesTheTablesAndTheColumn_AndRerunning "pending_id:bigint:NO,source_id:integer:NO,kind:text:NO,alert_key:text:NO,severity:text:NO,title:text:YES,detail:text:YES,server_name:text:YES,value:double precision:YES,submitted_at:timestamp with time zone:NO,expires_at:timestamp with time zone:NO,status:text:NO,claimed_at:timestamp with time zone:YES,service_instance:text:YES,attempts:integer:NO,next_attempt_at:timestamp with time zone:YES,reason:text:YES,completed_at:timestamp with time zone:YES", await ColumnsAsync(connection, "external_alert_pending", ct)); Assert.Equal( - "source_id:integer:NO,alert_key:text:NO,server_label:text:NO,server_id:integer:NO,server_key:text:NO,server_name:text:NO,firing:boolean:NO,fired_severity:text:YES,last_fired_at:timestamp with time zone:YES", + "source_id:integer:NO,alert_key:text:NO,server_label:text:NO,server_id:integer:NO,server_key:text:NO,server_name:text:NO,firing:boolean:NO,fired_severity:text:YES,last_fired_at:timestamp with time zone:YES,fired_severities:text:YES", await ColumnsAsync(connection, "external_alert_state", ct)); Assert.Equal("external_source:text:YES", await ScalarAsync(connection, "SELECT column_name || ':' || data_type || ':' || is_nullable FROM information_schema.columns WHERE table_schema = 'config' AND table_name = 'config_mute_rules' AND column_name = 'external_source'", ct)); diff --git a/Darling/Darling.Tests/InvestigationPracticeTests.cs b/Darling/Darling.Tests/InvestigationPracticeTests.cs index bb858d6a9f..2d37f0669a 100644 --- a/Darling/Darling.Tests/InvestigationPracticeTests.cs +++ b/Darling/Darling.Tests/InvestigationPracticeTests.cs @@ -336,7 +336,7 @@ await RunAsync(async (dataSource, store, ct) => var writes = 0; var hooks = new AgentActionHooks { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => 2), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => 2), ReadBefore = (_, _, _) => Task.FromResult("{}"), Write = (_, _, _) => { writes++; return Task.FromResult("{}"); }, }; diff --git a/Darling/Darling.Tests/NotificationRouteSeverityLiveTests.cs b/Darling/Darling.Tests/NotificationRouteSeverityLiveTests.cs index 948d25601e..7ea97a5bcf 100644 --- a/Darling/Darling.Tests/NotificationRouteSeverityLiveTests.cs +++ b/Darling/Darling.Tests/NotificationRouteSeverityLiveTests.cs @@ -137,6 +137,84 @@ await WithMcpRoleAsync(async (owner, asMcp, ct) => }); } + /// The viewer and web writer: changing the severity filter of an agent route writes a route_agent_changed row naming it with the + /// words before and after, and the same change to a route with no agent level writes nothing. + [Fact] + public async Task Writer_ChangingTheSeverityFilterOfAnAgentRoute_IsJournaled_AndOfARouteWithNoAgentLevelIsNot() + { + Assert.SkipWhen(string.IsNullOrEmpty(ConnectionString), "Set DARLING_TEST_PG to run the route severity journal pin (it mints its own scratch database)."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(ConnectionString!, ct); + await using (var connection = new NpgsqlConnection(scratch.ConnectionString)) + { + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + } + + await using var owner = NpgsqlDataSource.Create(scratch.ConnectionString); + var bodySucceeded = false; + try + { + var proxies = new WebhookProxies(null, null, null, null); + var agentRoute = await NotificationRouteWriter.SaveAsync( + owner, new NotificationRouteDraft { MetricMatch = "Blocking", SmtpRecipients = "ops@example.com", AgentLevel = 2 }, null, proxies, null, null, 30, ct); + var plainRoute = await NotificationRouteWriter.SaveAsync( + owner, new NotificationRouteDraft { MetricMatch = "Deadlocks", SmtpRecipients = "ops@example.com" }, null, proxies, null, null, 30, ct); + const string Journal = "SELECT count(*) FROM config.investigation_journal WHERE event = 'route_agent_changed'"; + + /* Both to warning: Critical alerts move to a broader agent route, so the change is journaled. */ + var stored = (await NotificationRouteWriter.ReadStoredAsync(owner, agentRoute, 30, ct))!; + var typed = stored.Clone(); + typed.Severities = "warning"; + await NotificationRouteWriter.SaveAsync(owner, typed, stored, proxies, null, null, 30, ct, "bob"); + Assert.Equal("1", await ScalarAsync(owner, Journal, ct)); + Assert.Equal("person:bob|changed severities|both|warning", await ScalarAsync(owner, + "SELECT actor || '|' || reason || '|' || (detail->'before'->>'severities') || '|' || (detail->'after'->>'severities') FROM config.investigation_journal WHERE event = 'route_agent_changed'", ct)); + + /* A route with no agent level is not in the agent lookup. */ + var plainStored = (await NotificationRouteWriter.ReadStoredAsync(owner, plainRoute, 30, ct))!; + var plainTyped = plainStored.Clone(); + plainTyped.Severities = "critical"; + await NotificationRouteWriter.SaveAsync(owner, plainTyped, plainStored, proxies, null, null, 30, ct, "bob"); + Assert.Equal("critical", await ScalarAsync(owner, $"SELECT severities FROM config.config_notification_routes WHERE route_id = {plainRoute}", ct)); + Assert.Equal("1", await ScalarAsync(owner, Journal, ct)); + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, static (_, _) => Task.CompletedTask); + } + } + + /// The MCP update tool: an operator who changes the severity filter of an agent route gets a route_agent_changed row (through the + /// update function's before row), and the same change to a route with no agent level writes none. + [Fact] + public async Task Mcp_ChangingTheSeverityFilterOfAnAgentRoute_IsJournaled_AndOfARouteWithNoAgentLevelIsNot() + { + await WithMcpRoleAsync(async (owner, asMcp, ct) => + { + var agentRoute = RouteIdOf(await DarlingMcpAlertTools.CreateNotificationRoute( + asMcp, "Blocking", agent_level: 2, served: OperatorPolicy(), cancellationToken: ct)); + var plainRoute = RouteIdOf(await DarlingMcpAlertTools.CreateNotificationRoute( + asMcp, "Deadlocks", served: OperatorPolicy(), cancellationToken: ct)); + const string Journal = "SELECT count(*) FROM config.investigation_journal WHERE event = 'route_agent_changed'"; + Assert.Equal("1", await ScalarAsync(owner, Journal, ct)); + + Assert.Equal("updated", Status(await DarlingMcpAlertTools.UpdateNotificationRoute( + asMcp, agentRoute, severities: "warning", served: OperatorPolicy(), cancellationToken: ct))); + Assert.Equal("2", await ScalarAsync(owner, Journal, ct)); + Assert.Equal("operator:ops-lead|changed severities|both|warning", await ScalarAsync(owner, + $"SELECT actor || '|' || reason || '|' || (detail->'before'->>'severities') || '|' || (detail->'after'->>'severities') FROM config.investigation_journal WHERE route_id = {agentRoute} AND event = 'route_agent_changed' ORDER BY journal_id DESC LIMIT 1", ct)); + + /* The same word again changes nothing, and a route with no agent level writes no row. */ + await DarlingMcpAlertTools.UpdateNotificationRoute(asMcp, agentRoute, severities: "warning", served: OperatorPolicy(), cancellationToken: ct); + Assert.Equal("updated", Status(await DarlingMcpAlertTools.UpdateNotificationRoute( + asMcp, plainRoute, severities: "critical", served: OperatorPolicy(), cancellationToken: ct))); + Assert.Equal("2", await ScalarAsync(owner, Journal, ct)); + }); + } + private static async Task ReadSeveritiesAsync(NpgsqlDataSource source, int routeId, CancellationToken ct) => JsonDocument.Parse(await DarlingMcpAlertTools.GetNotificationRoutes(source, ct)).RootElement .GetProperty("routes").EnumerateArray().Single(r => r.GetProperty("route_id").GetInt32() == routeId) diff --git a/Darling/Darling.Tests/NotificationRouteSeverityTests.cs b/Darling/Darling.Tests/NotificationRouteSeverityTests.cs index 043dea4ca6..df9667744c 100644 --- a/Darling/Darling.Tests/NotificationRouteSeverityTests.cs +++ b/Darling/Darling.Tests/NotificationRouteSeverityTests.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; @@ -187,6 +188,131 @@ public async Task ThePagerDutyClose_UsesTheFiresSeverity_ToPickTheKey(AlertSever Assert.Equal("resolve", body.GetProperty("event_action").GetString()); } + /// An episode that fired at Warning and climbed to Critical closes the incident on each route's key, and a route that + /// takes both tiers is closed once. + [Fact] + public async Task ThePagerDutyClose_ForAnEpisodeAtBothTiers_ReachesEachKeyOnce() + { + var config = new DarlingConfig(); + config.Webhooks.PagerDutyAutoResolve = true; + config.NotificationRoutes = new[] + { + Route(1, Metric, "critical", pagerDuty: "rk-critical"), + Route(2, Metric, "warning", pagerDuty: "rk-warning"), + }; + var posted = new List(); + var webhooks = new WebhookAlertService(new DarlingAlertSettings(config), DarlingAlertDeliverer.Branding, NullLogger.Instance, new DiscardingHistoryStore()) + { + PostPagerDutyAsyncOverride = (_, payload) => + { + posted.Add(payload); + return Task.FromResult(null); + }, + }; + + var outcome = await webhooks.TrySendPagerDutyResolveAsync( + Metric, "SQL01", "7", "back to normal", new AlertSeverityLevel?[] { AlertSeverityLevel.Warning, AlertSeverityLevel.Critical }, + TestContext.Current.CancellationToken); + + Assert.Equal(AlertChannelOutcome.Delivered, outcome); + Assert.Equal( + new[] { "rk-critical", "rk-warning" }, + posted.Select(p => JsonDocument.Parse(p).RootElement.GetProperty("routing_key").GetString()!).OrderBy(k => k, StringComparer.Ordinal).ToArray()); + + posted.Clear(); + var both = new DarlingConfig(); + both.Webhooks.PagerDutyAutoResolve = true; + both.NotificationRoutes = new[] { Route(1, Metric, "both", pagerDuty: "rk-both") }; + var bothWebhooks = new WebhookAlertService(new DarlingAlertSettings(both), DarlingAlertDeliverer.Branding, NullLogger.Instance, new DiscardingHistoryStore()) + { + PostPagerDutyAsyncOverride = (_, payload) => + { + posted.Add(payload); + return Task.FromResult(null); + }, + }; + await bothWebhooks.TrySendPagerDutyResolveAsync( + Metric, "SQL01", "7", "back to normal", new AlertSeverityLevel?[] { AlertSeverityLevel.Warning, AlertSeverityLevel.Critical }, + TestContext.Current.CancellationToken); + Assert.Single(posted); + } + + /// The deliverer path: Warning-only key A, Critical-only key B, auto-resolve on. The alert fires Warning, escalates to + /// Critical and clears, and the clear resolves the incident on A as well as on B. + [Fact] + public async Task AWarningThatEscalatesToCritical_ClosesThePagerDutyIncidentOnBothKeys() + { + var config = new DarlingConfig(); + config.Webhooks.PagerDutyAutoResolve = true; + config.NotificationRoutes = new[] + { + Route(1, Metric, "warning", pagerDuty: "rk-warning"), + Route(2, Metric, "critical", pagerDuty: "rk-critical"), + }; + var settings = new DarlingAlertSettings(config); + var posted = new List(); + var history = new CapturingHistoryStore(); + var webhooks = new WebhookAlertService(settings, DarlingAlertDeliverer.Branding, NullLogger.Instance, history) + { + PostPagerDutyAsyncOverride = (_, payload) => + { + posted.Add(payload); + return Task.FromResult(null); + }, + }; + var deliverer = new DarlingAlertDeliverer(settings, history, webhooks, NullLogger.Instance); + + await deliverer.DeliverAsync(Fire("7", "SQL01", AlertSeverityLevel.Warning), TestContext.Current.CancellationToken); + await deliverer.DeliverAsync( + Fire("7", "SQL01", AlertSeverityLevel.Critical) with { Context = new AlertContext { EscalationBypassesCooldown = true } }, + TestContext.Current.CancellationToken); + posted.Clear(); + await deliverer.DeliverResolutionAsync( + new AlertResolution("7", "SQL01", Metric, "CPU Resolved", "SQL01: CPU back to normal"), + cancellationToken: TestContext.Current.CancellationToken); + + var resolves = posted + .Select(p => JsonDocument.Parse(p).RootElement) + .Where(e => e.GetProperty("event_action").GetString() == "resolve") + .Select(e => e.GetProperty("routing_key").GetString()!) + .OrderBy(k => k, StringComparer.Ordinal) + .ToArray(); + Assert.Equal(new[] { "rk-critical", "rk-warning" }, resolves); + } + + /// A chat channel: the same escalation sends the clear to the Warning-only channel as well as the Critical-only one. + [Fact] + public async Task AWarningThatEscalatesToCritical_SendsTheClearToTheWarningOnlyChatChannelToo() + { + using var warningSlack = new CapturingWebhookEndpoint(); + using var criticalSlack = new CapturingWebhookEndpoint(); + var config = new DarlingConfig(); + config.Webhooks.SlackSendResolutions = true; + config.NotificationRoutes = new[] + { + Route(1, Metric, "warning", slack: warningSlack.Url), + Route(2, Metric, "critical", slack: criticalSlack.Url), + }; + var settings = new DarlingAlertSettings(config); + var history = new CapturingHistoryStore(); + var webhooks = new WebhookAlertService(settings, DarlingAlertDeliverer.Branding, NullLogger.Instance, history); + var deliverer = new DarlingAlertDeliverer(settings, history, webhooks, NullLogger.Instance); + + await deliverer.DeliverAsync(Fire("7", "SQL01", AlertSeverityLevel.Warning), TestContext.Current.CancellationToken); + await deliverer.DeliverAsync( + Fire("7", "SQL01", AlertSeverityLevel.Critical) with { Context = new AlertContext { EscalationBypassesCooldown = true } }, + TestContext.Current.CancellationToken); + Assert.Single(warningSlack.Bodies); + Assert.Single(criticalSlack.Bodies); + await deliverer.DeliverResolutionAsync( + new AlertResolution("7", "SQL01", Metric, "CPU Resolved", "SQL01: CPU back to normal"), + cancellationToken: TestContext.Current.CancellationToken); + + Assert.Equal(2, warningSlack.Bodies.Count); + Assert.Equal(2, criticalSlack.Bodies.Count); + Assert.Contains("Resolved", warningSlack.Bodies[1], StringComparison.Ordinal); + } + private static AlertOutcome Fire(string serverKey, string server, AlertSeverityLevel severity) => new(serverKey, server, Metric, "97%", "90%", Context: null, DetailText: null, NumericCurrentValue: 97, NumericThresholdValue: 90, Muted: false, Severity: severity); diff --git a/Darling/Darling.Tests/PlanForceCheckLiveTests.cs b/Darling/Darling.Tests/PlanForceCheckLiveTests.cs index 48946d7bfa..af37a674c7 100644 --- a/Darling/Darling.Tests/PlanForceCheckLiveTests.cs +++ b/Darling/Darling.Tests/PlanForceCheckLiveTests.cs @@ -363,7 +363,7 @@ await RunAsync(async (dataSource, connection, at, ct) => new AgentActionTarget(ServerId, "srv-7", NewPlan.ToString(CultureInfo.InvariantCulture), Database)); var hooks = new ServerActionHooks { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => 3), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => 3), ReadCredential = (serverId, token) => store.ReadServerCredentialStateAsync(dataSource, serverId, token), }; var decision = await store.AuthorizeServerActionAsync(call, hooks, ct); diff --git a/Darling/Darling.Tests/RungRewindArtifacts.cs b/Darling/Darling.Tests/RungRewindArtifacts.cs index 741c07c359..6c21dd8120 100644 --- a/Darling/Darling.Tests/RungRewindArtifacts.cs +++ b/Darling/Darling.Tests/RungRewindArtifacts.cs @@ -580,7 +580,9 @@ ALTER TABLE config.config_alert_settings // V229 (#6928): the severity filter on the notification routes. The column is the probe's newest sentinel. The rewind drops the column // (its CHECK goes with it), so a re-run of V229 does everything again. Safe on a store that never had it. new(229, "hasRouteSeverities", - @"ALTER TABLE config.config_notification_routes DROP COLUMN IF EXISTS severities;"), + @"ALTER TABLE config.config_notification_routes DROP COLUMN IF EXISTS severities; +ALTER TABLE config.external_alert_state DROP COLUMN IF EXISTS fired_severities; +ALTER TABLE config.investigation_request_alerts DROP COLUMN IF EXISTS severity;"), ]; /// diff --git a/Darling/Darling.Tests/ServerActionGateTests.cs b/Darling/Darling.Tests/ServerActionGateTests.cs index 37e5637457..ba78a3ea96 100644 --- a/Darling/Darling.Tests/ServerActionGateTests.cs +++ b/Darling/Darling.Tests/ServerActionGateTests.cs @@ -132,7 +132,7 @@ private static ServerActionCall Call(Guid request, Guid claim, string action = S private static ServerActionHooks Hooks(InvestigationRequestStore store, NpgsqlDataSource source, int? resolved = 3, Action? onCredential = null) => new() { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => resolved), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => resolved), ReadCredential = (serverId, ct) => { onCredential?.Invoke(); @@ -293,7 +293,7 @@ await RunAsync(async (store, connection, source, ct) => var writes = 0; var hooks = new AgentActionHooks { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => level), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => level), ReadBefore = (_, _, _) => Task.FromResult("{\"reason\":\"old\"}"), Write = (_, _, _) => { writes++; return Task.FromResult("{\"reason\":\"new\"}"); }, }; @@ -314,7 +314,7 @@ await RunAsync(async (store, connection, source, ct) => /// write's own transaction opens, which is where a person changes something mid-flight. private AgentActionHooks WriteHooks(Func resolved, Func inFlight) => new() { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => resolved()), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => resolved()), ReadBefore = async (_, _, token) => { await inFlight(token); @@ -465,7 +465,7 @@ await RunAsync(async (store, connection, source, ct) => await using var read = new NpgsqlCommand( "SELECT CASE WHEN EXISTS (SELECT 1 FROM config.server_tag_map WHERE server_id = 7) THEN agent_level ELSE 2 END FROM config.config_notification_routes WHERE route_id = 4001", c, t); var level = Convert.ToInt32(await read.ExecuteScalarAsync(token), CultureInfo.InvariantCulture); - return (_, _) => level; + return (_, _, _) => level; }, ReadBefore = (_, _, _) => Task.FromResult("{\"reason\":\"old\"}"), Write = async (_, _, token) => @@ -557,7 +557,7 @@ await RunAsync(async (store, connection, source, ct) => var bothReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); AgentActionHooks Hooks(string key) => new() { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => 3), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => 3), ReadBefore = (_, _, _) => Task.FromResult(null), Write = async (c, t, token) => { @@ -594,7 +594,7 @@ await RunAsync(async (store, connection, source, ct) => var hooks = new AgentActionHooks { ServerChange = signal, - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => 3), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => 3), ReadBefore = (_, _, _) => Task.FromResult("{\"reason\":\"old\"}"), Write = async (_, _, token) => { @@ -628,7 +628,7 @@ await RunAsync(async (store, connection, source, ct) => var hooks = new AgentActionHooks { ServerChange = new ServerChangeSignal(), - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => 3), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => 3), ReadBefore = (_, _, _) => Task.FromResult("{\"reason\":\"old\"}"), Write = async (_, _, token) => { diff --git a/Darling/Darling.Tests/ServerActionSwitchWriteLiveTests.cs b/Darling/Darling.Tests/ServerActionSwitchWriteLiveTests.cs index 016e89b16b..9cd16bb2ae 100644 --- a/Darling/Darling.Tests/ServerActionSwitchWriteLiveTests.cs +++ b/Darling/Darling.Tests/ServerActionSwitchWriteLiveTests.cs @@ -149,7 +149,7 @@ await WithStoreAsync(async (owner, ct) => new AgentActionTarget(SqlServerId, "alpha-sql-01", null, null)); var hooks = new ServerActionHooks { - LoadLevels = (_, _, _) => Task.FromResult>((_, _) => 3), + LoadLevels = (_, _, _) => Task.FromResult>((_, _, _) => 3), ReadCredential = (serverId, token) => store.ReadServerCredentialStateAsync(owner, serverId, token), }; diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs index b27f3abe38..271f4b9bb2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs @@ -112,8 +112,8 @@ public DarlingAlertDeliverer( /// public Task TryResolvePagerDutyAsync( string firingMetric, string serverName, string serverKey, string? detailText, - AlertSeverityLevel? firedSeverity = null, CancellationToken cancellationToken = default) => - _webhooks.TrySendPagerDutyResolveAsync(firingMetric, serverName, serverKey, detailText, firedSeverity, cancellationToken); + IReadOnlyCollection firedSeverities, CancellationToken cancellationToken = default) => + _webhooks.TrySendPagerDutyResolveAsync(firingMetric, serverName, serverKey, detailText, firedSeverities, cancellationToken); public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) => DeliverAndReportAsync(outcome, cancellationToken); @@ -338,7 +338,22 @@ private sealed class DeliveredFire /// against, read the way the viewer reads a row (). public string? DetailText { get; set; } - public Dictionary Channels { get; } = new(StringComparer.Ordinal); + /// Every destination each channel was fired to during the episode, each once (#6928). A severity filter can send a + /// Warning fire and the Critical fire after it to different routes, and the clear owes each of them its close. + public Dictionary> Channels { get; } = new(StringComparer.Ordinal); + + public void AddDestination(string channel, string destination, int? routeId) + { + if (!Channels.TryGetValue(channel, out var list)) + { + Channels[channel] = list = new List<(string Destination, int? RouteId)>(); + } + + if (!list.Contains((destination, routeId))) + { + list.Add((destination, routeId)); + } + } public HashSet PagerDutyDedupKeys { get; } = new(StringComparer.Ordinal); @@ -411,7 +426,7 @@ private void NoteDeliveredFire( fire.DetailText = detailText; foreach (var (channel, where, routeId) in delivered) { - fire.Channels[channel] = (where, routeId); + fire.AddDestination(channel, where, routeId); if (channel == NotificationRouter.PagerDutyChannel) { fire.PagerDutyDedupKeys.Add(dedupKey); @@ -553,33 +568,38 @@ private async Task SendResolutionAsync( var routeGone = new List(); foreach (var channel in s_channelOrder) { - if (!fire.Channels.TryGetValue(channel, out var reached)) + if (!fire.Channels.TryGetValue(channel, out var reachedList)) { continue; } - /* The route that carried the fire, as it is now. Deleted or disabled since the fire: the same answer as the - history path below (#5855 review), so the clear goes nowhere on that channel and the row says why. */ - var route = RouteNow(reached.RouteId, routes, out var gone); - if (gone) + /* #6928: a severity filter can fire one channel to different destinations during an episode, so each is closed. */ + foreach (var reached in reachedList) { - if (!fire.RouteGoneChannels.Contains(channel)) + /* The route that carried the fire, as it is now. Deleted or disabled since the fire: the same answer as the + history path below (#5855 review), so the clear goes nowhere on that destination and the row says why. */ + var route = RouteNow(reached.RouteId, routes, out var gone); + if (gone) { - fire.RouteGoneChannels.Add(channel); - } + if (!fire.RouteGoneChannels.Contains(channel)) + { + fire.RouteGoneChannels.Add(channel); + } - continue; - } + continue; + } - if (channel == NotificationRouter.PagerDutyChannel - && (!options.ClosePagerDuty || fire.PagerDutyDedupKeys.Count == 0)) - { - continue; - } + if (channel == NotificationRouter.PagerDutyChannel + && (!options.ClosePagerDuty || fire.PagerDutyDedupKeys.Count == 0)) + { + continue; + } - if (NotificationRouteResolutions.SendsResolutions(route, channel, flags)) - { - targets.Add((channel, reached.Destination)); + if (NotificationRouteResolutions.SendsResolutions(route, channel, flags) + && !targets.Contains((channel, reached.Destination))) + { + targets.Add((channel, reached.Destination)); + } } } @@ -659,7 +679,12 @@ the clear. */ error = await _webhooks.TrySendResolutionAsync(channel, destination, facts, serverId, genericDedupKey: null, cancellationToken); } - outcomes[channel] = error is null ? AlertChannelOutcome.Delivered : AlertChannelOutcome.Failed; + /* A channel with several destinations is Failed when any of them failed. */ + if (error is not null || !outcomes.ContainsKey(channel)) + { + outcomes[channel] = error is null ? AlertChannelOutcome.Delivered : AlertChannelOutcome.Failed; + } + if (error is not null && channel != NotificationRouter.EmailChannel) { webhookError ??= $"{channel}: {error}"; @@ -746,7 +771,7 @@ the clear. */ }; string? firstDedupKey = null; - /* Newest first: a channel takes the destination its newest delivered fire used, as the memory path does. */ + /* Every delivered fire of the episode adds its destination, as the memory path does (#6928). */ foreach (var row in history.Rows) { if (AlertContextSerializer.TryReadRoute(row.ContextJson) is not { } route) @@ -768,25 +793,30 @@ matches the trigger whenever the #3313 filter dropped none of them. The triage l #3313 filter dropped none of them. */ var genericDedupKey = destination.SentKey ?? WebhookAlertService.DeriveGenericDedupKey(resolution.ServerKey, resolution.ServerName, resolution.MetricName, context); - if (destination.Outcome != AlertRouteOutcomes.Delivered - || fire.Channels.ContainsKey(destination.Channel) || fire.RouteGoneChannels.Contains(destination.Channel)) + /* #6928: every delivered fire's destination counts, not only the newest, because a severity filter can send the + Warning fire and the Critical fire of one episode to different routes, and each owes its close. */ + if (destination.Outcome != AlertRouteOutcomes.Delivered) { - if (destination.Outcome == AlertRouteOutcomes.Delivered && destination.Channel == NotificationRouter.PagerDutyChannel) - { - fire.PagerDutyDedupKeys.Add(dedupKey); - } - else if (destination.Outcome == AlertRouteOutcomes.Delivered && destination.Channel == NotificationRouter.GenericChannel) - { - fire.GenericDedupKeys.Add(genericDedupKey); - } - continue; } + if (destination.Channel == NotificationRouter.PagerDutyChannel) + { + fire.PagerDutyDedupKeys.Add(dedupKey); + } + else if (destination.Channel == NotificationRouter.GenericChannel) + { + fire.GenericDedupKeys.Add(genericDedupKey); + } + var where = CurrentDestination(destination.Channel, destination.RouteId, routes, out var gone); if (gone) { - fire.RouteGoneChannels.Add(destination.Channel); + if (!fire.RouteGoneChannels.Contains(destination.Channel)) + { + fire.RouteGoneChannels.Add(destination.Channel); + } + continue; } @@ -795,15 +825,7 @@ matches the trigger whenever the #3313 filter dropped none of them. The triage l continue; } - fire.Channels[destination.Channel] = (where, destination.RouteId); - if (destination.Channel == NotificationRouter.PagerDutyChannel) - { - fire.PagerDutyDedupKeys.Add(dedupKey); - } - else if (destination.Channel == NotificationRouter.GenericChannel) - { - fire.GenericDedupKeys.Add(genericDedupKey); - } + fire.AddDestination(destination.Channel, where, destination.RouteId); } } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs index 26ea2ae9e2..12bc4d5212 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs @@ -2998,7 +2998,8 @@ AND NOT EXISTS (SELECT 1 FROM {config}.config_notification_routes WHERE route_id RETURN jsonb_build_object( 'metric_match', btrim(v_row.metric_match), 'scope_server_id', v_row.scope_server_id, 'scope_tag_id', v_row.scope_tag_id, 'agent_level', v_row.agent_level, 'agent_max_per_hour', v_row.agent_max_per_hour, 'agent_max_per_day', v_row.agent_max_per_day, - 'agent_lease_minutes', v_row.agent_lease_minutes, 'agent_expiry_minutes', v_row.agent_expiry_minutes, 'enabled', v_row.enabled); + 'agent_lease_minutes', v_row.agent_lease_minutes, 'agent_expiry_minutes', v_row.agent_expiry_minutes, 'enabled', v_row.enabled, + 'severities', v_row.severities); END $fn$; REVOKE ALL ON FUNCTION {config}.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer, text) FROM PUBLIC; @@ -3083,6 +3084,10 @@ IF p_before IS NOT NULL THEN IF (p_before ->> 'metric_match') IS DISTINCT FROM btrim(v_row.metric_match) THEN v_changed := array_append(v_changed, 'metric_match'); END IF; IF (p_before ->> 'scope_server_id')::integer IS DISTINCT FROM v_row.scope_server_id THEN v_changed := array_append(v_changed, 'scope_server_id'); END IF; IF (p_before ->> 'scope_tag_id')::integer IS DISTINCT FROM v_row.scope_tag_id THEN v_changed := array_append(v_changed, 'scope_tag_id'); END IF; + -- #6928: the severity filter moves alerts between agent routes, so it is named like the match and the scope. Only a route with an agent setting. + IF (p_before ->> 'severities') IS DISTINCT FROM v_row.severities AND ((p_before ->> 'agent_level') IS NOT NULL OR v_level IS NOT NULL) THEN + v_changed := array_append(v_changed, 'severities'); + END IF; END IF; IF v_level IS DISTINCT FROM v_row.agent_level THEN v_changed := array_append(v_changed, 'agent_level'); END IF; IF v_hour IS DISTINCT FROM v_row.agent_max_per_hour THEN v_changed := array_append(v_changed, 'agent_max_per_hour'); END IF; @@ -3108,11 +3113,13 @@ IF p_before IS NOT NULL AND ((p_before ->> 'agent_level') IS NOT NULL OR v_level v_before := CASE WHEN COALESCE(p_created, false) THEN NULL WHEN p_before IS NOT NULL THEN p_before ELSE jsonb_build_object( 'metric_match', btrim(v_row.metric_match), 'scope_server_id', v_row.scope_server_id, 'scope_tag_id', v_row.scope_tag_id, 'agent_level', v_row.agent_level, 'agent_max_per_hour', v_row.agent_max_per_hour, 'agent_max_per_day', v_row.agent_max_per_day, - 'agent_lease_minutes', v_row.agent_lease_minutes, 'agent_expiry_minutes', v_row.agent_expiry_minutes, 'enabled', v_row.enabled) END; + 'agent_lease_minutes', v_row.agent_lease_minutes, 'agent_expiry_minutes', v_row.agent_expiry_minutes, 'enabled', v_row.enabled, + 'severities', v_row.severities) END; v_after := jsonb_build_object( 'metric_match', btrim(v_row.metric_match), 'scope_server_id', v_row.scope_server_id, 'scope_tag_id', v_row.scope_tag_id, 'agent_level', v_level, 'agent_max_per_hour', v_hour, 'agent_max_per_day', v_day, - 'agent_lease_minutes', v_lease, 'agent_expiry_minutes', v_expiry, 'enabled', v_row.enabled); + 'agent_lease_minutes', v_lease, 'agent_expiry_minutes', v_expiry, 'enabled', v_row.enabled, + 'severities', v_row.severities); INSERT INTO {config}.investigation_journal (event_time, request_id, route_id, server_id, event, actor, reason, detail) VALUES (now(), NULL, p_route_id, v_row.scope_server_id, 'route_agent_changed', p_actor, CASE WHEN COALESCE(p_created, false) THEN 'created route' ELSE 'changed ' || array_to_string(v_changed, ', ') END, diff --git a/Darling/PerformanceMonitor.Darling.Service/ExternalAlertDrain.cs b/Darling/PerformanceMonitor.Darling.Service/ExternalAlertDrain.cs index 00ffe85a02..f6f3e4a48f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/ExternalAlertDrain.cs +++ b/Darling/PerformanceMonitor.Darling.Service/ExternalAlertDrain.cs @@ -75,7 +75,7 @@ public sealed class ExternalAlertDrain private readonly ExternalAlertStore _store; private readonly Func _alertsEnabled; private readonly Func> _deliver; - private readonly Func> _resolvePagerDuty; + private readonly Func, CancellationToken, Task> _resolvePagerDuty; private readonly IAlertHistoryStore _history; private readonly Func _isMuted; private readonly Func _resolveServer; @@ -86,7 +86,7 @@ public sealed class ExternalAlertDrain /// The live master switch every alert family consults (#3464). Off, a pass only expires rows. /// The record-and-send seam: . - /// (firing metric, server name, server key, detail, the severity the fire went out at, token): closes the PagerDuty incident + /// (firing metric, server name, server key, detail, every severity the episode's fires went out at, token): closes the PagerDuty incident /// of a fire, and only with auto-resolve on: . /// The mute check every family shares (MuteRuleService.IsAlertMuted). /// Maps the submitted server name to a monitored server, or null when it is not registered; @@ -95,7 +95,7 @@ public ExternalAlertDrain( ExternalAlertStore store, Func alertsEnabled, Func> deliver, - Func> resolvePagerDuty, + Func, CancellationToken, Task> resolvePagerDuty, IAlertHistoryStore history, Func isMuted, Func resolveServer, @@ -361,6 +361,23 @@ await TryMarkAsync(row.PendingId, ExternalAlertOutcome.Delivered, : string.Equals(firedSeverity, "warning", StringComparison.OrdinalIgnoreCase) ? AlertSeverityLevel.Warning : null; + /// Every tier an episode's fires went out at (#6928), so the close reaches the destination of each tier's route: a + /// Warning-only route's incident stays open when the alert climbs to Critical unless the close names Warning too. An episode + /// opened before the list was kept has only the newest word. Empty (one close with no tier) when none was confirmed. + private static List SeveritiesOfFired(ExternalAlertFiringState state) + { + var tiers = new List(); + foreach (var word in (state.FiredSeverities ?? state.FiredSeverity ?? "").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (SeverityOfFired(word) is { } tier && !tiers.Contains(tier)) + { + tiers.Add(tier); + } + } + + return tiers; + } + private async Task HandleClearAsync(ExternalAlertPendingRow row, CancellationToken cancellationToken) { var server = Identify(row); @@ -384,7 +401,7 @@ fire must not move the incident this clear closes. */ /* The PagerDuty resolve goes first: it posts only with auto-resolve on and a PagerDuty destination for this alert, and if it fails the firing state is left alone so the retry still finds a fire to close. */ var pagerDuty = await _resolvePagerDuty( - firingMetric, firedLabel, firedKey, message, SeverityOfFired(state.FiredSeverity), cancellationToken); + firingMetric, firedLabel, firedKey, message, SeveritiesOfFired(state), cancellationToken); if (pagerDuty == AlertChannelOutcome.Failed) { return await RetryOrFailAsync(row, "the PagerDuty resolve failed"); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAgentActionGate.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAgentActionGate.cs index 814577d8e4..24f10cbe6e 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAgentActionGate.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAgentActionGate.cs @@ -116,11 +116,11 @@ internal static string RefusalAnswer(AgentActionRefusal refusal) => /// that one list, so a route an operator lowered refuses the next change, not the next reload. A failed tag read logs /// through and leaves a tag-scoped route unexpanded, so it matches nothing: fail closed. /// - internal static async Task> LoadLevelsAsync(NpgsqlConnection connection, ILogger logger, + internal static async Task> LoadLevelsAsync(NpgsqlConnection connection, ILogger logger, CancellationToken cancellationToken) { var routes = await StoreConfigProvider.ReadNotificationRoutesAsync(connection, logger, cancellationToken); - return (metric, serverId) => NotificationRouter.ResolveAgent(metric, routes, serverId)?.AgentLevel; + return (metric, serverId, severity) => NotificationRouter.ResolveAgent(metric, routes, serverId, severity)?.AgentLevel; } /// The request id of a refused argument, when one parses, so its refusal is journaled with its request. diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.RouteAdmin.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.RouteAdmin.cs index 3534215f48..8f0fe6d719 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.RouteAdmin.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.RouteAdmin.cs @@ -370,7 +370,7 @@ internal static async Task UpdateNotificationRouteAsync( /* The Viewer journals a match, scope or enabled change to a route with an agent setting (NotificationRouteWriter.AgentChange), so an operator's change to one is journaled too, by the agent function, with the before row the update function returns. */ var journalsMatchOrFlag = operatorActor is not null && existing.AgentLevel is not null - && columns.Any(c => c is "metric_match" or "scope" or "enabled"); + && columns.Any(c => c is "metric_match" or "scope" or "enabled" or "severities"); if (agent.Any) { diff --git a/Darling/PerformanceMonitor.Darling.Storage/ExternalAlertStore.cs b/Darling/PerformanceMonitor.Darling.Storage/ExternalAlertStore.cs index 8ab14a81a6..aa48d152bf 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/ExternalAlertStore.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/ExternalAlertStore.cs @@ -87,7 +87,8 @@ public sealed record ExternalAlertFireResult( /// the incident under these, whatever the server registry says by then. /// public sealed record ExternalAlertFiringState( - bool Firing, string? FiredSeverity, DateTime? LastFiredAtUtc, int ServerId, string ServerKey, string ServerName); + bool Firing, string? FiredSeverity, DateTime? LastFiredAtUtc, int ServerId, string ServerKey, string ServerName, + string? FiredSeverities = null); /// /// The service's side of the external alert queue (#5654): claims what a source submitted through @@ -299,7 +300,7 @@ UPDATE config.external_alert_pending /// Reads one subject's firing state. $1 source_id, $2 alert_key, $3 server_label (the normalized submitted name). public const string GetStateSql = @" -SELECT firing, fired_severity, last_fired_at, server_id, server_key, server_name +SELECT firing, fired_severity, last_fired_at, server_id, server_key, server_name, fired_severities FROM config.external_alert_state WHERE source_id = $1 AND alert_key = $2 AND server_label = $3"; @@ -329,6 +330,7 @@ ON CONFLICT (source_id, alert_key, server_label) DO UPDATE SET server_key = CASE WHEN config.external_alert_state.firing THEN config.external_alert_state.server_key ELSE EXCLUDED.server_key END, server_name = CASE WHEN config.external_alert_state.firing THEN config.external_alert_state.server_name ELSE EXCLUDED.server_name END, fired_severity = CASE WHEN config.external_alert_state.firing THEN config.external_alert_state.fired_severity ELSE NULL END, + fired_severities = CASE WHEN config.external_alert_state.firing THEN config.external_alert_state.fired_severities ELSE NULL END, firing = true, last_fired_at = EXCLUDED.last_fired_at RETURNING server_id, server_key, server_name @@ -339,10 +341,16 @@ SELECT COALESCE((SELECT firing FROM prev), false), (SELECT fired_severity FROM p /// /// Records that a fire's send went out (#5654), so the severity is the one the operator was told. $1 source_id, /// $2 alert_key, $3 server_label, $4 severity. Touches only a subject that is firing. + /// fired_severities (#6928) is every severity sent during the open episode, comma separated, each once, so the + /// clear can close what a Warning-only route opened after the alert climbed to Critical. fired_severity stays the newest. /// public const string ConfirmFireSql = @" UPDATE config.external_alert_state -SET fired_severity = $4 +SET fired_severity = $4, + fired_severities = CASE + WHEN fired_severities IS NULL THEN $4::text + WHEN position(',' || $4::text || ',' IN ',' || fired_severities || ',') > 0 THEN fired_severities + ELSE fired_severities || ',' || $4::text END WHERE source_id = $1 AND alert_key = $2 AND server_label = $3 AND firing"; @@ -672,7 +680,8 @@ public async Task ReapAsync(TimeSpan claimedFor, CancellationToken cancella reader.IsDBNull(2) ? null : reader.GetFieldValue(2), reader.GetInt32(3), reader.GetString(4), - reader.GetString(5)); + reader.GetString(5), + reader.IsDBNull(6) ? null : reader.GetString(6)); } /// diff --git a/Darling/PerformanceMonitor.Darling.Storage/InvestigationPracticeStore.cs b/Darling/PerformanceMonitor.Darling.Storage/InvestigationPracticeStore.cs index 0c54dbe3c5..82e9b865e9 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/InvestigationPracticeStore.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/InvestigationPracticeStore.cs @@ -179,8 +179,8 @@ INSERT INTO config.investigation_requests /// The case's alert rows, copied without the dedup key. $1 new request, $2 case. internal const string CopyAlertsSql = @" INSERT INTO config.investigation_request_alerts - (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json) -SELECT $1, alert_time, server_id, metric_name, NULL, current_value, threshold_value, muted, detail_text, context_json + (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json, severity) +SELECT $1, alert_time, server_id, metric_name, NULL, current_value, threshold_value, muted, detail_text, context_json, severity FROM config.investigation_request_alerts WHERE request_id = $2"; diff --git a/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.AgentAction.cs b/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.AgentAction.cs index bddba77ca9..bc297e3b36 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.AgentAction.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.AgentAction.cs @@ -16,6 +16,7 @@ using Npgsql; using NpgsqlTypes; using PerformanceMonitor.Common; +using PerformanceMonitor.Notifications; namespace PerformanceMonitor.Darling.Storage; @@ -62,7 +63,7 @@ public sealed class AgentActionHooks /// at most once per gate pass, so a request with several metrics does not read the route table (and its scope inputs) again /// for each one. /// - public required Func>> LoadLevels { get; init; } + public required Func>> LoadLevels { get; init; } /// Check 7, the kind rule; null when the kind has none. public Func>? KindRule { get; init; } @@ -160,7 +161,7 @@ FROM config.investigation_requests /// Every metric that opened or joined the request, for the re-resolve: the lowest level wins. internal const string AgentActionMetricsSql = @" -SELECT DISTINCT metric_name +SELECT DISTINCT metric_name, severity FROM config.investigation_request_alerts WHERE request_id = $1"; @@ -528,18 +529,18 @@ await ServerActionSwitchLock.AcquireSharedAsync(connection, transaction, call.Ta /// after the request opened resolves to null and lowers the request to 0: fail closed (pinned live). /// private static async Task ResolveRequestLevelAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, - AgentActionRequest request, Func>> loadLevels, + AgentActionRequest request, Func>> loadLevels, int floor, CancellationToken cancellationToken) { var level = request.AutonomyLevel; if (level >= floor) { var resolve = await loadLevels(connection, transaction, cancellationToken); - foreach (var metric in request.Metrics) + foreach (var (metric, severity) in request.Metrics) { if (level < floor) break; - level = Math.Min(level, resolve(metric, request.ServerId) ?? 0); + level = Math.Min(level, resolve(metric, request.ServerId, severity) ?? 0); } } @@ -548,7 +549,7 @@ private static async Task ResolveRequestLevelAsync(NpgsqlConnection connect private sealed record AgentActionRequest(string State, Guid? ClaimId, string? ClaimedBy, DateTime? LeaseUntil, DateTime ExpiresAt, int ServerId, string ServerName, Guid? PracticeOf, int AutonomyLevel, int ClaimCount, - IReadOnlyList Metrics); + IReadOnlyList<(string Metric, AlertSeverityLevel? Severity)> Metrics); private async Task ReadAgentActionRequestAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, Guid requestId, CancellationToken cancellationToken) @@ -572,21 +573,30 @@ private sealed record AgentActionRequest(string State, Guid? ClaimId, string? Cl reader.IsDBNull(7) ? null : reader.GetGuid(7), reader.GetInt16(8), reader.GetInt32(9), - Array.Empty()); + Array.Empty<(string, AlertSeverityLevel?)>()); } - var metrics = new List { firstMetric }; + /* #6928: each alert's metric at the tier it arrived at, so the live routes are read at the tier dispatch used: a Critical alert + dispatched under one route is not re-resolved under a Warning-only route that dispatch skipped. The first metric stands + alone only when no alert row names it (an old request), and resolves with no tier. */ + var metrics = new List<(string Metric, AlertSeverityLevel? Severity)>(); await using (var command = Command(connection, transaction, AgentActionMetricsSql, P(requestId))) await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) { while (await reader.ReadAsync(cancellationToken)) { var metric = reader.GetString(0); - if (!metrics.Contains(metric, StringComparer.Ordinal)) - metrics.Add(metric); + var severity = reader.IsDBNull(1) ? null + : string.Equals(reader.GetString(1), InvestigationEvents.Critical, StringComparison.Ordinal) ? AlertSeverityLevel.Critical + : (AlertSeverityLevel?)AlertSeverityLevel.Warning; + if (!metrics.Contains((metric, severity))) + metrics.Add((metric, severity)); } } + if (!metrics.Any(m => string.Equals(m.Metric, firstMetric, StringComparison.Ordinal))) + metrics.Insert(0, (firstMetric, null)); + return row with { Metrics = metrics }; } diff --git a/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.ServerAction.cs b/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.ServerAction.cs index 58a32c6fc9..206cb584cb 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.ServerAction.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.ServerAction.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Npgsql; using PerformanceMonitor.Common; +using PerformanceMonitor.Notifications; namespace PerformanceMonitor.Darling.Storage; @@ -24,7 +25,7 @@ public sealed record ServerActionCall(string Tool, string ServerAction, AgentAct /// . It never resolves the secret. public sealed class ServerActionHooks { - public required Func>> LoadLevels { get; init; } + public required Func>> LoadLevels { get; init; } public required Func> ReadCredential { get; init; } } @@ -131,7 +132,7 @@ await JournalRefusedAsync(connection, transaction, now, request is null ? null : /// private async Task ServerActionConditionsRefusalAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, AgentActionRequest? request, string serverAction, AgentAction action, AgentActionTarget target, - Func>> loadLevels, DateTime now, bool practice, + Func>> loadLevels, DateTime now, bool practice, CancellationToken cancellationToken) { if (ClaimOrPracticeRefusal(request, action, now, practice) is { } claimRefusal) diff --git a/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.cs b/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.cs index be2a94a2a0..52ab9a7177 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.cs @@ -125,8 +125,8 @@ INSERT INTO config.investigation_requests internal const string AlertRowsSql = @" INSERT INTO config.investigation_request_alerts - (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json) -SELECT $1, $2, $3, $4, k, $6, $7, $8, $9, $10 + (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json, severity) +SELECT $1, $2, $3, $4, k, $6, $7, $8, $9, $10, $11 FROM unnest($5::text[]) AS k"; internal const string JournalSql = @" @@ -270,8 +270,8 @@ FROM config.investigation_requests AS r /* #6008: the source's alert rows, without the dedup key, so the brief's alerts section reads what the source's read. $1 new, $2 source. */ internal const string CopyFollowUpAlertsSql = @" INSERT INTO config.investigation_request_alerts - (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json) -SELECT $1, alert_time, server_id, metric_name, NULL, current_value, threshold_value, muted, detail_text, context_json + (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json, severity) +SELECT $1, alert_time, server_id, metric_name, NULL, current_value, threshold_value, muted, detail_text, context_json, severity FROM config.investigation_request_alerts WHERE request_id = $2"; @@ -1656,7 +1656,9 @@ private Task InsertAlertRowsAsync(NpgsqlConnection connection, NpgsqlTransa new NpgsqlParameter { TypedValue = facts?.ThresholdValue, NpgsqlDbType = NpgsqlDbType.Double }, new NpgsqlParameter { TypedValue = facts?.Muted, NpgsqlDbType = NpgsqlDbType.Boolean }, new NpgsqlParameter { TypedValue = DerivedStoreScrub.Text(facts?.DetailText), NpgsqlDbType = NpgsqlDbType.Text }, - new NpgsqlParameter { TypedValue = DerivedStoreScrub.Json(facts?.ContextJson), NpgsqlDbType = NpgsqlDbType.Text }); + new NpgsqlParameter { TypedValue = DerivedStoreScrub.Json(facts?.ContextJson), NpgsqlDbType = NpgsqlDbType.Text }, + /* #6928: the tier this alert arrived at, so the agent-action gate resolves its route at that tier. */ + new NpgsqlParameter { TypedValue = AlertSeverityWord(alert.Severity), NpgsqlDbType = NpgsqlDbType.Text }); } /// diff --git a/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs b/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs index 3981a55ebc..be655c035e 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/NotificationRouteWriter.cs @@ -816,6 +816,12 @@ void Compare(string name, object? was, object? now) Compare("metric_match", before?.MetricMatch.Trim(), after?.MetricMatch.Trim()); Compare("scope_server_id", before?.ScopeServerId, after?.ScopeServerId); Compare("scope_tag_id", before?.ScopeTagId, after?.ScopeTagId); + /* #6928: the severity filter moves alerts between agent routes. A route with no agent setting is not in the agent lookup. */ + if (before?.AgentLevel is not null || after?.AgentLevel is not null) + { + Compare("severities", NotificationRouteKinds.NormalizeSeverities(before?.Severities), NotificationRouteKinds.NormalizeSeverities(after?.Severities)); + } + Compare("agent_level", before?.AgentLevel, after?.AgentLevel); Compare("agent_max_per_hour", before?.AgentMaxPerHour, after?.AgentMaxPerHour); Compare("agent_max_per_day", before?.AgentMaxPerDay, after?.AgentMaxPerDay); @@ -842,6 +848,7 @@ void Compare(string name, object? was, object? now) ["agent_lease_minutes"] = row.AgentLeaseMinutes, ["agent_expiry_minutes"] = row.AgentExpiryMinutes, ["enabled"] = row.Enabled, + ["severities"] = NotificationRouteKinds.NormalizeSeverities(row.Severities), }; var detail = new System.Text.Json.Nodes.JsonObject { ["before"] = Describe(before), ["after"] = Describe(after) }; diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index 984edc5715..978b9e2735 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -3009,10 +3009,19 @@ ALTER TABLE collect.servers /// the default, so every stored route reads both and keeps its effect, and a CHECK holds the three words. An alert that is not /// Critical counts as Warning. A route whose filter excludes an alert does not match it, and matching goes on to the next route. /// Idempotent: the column and the constraint are each added only when absent. No backfill and no view refresh: a registry table. + /// Also config.external_alert_state.fired_severities, every severity an external alert sent during its open episode + /// (comma separated), so its clear closes the incident of each tier's route. Null on an episode already open, which reads as + /// the newest severity alone. And config.investigation_request_alerts.severity, the tier (critical or warning) each + /// alert row of an agent request arrived at, so the agent-action gate re-resolves each alert's route at the tier dispatch used. Null on + /// a row written before, which resolves with no tier. /// private const string V229Sql = @" ALTER TABLE config.config_notification_routes ADD COLUMN IF NOT EXISTS severities text NOT NULL DEFAULT 'both'; +ALTER TABLE config.external_alert_state + ADD COLUMN IF NOT EXISTS fired_severities text; +ALTER TABLE config.investigation_request_alerts + ADD COLUMN IF NOT EXISTS severity text; DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'ck_config_notification_routes_severities' diff --git a/Darling/tools/provision-roles.sql b/Darling/tools/provision-roles.sql index cd7761d109..fc0da2a4c6 100644 --- a/Darling/tools/provision-roles.sql +++ b/Darling/tools/provision-roles.sql @@ -1718,7 +1718,8 @@ BEGIN RETURN jsonb_build_object( 'metric_match', btrim(v_row.metric_match), 'scope_server_id', v_row.scope_server_id, 'scope_tag_id', v_row.scope_tag_id, 'agent_level', v_row.agent_level, 'agent_max_per_hour', v_row.agent_max_per_hour, 'agent_max_per_day', v_row.agent_max_per_day, - 'agent_lease_minutes', v_row.agent_lease_minutes, 'agent_expiry_minutes', v_row.agent_expiry_minutes, 'enabled', v_row.enabled); + 'agent_lease_minutes', v_row.agent_lease_minutes, 'agent_expiry_minutes', v_row.agent_expiry_minutes, 'enabled', v_row.enabled, + 'severities', v_row.severities); END $fn$; REVOKE ALL ON FUNCTION config.update_notification_route(integer, text, text[], text, boolean, integer, integer, boolean, integer, text) FROM PUBLIC; @@ -1803,6 +1804,10 @@ BEGIN IF (p_before ->> 'metric_match') IS DISTINCT FROM btrim(v_row.metric_match) THEN v_changed := array_append(v_changed, 'metric_match'); END IF; IF (p_before ->> 'scope_server_id')::integer IS DISTINCT FROM v_row.scope_server_id THEN v_changed := array_append(v_changed, 'scope_server_id'); END IF; IF (p_before ->> 'scope_tag_id')::integer IS DISTINCT FROM v_row.scope_tag_id THEN v_changed := array_append(v_changed, 'scope_tag_id'); END IF; + -- #6928: the severity filter moves alerts between agent routes, so it is named like the match and the scope. Only a route with an agent setting. + IF (p_before ->> 'severities') IS DISTINCT FROM v_row.severities AND ((p_before ->> 'agent_level') IS NOT NULL OR v_level IS NOT NULL) THEN + v_changed := array_append(v_changed, 'severities'); + END IF; END IF; IF v_level IS DISTINCT FROM v_row.agent_level THEN v_changed := array_append(v_changed, 'agent_level'); END IF; IF v_hour IS DISTINCT FROM v_row.agent_max_per_hour THEN v_changed := array_append(v_changed, 'agent_max_per_hour'); END IF; @@ -1828,11 +1833,13 @@ BEGIN v_before := CASE WHEN COALESCE(p_created, false) THEN NULL WHEN p_before IS NOT NULL THEN p_before ELSE jsonb_build_object( 'metric_match', btrim(v_row.metric_match), 'scope_server_id', v_row.scope_server_id, 'scope_tag_id', v_row.scope_tag_id, 'agent_level', v_row.agent_level, 'agent_max_per_hour', v_row.agent_max_per_hour, 'agent_max_per_day', v_row.agent_max_per_day, - 'agent_lease_minutes', v_row.agent_lease_minutes, 'agent_expiry_minutes', v_row.agent_expiry_minutes, 'enabled', v_row.enabled) END; + 'agent_lease_minutes', v_row.agent_lease_minutes, 'agent_expiry_minutes', v_row.agent_expiry_minutes, 'enabled', v_row.enabled, + 'severities', v_row.severities) END; v_after := jsonb_build_object( 'metric_match', btrim(v_row.metric_match), 'scope_server_id', v_row.scope_server_id, 'scope_tag_id', v_row.scope_tag_id, 'agent_level', v_level, 'agent_max_per_hour', v_hour, 'agent_max_per_day', v_day, - 'agent_lease_minutes', v_lease, 'agent_expiry_minutes', v_expiry, 'enabled', v_row.enabled); + 'agent_lease_minutes', v_lease, 'agent_expiry_minutes', v_expiry, 'enabled', v_row.enabled, + 'severities', v_row.severities); INSERT INTO config.investigation_journal (event_time, request_id, route_id, server_id, event, actor, reason, detail) VALUES (now(), NULL, p_route_id, v_row.scope_server_id, 'route_agent_changed', p_actor, CASE WHEN COALESCE(p_created, false) THEN 'created route' ELSE 'changed ' || array_to_string(v_changed, ', ') END, diff --git a/PerformanceMonitor.Notifications/WebhookAlertService.cs b/PerformanceMonitor.Notifications/WebhookAlertService.cs index aba02322ce..1fb177c9a1 100644 --- a/PerformanceMonitor.Notifications/WebhookAlertService.cs +++ b/PerformanceMonitor.Notifications/WebhookAlertService.cs @@ -2904,9 +2904,20 @@ private static bool IsWellFormedJson(string payload, out string? error) /// is off or PagerDuty is not routed for the metric, or /// otherwise. Never throws except for the caller's own cancellation. /// + public Task TrySendPagerDutyResolveAsync( + string firingMetric, string serverName, string serverId, string? detailText, + AlertSeverityLevel? firedSeverity = null, CancellationToken cancellationToken = default) => + TrySendPagerDutyResolveAsync(firingMetric, serverName, serverId, detailText, new[] { firedSeverity }, cancellationToken); + + /// + /// The same close for an episode that fired at more than one tier (#6928). The route is resolved at each of + /// (a null entry resolves with no tier, as before the filter), and a resolve goes to every + /// distinct routing key that gives, so a Warning-only route's incident is closed after the alert climbed to Critical and a + /// Critical-only route took the later fires. Delivered when a resolve posted and none failed, Failed when any failed. + /// public async Task TrySendPagerDutyResolveAsync( string firingMetric, string serverName, string serverId, string? detailText, - AlertSeverityLevel? firedSeverity = null, CancellationToken cancellationToken = default) + IReadOnlyCollection firedSeverities, CancellationToken cancellationToken = default) { try { @@ -2916,28 +2927,44 @@ public async Task TrySendPagerDutyResolveAsync( return AlertChannelOutcome.NotAttempted; } - /* #6928: routed at the severity the fire went out at, so the close lands on the key the fire's route carried, not on a + /* #6928: routed at each severity the fire went out at, so the close lands on the key each fire's route carried, not on a route the severity filter would send another tier to. */ - var route = NotificationRouter.Resolve(firingMetric, delivery.NotificationRoutes, delivery, NotificationRouter.ParseServerId(serverId), firedSeverity); - if (route.PagerDuty.Destination is not { } pagerDutyKey) + var keys = new List(); + foreach (var severity in firedSeverities.Count == 0 ? new AlertSeverityLevel?[] { null } : firedSeverities) + { + var route = NotificationRouter.Resolve(firingMetric, delivery.NotificationRoutes, delivery, NotificationRouter.ParseServerId(serverId), severity); + if (route.PagerDuty.Destination is { } pagerDutyKey && !keys.Contains(pagerDutyKey, StringComparer.Ordinal)) + { + keys.Add(pagerDutyKey); + } + } + + if (keys.Count == 0) { return AlertChannelOutcome.NotAttempted; } - var error = await TrySendPagerDutyAlertAsync( - delivery, autoResolve: true, pagerDutyKey, firingMetric, serverName, "resolved", "", serverId, - context: null, triageUrl: null, detailText, _utcNow(), displayName: null, - resolveIncident: true, cancellationToken: cancellationToken); - if (error is null) + var failed = false; + foreach (var pagerDutyKey in keys) { - /* As the built-in pairs do after a delivered close: the cooldown entry the fire stamped describes an - incident PagerDuty just closed, so the next fire of the same key must open a new one instead of being - throttled against it. Only a resolve that posted clears it (#5654). */ - _cooldown.ClearMetric(serverId, firingMetric, null); - return AlertChannelOutcome.Delivered; + var error = await TrySendPagerDutyAlertAsync( + delivery, autoResolve: true, pagerDutyKey, firingMetric, serverName, "resolved", "", serverId, + context: null, triageUrl: null, detailText, _utcNow(), displayName: null, + resolveIncident: true, cancellationToken: cancellationToken); + if (error is null) + { + /* As the built-in pairs do after a delivered close: the cooldown entry the fire stamped describes an + incident PagerDuty just closed, so the next fire of the same key must open a new one instead of being + throttled against it. Only a resolve that posted clears it (#5654). */ + _cooldown.ClearMetric(serverId, firingMetric, null); + } + else + { + failed = true; + } } - return AlertChannelOutcome.Failed; + return failed ? AlertChannelOutcome.Failed : AlertChannelOutcome.Delivered; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { From 3dee3305c98d4ad2b50ada268ae3f1173a0a85cf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:53:43 -0400 Subject: [PATCH 15/22] Keep tag overrides on a failed tag read and match the loader's accept rule in the server view (#6926) LoadAsync now throws when the tag tables could not be read, so the worker keeps the whole snapshot in force instead of installing one with no tag winners until the next config change. get_alert_settings filtered tag rows through its own known-key and integer checks. It now calls the loader's Accept, so a deeper tag whose value is outside the key's bounds is skipped there as it is in the engine, and the view names the tag the engine applies. --- .../ThresholdOverrideTagScopeTests.cs | 53 +++++++++++++++++++ .../DarlingThresholdOverrides.cs | 9 +++- .../Mcp/DarlingMcpThresholdOverrideTools.cs | 3 +- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs index b97cef96a9..7aeb652800 100644 --- a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -218,6 +218,59 @@ await RunAsync(async (ds, connection, store, ct) => }); } + [Fact] + public async Task AFailedTagTableRead_KeepsTheSnapshotInForce_NotAGlobalOne() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 92, tag: "Production"))); + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, "srv-9", "cpu.threshold_percent", 70))); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); + Assert.Equal(3, loader.Count); /* srv-9's own row, and the Production override for srv-7 and srv-8 */ + + /* The server-to-tag map cannot be read (a reset or a timeout stands in as a missing table), and a server override changes. */ + await ExecAsync(connection, "ALTER TABLE config.server_tag_map RENAME TO server_tag_map_away", ct); + await ExecAsync(connection, "UPDATE config.config_threshold_overrides SET value = 71 WHERE server_id = 9", ct); + await Assert.ThrowsAnyAsync(() => loader.LoadAsync(ds, ct)); + + /* Nothing was replaced: the tag override still applies and the changed server row is not half-loaded. */ + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); + Assert.Equal(70, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + Assert.Equal(3, loader.Count); /* srv-9's own row, and the Production override for srv-7 and srv-8 */ + + await ExecAsync(connection, "ALTER TABLE config.server_tag_map_away RENAME TO server_tag_map", ct); + await loader.LoadAsync(ds, ct); + Assert.Equal(71, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); + }); + } + + [Fact] + public async Task TheServerView_SkipsATagRowOutsideTheBounds_AsTheLoaderDoes() + { + await RunAsync(async (ds, connection, store, ct) => + { + /* srv-7 is under east (2) under Production (1). The deeper tag holds a value outside a percent's bounds (written by direct SQL). */ + await ExecAsync(connection, "INSERT INTO config.config_threshold_overrides (scope_tag_id, threshold_key, value, set_by) VALUES " + + "(1, 'cpu.threshold_percent', 85, 'erik'), (2, 'cpu.threshold_percent', 5000, 'erik')", ct); + await ExecAsync(connection, "INSERT INTO config.config_alert_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + await ExecAsync(connection, "INSERT INTO config.config_notification (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(85, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "7", null, 80)); + + var view = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-7", ct))!["server_view"]!; + var cpu = view["thresholds"]!.AsArray().First(n => n!["key"]!.GetValue() == "cpu.threshold_percent")!; + Assert.Equal("tag_override", cpu["source"]!.GetValue()); + Assert.Equal("Production", cpu["tag"]!.GetValue()); + Assert.Equal(85, cpu["effective_value"]!.GetValue()); + }); + } + [Fact] public async Task ListShowsTagRowsWithTheirTagName() { diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs index 1ea10e46c2..cb4906e908 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs @@ -46,7 +46,7 @@ private sealed record Snapshot(Dictionary<(int Server, string Database, string K /// read that throws leaves the overrides already in force. A row whose key is not a tunable threshold, whose value is /// not an integer inside the key's bounds, or whose database name sits on a server-only key is skipped and logged, /// never applied: the engine uses the value as-is. Tag overrides (#6926) are resolved with the tag tree and the server-to-tag - /// map, read only when the table holds a tag row; a tag-table read that fails leaves them matching nothing, as a tag-scoped route does. + /// map, read only when the table holds a tag row; a tag-table read that fails throws, so the whole snapshot already in force stays (it is not replaced by one with no tag overrides). /// internal async Task LoadAsync(NpgsqlDataSource postgres, CancellationToken cancellationToken = default) { @@ -72,6 +72,10 @@ internal async Task LoadAsync(NpgsqlDataSource postgres, CancellationToken cance { await using var connection = await postgres.OpenConnectionAsync(cancellationToken); inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, _logger, cancellationToken); + /* ReadRouteScopeInputsAsync swallows a failed tag read, and installing a snapshot with no tag winners would drop every + tag override to the global value until the next config change. Throw so the worker keeps the snapshot in force. */ + if (!inputs.TagsReadable) + throw new InvalidOperationException("The server tag tables could not be read, so the tag overrides cannot be resolved."); } Replace(direct, tag, inputs); @@ -125,7 +129,8 @@ internal void Replace( _snapshot = new Snapshot(direct, tagWinners); } - private static string? Accept(string key, double raw, string database, out int value) + /// Why a stored value cannot be applied (null: it can). The MCP server view filters tag rows through this too, so it never names a tag the engine skips. + internal static string? Accept(string key, double raw, string database, out int value) { value = 0; if (!DarlingThresholdKeys.TryGet(key, out var entry)) diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs index edab71bae2..f27e14ca86 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs @@ -369,7 +369,8 @@ internal static async Task BuildServerViewAsync( { var key = reader.GetString(2); var raw = reader.GetDouble(3); - if (!DarlingThresholdKeys.TryGet(key, out _) || raw != Math.Floor(raw) || raw < int.MinValue || raw > int.MaxValue) + /* The loader's own accept rule (known key, integer, inside the key's bounds), so a row the engine skips never names a tag here. */ + if (DarlingThresholdOverrides.Accept(key, raw, string.Empty, out _) is not null) continue; candidates.Add(new TagOverrideCandidate(reader.GetInt64(0), reader.GetInt32(1), key, raw)); meta[(reader.GetInt32(1), key)] = (reader.GetString(4), reader.GetDateTime(5)); From 9e89adc1969607ddfcad3c51b6ff3290accde1a6 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:00:29 -0400 Subject: [PATCH 16/22] Remove the unused DarlingAlertSettings.BlockingBandThresholds property (#6927) Nothing read it: the service, the viewer and the MCP tools all read the tiers through BlockingBandStore. The config fields it wrapped are still bound by the seed insert and the store load, so they stay. --- .../DarlingAlertSettings.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs index 048c648cc8..add0198c81 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs @@ -296,15 +296,6 @@ public IReadOnlyList WebhookChannelProblems _config.Alerts.DeadlockWarnPerHour, _config.Alerts.DeadlockCriticalPerHour); - /// #6927 (V228): the blocking health band's four tiers as ONE value, for the reason - /// is one. The clamp lives on - /// , so the viewer's card and the service's read clamp alike. - public BlockingBandThresholds BlockingBandThresholds => new( - _config.Alerts.BlockingWarnPerHour, - _config.Alerts.BlockingCriticalPerHour, - _config.Alerts.BlockingWarnWaitSeconds, - _config.Alerts.BlockingCriticalWaitSeconds); - public int CollectionFailureThreshold => Math.Clamp(_config.Alerts.CollectionFailureThreshold, 1, 1000); /* #1984: percent clamped like low-disk's (0 = off); the GB floor merely floored at 0 — unlike From 8f903add99ba6503b9aa8e22b23410eea06f7e46 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:07:58 -0400 Subject: [PATCH 17/22] Send the saved version with a route edit from the web editor (part of #6928) The route form's read() rebuilt its entries without modifiedAt, so every Save on an existing route went out with no expected_modified_at and the service refused it with a 400. The form now carries modifiedAt through. A test drives the Routes tab: Edit, choose a severity, Save, and check the write the page sent. --- .../Darling.Tests/AdminPendingNotesTests.cs | 16 ++++++++++++++ .../admin-server-edit-harness.mjs | 21 +++++++++++++++++++ .../wwwroot/js/pages/admin-agent-setup.js | 3 ++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/AdminPendingNotesTests.cs b/Darling/Darling.Tests/AdminPendingNotesTests.cs index e3a6755624..07528e5fb5 100644 --- a/Darling/Darling.Tests/AdminPendingNotesTests.cs +++ b/Darling/Darling.Tests/AdminPendingNotesTests.cs @@ -152,6 +152,22 @@ public void TheRoutesList_ShowsTheSeverityBesideTheScope_AndTheEditorRoundTripsI Assert.Equal("both", Sent("someday")); } + /* #6936: Edit then Save on the Routes tab sends the version the form read (expected_modified_at), because the service refuses an edit + without it with a 400. The test drives the page itself: it opens the editor on a stored Critical only route, picks Warning only, + presses Save and reads the write the page made. */ + [Fact] + public void SavingAnEditedRoute_SendsTheVersionTheFormRead_AndTheChosenSeverity() + { + var writes = AdminServerEditBehaviourTests.Run("routeEdit:warning").GetProperty("writes").EnumerateArray().ToArray(); + + var write = Assert.Single(writes); + Assert.Equal("/api/notification-routes/7", write.GetProperty("url").GetString()); + var body = write.GetProperty("body"); + Assert.Equal("2026-01-01T00:00:00.1234567Z", body.GetProperty("expected_modified_at").GetString()); + Assert.Equal("warning", body.GetProperty("severities").GetString()); + Assert.Equal("blocking", body.GetProperty("metric_match").GetString()); + } + [Fact] public void OnlyASeatThatCanEdit_GetsTheDecideColumn_Pure() { diff --git a/Darling/Darling.Tests/admin-server-edit-harness.mjs b/Darling/Darling.Tests/admin-server-edit-harness.mjs index a76d124216..514abd9946 100644 --- a/Darling/Darling.Tests/admin-server-edit-harness.mjs +++ b/Darling/Darling.Tests/admin-server-edit-harness.mjs @@ -1248,6 +1248,27 @@ try { writes: requests.filter((r) => r.method === "PUT").length, }; }, + /* #6936: the Routes tab on an editing seat. Edit on route 7 (a stored Critical only route), choose `severity` in the Severity box, Save: + returns the write the page sent (method, url, body). The service answers 200 so the page repaints; the body is what is under test. */ + routeEdit: async (severity) => { + state.canEdit = true; + const route = { route_id: 7, metric_match: "blocking", match_kind: "exact_metric", family: "performance", configured_channels: [], smtp_recipients: [], + enabled: true, modified_at_utc: "2026-01-01T00:00:00.1234567Z", scope_server_id: null, scope_tag_id: null, agent_level: null, agent_max_per_hour: 10, + agent_max_per_day: 50, agent_lease_minutes: 30, agent_expiry_minutes: 120, send_resolutions: null, fallback_route_id: null, severities: "critical" }; + state.responder = (method, url) => { + if (method === "GET" && url === "/api/read/get_notification_routes") return { status: 200, body: { routes: [route] } }; + if (method === "GET" && url === "/api/agent-setup/access") return { status: 200, body: { can_edit_routes: true, can_edit_profiles: true } }; + if (method === "PUT" && url === "/api/notification-routes/7") return { status: 200, body: { status: "updated", route_id: 7 } }; + return { status: 404, body: { error: "no such request " + url } }; + }; + await mountPage("routes"); + await fire(byAttr(main, "data-action", "edit-route")[0], "click"); + await settle(30); + await typeInto(field("severities"), severity); + await fire(byAttr(main, "data-action", "save-route")[0], "click"); + await settle(30); + return { writes: requests.filter((r) => r.method === "PUT").map((r) => ({ method: r.method, url: r.url, body: r.body })) }; + }, /* One call of an admin.js export, {"fn": "buildEditBody", "args": [...]} on stdin, printed as {"result": ...}; or {"calls": [{"fn": ..., "args": [...]}, ...]} for a table of calls in one process, printed as {"results": [...]}. An export that is not a function (EDIT_FIELDS) is returned as it is. */ diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin-agent-setup.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin-agent-setup.js index 469793c803..f6add02599 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin-agent-setup.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin-agent-setup.js @@ -243,7 +243,8 @@ function routeForm(options) { const recipients = text("recipients", entries.recipients, { placeholder: "name@example.com, other@example.com" }); const read = () => { const e = { - id: entries.id, enabled: enabled.checked, saved: entries.saved, recipientsWas: entries.recipientsWas, hooks: {}, removes: {}, + /* #6936: modifiedAt rides along, or an edit goes out without expected_modified_at and the service refuses it with a 400. */ + id: entries.id, modifiedAt: entries.modifiedAt, enabled: enabled.checked, saved: entries.saved, recipientsWas: entries.recipientsWas, hooks: {}, removes: {}, }; for (const k of Object.keys(controls)) e[k] = controls[k].value; for (const d of WEBHOOK_DESTINATIONS) { From 9dd1a4677d154d9feebce6e0e3b572243ab7c5bf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:30:15 -0400 Subject: [PATCH 18/22] Declare the V227 rung in the data-moving census and build the tag-name read from the store source (#6926) V227 adds a nullable column, two CHECKs and a partial unique index to the operator-written config.config_threshold_overrides table, a handful of rows, so it is declared SetsTheFloor false and the lock-wait multiple stays 5. The tag-name read in the override tool now uses postgres.CreateCommand like its sibling reads, which is the store receiver the read-deadline census recognises. --- .../Darling.Tests/MigrationDataMovingRungCensusPins.cs | 10 ++++++++++ .../Mcp/DarlingMcpThresholdOverrideTools.cs | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs b/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs index 2c222b64ce..5bd785e1c8 100644 --- a/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs +++ b/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs @@ -359,6 +359,16 @@ public sealed class MigrationDataMovingRungCensusPins + "singleton (id = 1): turns the Client Timeouts alert on by default (#5736). V147's shape on the same kind of table. The " + "UPDATE's WHERE client_timeouts_enabled = FALSE touches at most one row, the SET DEFAULT is catalog-only, and neither reads " + "any collected data, so the rung spends none of the lock-wait budget and the multiple stays 5."), + new( + 227, + SetsTheFloor: false, + "one CREATE UNIQUE INDEX and two ADD CONSTRAINT ... CHECK on config.config_threshold_overrides (#6926, created V191), " + + "which hold the new tag scope. The table is an operator-written config table: one row per threshold somebody chose to " + + "override for a server, so a store holds a handful of rows and a very large one a few hundred. The CHECKs are validated " + + "against every stored row and the partial index (WHERE scope_tag_id IS NOT NULL) indexes nothing, because the column the " + + "same rung adds is null on every stored row, so each statement reads the table once and takes milliseconds. The ADD " + + "COLUMN is nullable with no default and the DROP NOT NULL on server_id is catalog-only, so neither rewrites a row. It does " + + "not move the lock-wait floor, so the multiple stays 5."), ]; /// diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs index f27e14ca86..cf8748de8d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs @@ -382,9 +382,10 @@ internal static async Task BuildServerViewAsync( await using var connection = await postgres.OpenConnectionAsync(cancellationToken); var inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, null, cancellationToken); var names = new System.Collections.Generic.Dictionary(); - await using (var command = new NpgsqlCommand("SELECT id, name FROM config.server_tags", connection) { CommandTimeout = McpCommandDeadlines.ReadSeconds }) - await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) + await using (var command = postgres.CreateCommand("SELECT id, name FROM config.server_tags")) { + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) names[reader.GetInt32(0)] = reader.GetString(1); } From fd3bda92a6a9b55003d7709550a80f0ba7f65d1e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:52:03 -0400 Subject: [PATCH 19/22] Let the V227 live test follow the newest schema version instead of pinning 227 --- Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs index 7aeb652800..553cdd8a60 100644 --- a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -313,7 +313,7 @@ await ExecAsync(store, "INSERT INTO config.config_threshold_overrides (server_id "SELECT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'config.config_threshold_overrides'::regclass AND attname = 'scope_tag_id' AND NOT attisdropped)", ct)); await PgMigrations.MigrateAsync(store, ct); - Assert.Equal("227", await ScalarAsync(store, "SELECT max(version) FROM darling_schema_version", ct)); + Assert.Equal(StorageVersion.SchemaVersion.ToString(CultureInfo.InvariantCulture), await ScalarAsync(store, "SELECT max(version) FROM darling_schema_version", ct)); Assert.Equal("2", await ScalarAsync(store, "SELECT count(*) FROM config.config_threshold_overrides WHERE server_id = 7 AND scope_tag_id IS NULL", ct)); Assert.Equal("91", await ScalarAsync(store, "SELECT value FROM config.config_threshold_overrides WHERE database_name IS NULL", ct)); From cc629f90ecc007cfd5f6aa6230da56211d99e670 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:27:19 -0400 Subject: [PATCH 20/22] Make the V229 rewind safe on a store whose external alert tables were dropped --- Darling/Darling.Tests/RungRewindArtifacts.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Darling/Darling.Tests/RungRewindArtifacts.cs b/Darling/Darling.Tests/RungRewindArtifacts.cs index 1a5b713ed9..8277f25bc5 100644 --- a/Darling/Darling.Tests/RungRewindArtifacts.cs +++ b/Darling/Darling.Tests/RungRewindArtifacts.cs @@ -599,8 +599,8 @@ ALTER TABLE config.config_alert_settings // (its CHECK goes with it), so a re-run of V229 does everything again. Safe on a store that never had it. new(229, "hasRouteSeverities", @"ALTER TABLE config.config_notification_routes DROP COLUMN IF EXISTS severities; -ALTER TABLE config.external_alert_state DROP COLUMN IF EXISTS fired_severities; -ALTER TABLE config.investigation_request_alerts DROP COLUMN IF EXISTS severity;"), +ALTER TABLE IF EXISTS config.external_alert_state DROP COLUMN IF EXISTS fired_severities; +ALTER TABLE IF EXISTS config.investigation_request_alerts DROP COLUMN IF EXISTS severity;"), ]; /// From 8499a93ee914a158ebbae30f6e5db895a3ca84af Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:01:33 -0400 Subject: [PATCH 21/22] Fix the two test failures on the route severity change (#6928) The route severity test class now records its own-store exemption above the class, where the collection hygiene test looks for it. The Lite and Darling route field parity test lists severities as a Darling-only field. --- .../Darling.Tests/ViewerNotificationRouteSeveritiesTests.cs | 3 ++- Lite.Tests/NotificationRoutesMcpTests.cs | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/ViewerNotificationRouteSeveritiesTests.cs b/Darling/Darling.Tests/ViewerNotificationRouteSeveritiesTests.cs index bcf9205021..c6ccd2c6d6 100644 --- a/Darling/Darling.Tests/ViewerNotificationRouteSeveritiesTests.cs +++ b/Darling/Darling.Tests/ViewerNotificationRouteSeveritiesTests.cs @@ -22,6 +22,8 @@ namespace Darling.Tests; /// the filter in words, and a route's choice round-trips through the viewer's insert, update and sealed save. A store below V229 has no /// column, so the list still opens and every route reads as Both. /// +/* #1776 own-store: its one live fact mints its own scratch database through ScratchPostgres and touches nothing on the shared + one, so it is not serialized against the live-postgres collection. */ public sealed class ViewerNotificationRouteSeveritiesTests { /// The first sixteen sentinels, all present (the V16 to V31 floor), so only the one named after them decides the answer. @@ -66,7 +68,6 @@ private static async Task ExecAsync(NpgsqlConnection connection, string sql, Sys await command.ExecuteNonQueryAsync(ct); } - /* #1776 own-store: this class mints its own scratch database, so it is not [Collection("live-postgres")]. */ [Fact] public async Task Live_TheSeverityRoundTripsThroughTheViewer_AndAStoreBelowV229StillOpensTheList() { diff --git a/Lite.Tests/NotificationRoutesMcpTests.cs b/Lite.Tests/NotificationRoutesMcpTests.cs index e6b59e6dad..66ebcdf57b 100644 --- a/Lite.Tests/NotificationRoutesMcpTests.cs +++ b/Lite.Tests/NotificationRoutesMcpTests.cs @@ -453,6 +453,8 @@ delegates to it (DarlingRoutePayloadTests pins that), so this is still the one b /* #6036: the fallback route (D8 of #5632 Phase C) is Darling-only; Lite has no request events. */ "fallback_route_id", "fallback_route_name", + /* #6928: route severity is Darling-only. */ + "severities", }; Assert.All(darlingOnly, f => Assert.Contains(f, darling)); Assert.Equal(darling.Except(darlingOnly).OrderBy(n => n, StringComparer.Ordinal), lite.OrderBy(n => n, StringComparer.Ordinal)); From ec2dd7bfa4399cb8e6de4994011c013d0cedcb41 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:27:37 -0400 Subject: [PATCH 22/22] Fix three route severity defects found by CI (part of #6928) The route cache key now includes the severity filter, so an edit of only that filter reloads the cached routes. The mcp role gets INSERT on investigation_request_alerts.severity, which the practice copy writes, so starting practice no longer fails with a permission refusal. The admin routes table test and the managed-roles grant pin name the new severities column and the new grant column. --- Darling/Darling.Tests/AdminPageTests.cs | 2 +- Darling/Darling.Tests/DarlingManagedRolesTests.cs | 2 +- .../PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs | 2 +- .../DarlingWebhookSecrets.cs | 3 +++ Darling/tools/provision-roles.sql | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Darling/Darling.Tests/AdminPageTests.cs b/Darling/Darling.Tests/AdminPageTests.cs index 5478aaeb24..4ad773cc58 100644 --- a/Darling/Darling.Tests/AdminPageTests.cs +++ b/Darling/Darling.Tests/AdminPageTests.cs @@ -43,7 +43,7 @@ public void TheServersTab_ReadsTheAdminRoute_NotListServers_AndDrawsDisabledRows [Theory] [InlineData("SERVER_COLUMNS", new[] { "display_name", "server_name", "auth", "engine", "version", "status", "freshness", "monthly_cost", "read_only", "added", "last_collected", "same_machine_as" })] - [InlineData("ROUTE_COLUMNS", new[] { "route_id", "enabled", "metric_match", "match_kind", "family", "applies_to", "status", "channels", "agent", "resolutions", "smtp_recipients", "modified_at_utc" })] + [InlineData("ROUTE_COLUMNS", new[] { "route_id", "enabled", "metric_match", "match_kind", "family", "applies_to", "severities", "status", "channels", "agent", "resolutions", "smtp_recipients", "modified_at_utc" })] [InlineData("SETTING_COLUMNS", new[] { "setting", "value" })] public void EachTab_DeclaresItsColumnKeys(string name, string[] keys) { diff --git a/Darling/Darling.Tests/DarlingManagedRolesTests.cs b/Darling/Darling.Tests/DarlingManagedRolesTests.cs index 0468c9b5f4..9c7c99c1fb 100644 --- a/Darling/Darling.Tests/DarlingManagedRolesTests.cs +++ b/Darling/Darling.Tests/DarlingManagedRolesTests.cs @@ -586,7 +586,7 @@ and viewer writes nothing here. */ Assert.DoesNotMatch(@"GRANT [A-Z ,()a-z_]*(INSERT|UPDATE|DELETE)[^;]* ON config\.investigation_request_alerts TO (?!admin;|mcp;)", flat); /* #6400: mcp (start_practice_request) copies a case's alert rows too, by the ten columns the copy sets, and never updates or deletes a link. */ - Assert.Contains("GRANT INSERT (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json) ON config.investigation_request_alerts TO mcp;", flat, StringComparison.Ordinal); + Assert.Contains("GRANT INSERT (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json, severity) ON config.investigation_request_alerts TO mcp;", flat, StringComparison.Ordinal); Assert.DoesNotMatch(@"GRANT [A-Z ,()a-z_]*(UPDATE|DELETE)[^;]* ON config\.investigation_request_alerts TO mcp;", flat); Assert.DoesNotMatch(@"GRANT (INSERT|UPDATE|DELETE)[^;]* ON config\.(investigation|agent_environment)[a-z_]* TO [a-z_, ]*viewer", flat); } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs index 12bc4d5212..fc85a093db 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs @@ -2010,7 +2010,7 @@ GRANT INSERT (request_id, route_id, server_id, server_name, family, first_metric -- every start, so no migration rung carries this. GRANT INSERT (request_id, route_id, server_id, server_name, family, first_metric, story_path_hash, first_alert_at, last_alert_at, window_until, opened_at, expires_at, lease_minutes, autonomy_level, alert_count, practice_of) ON {config}.investigation_requests TO {mcp}; -GRANT INSERT (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json) +GRANT INSERT (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json, severity) ON {config}.investigation_request_alerts TO {mcp}; GRANT INSERT (request_id, verdict, corrected_verdict, missed_step, comment, author) ON {config}.investigation_feedback TO {mcp}; GRANT UPDATE (verdict, corrected_verdict, missed_step, comment, author, created_at) ON {config}.investigation_feedback TO {mcp}; diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebhookSecrets.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebhookSecrets.cs index b592f48107..ced6795573 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebhookSecrets.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebhookSecrets.cs @@ -349,6 +349,9 @@ the old row cached (a cleared fallback kept receiving the expired requests). A m Add(r.AgentExpiryMinutes.ToString(System.Globalization.CultureInfo.InvariantCulture)); Add(r.FallbackRouteId?.ToString(System.Globalization.CultureInfo.InvariantCulture)); Add(r.OffChannels is null ? "-" : string.Join(",", r.OffChannels.OrderBy(c => c, StringComparer.Ordinal))); + + /* #6928: the severity filter. Delivery reads it from the cached route, so an edit of only this member must reopen the cache. */ + Add(r.Severities); } return text.ToString(); diff --git a/Darling/tools/provision-roles.sql b/Darling/tools/provision-roles.sql index fc0da2a4c6..45423d5c84 100644 --- a/Darling/tools/provision-roles.sql +++ b/Darling/tools/provision-roles.sql @@ -3233,7 +3233,7 @@ BEGIN -- statements set and nothing more. BRING-YOUR-OWN STORES: re-run this script to add them. GRANT INSERT (request_id, route_id, server_id, server_name, family, first_metric, story_path_hash, first_alert_at, last_alert_at, window_until, opened_at, expires_at, lease_minutes, autonomy_level, alert_count, practice_of) ON config.investigation_requests TO mcp; - GRANT INSERT (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json) + GRANT INSERT (request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json, severity) ON config.investigation_request_alerts TO mcp; GRANT INSERT (request_id, verdict, corrected_verdict, missed_step, comment, author) ON config.investigation_feedback TO mcp; GRANT UPDATE (verdict, corrected_verdict, missed_step, comment, author, created_at) ON config.investigation_feedback TO mcp;