From 4b19caf730fb02546da7615434b4683498710d54 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:18:05 -0400 Subject: [PATCH 01/10] Hold threshold overrides for a server tag (#6926) A threshold override can now be set for a server tag. Precedence is database, then server, then the deepest tag the server is under (the notification routes' own rule), then global. Adds V227, the MCP tag argument, the get_alert_settings tag source, the Viewer and web Admin tag rows, and tests. --- Darling/Darling.Tests/RungRewindArtifacts.cs | 10 + .../ThresholdOverrideTagScopeTests.cs | 329 ++++++++++++++++++ .../DarlingThresholdOverrides.cs | 105 ++++-- .../DarlingWebThresholdOverrides.cs | 29 +- .../Mcp/DarlingMcpThresholdOverrideTools.cs | 151 +++++++- .../wwwroot/js/pages/admin.js | 55 +-- .../PgMigrations.cs | 44 +++ .../StorageVersion.cs | 2 +- .../TagThresholdResolver.cs | 71 ++++ .../ThresholdOverrideStore.TagScope.cs | 94 +++++ .../ThresholdOverrideStore.cs | 31 +- .../ThresholdOverridesViewModel.cs | 56 ++- .../ThresholdOverridesWindow.xaml | 10 +- .../ThresholdOverridesWindow.xaml.cs | 11 +- .../ViewerDataService.ThresholdOverrides.cs | 31 ++ .../ViewerDataService.cs | 14 +- docs/darling/admin.md | 1 + 17 files changed, 953 insertions(+), 91 deletions(-) create mode 100644 Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs create mode 100644 Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs create mode 100644 Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs diff --git a/Darling/Darling.Tests/RungRewindArtifacts.cs b/Darling/Darling.Tests/RungRewindArtifacts.cs index 375b56efdd..a4273f778b 100644 --- a/Darling/Darling.Tests/RungRewindArtifacts.cs +++ b/Darling/Darling.Tests/RungRewindArtifacts.cs @@ -577,6 +577,16 @@ ALTER TABLE config.config_alert_settings // so a re-run of V226 does everything again. Safe on a store that never had it. new(226, "hasServerAwsRdsWebOrExpress", @"ALTER TABLE collect.servers DROP COLUMN IF EXISTS is_aws_rds_web_or_express;"), + // V227 (#6926): the tag scope on the threshold overrides. The column is the probe's newest sentinel. The rewind removes the tag rows, + // the two constraints and the tag index, puts server_id back to NOT NULL and drops the column, so a re-run of V227 does everything + // again. Safe on a store that never had it. + new(227, "hasThresholdOverrideTagScope", + @"DELETE FROM config.config_threshold_overrides WHERE server_id IS NULL; +ALTER TABLE config.config_threshold_overrides DROP CONSTRAINT IF EXISTS ck_config_threshold_overrides_one_scope; +ALTER TABLE config.config_threshold_overrides DROP CONSTRAINT IF EXISTS ck_config_threshold_overrides_tag_no_database; +DROP INDEX IF EXISTS config.ux_config_threshold_overrides_tag; +ALTER TABLE config.config_threshold_overrides ALTER COLUMN server_id SET NOT NULL; +ALTER TABLE config.config_threshold_overrides DROP COLUMN IF EXISTS scope_tag_id;"), ]; /// diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs new file mode 100644 index 0000000000..1c5c98400b --- /dev/null +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Npgsql; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/* #1776 own-store: every live fact mints its own scratch database through ScratchPostgres and never touches another test's rows, + so the class is deliberately NOT [Collection("live-postgres")]. */ + +/// +/// #6926, a threshold override held for a server tag. The precedence (database, then server, then tag, then global), the tag rule +/// (the notification routes' own: the deepest tag a server is under, the lowest row id on a same-depth tie, a deleted tag matching +/// nothing) and the whole path through a live store: set_alert_threshold_override with a tag, get_alert_settings +/// naming the tag, the list marking an orphan, and V227 on a store that already holds server overrides. +/// +public sealed class ThresholdOverrideTagScopeTests +{ + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private static readonly DarlingMcpWritePolicy.Effective WritesArm = new(true, false); + private static readonly DarlingMcpWritePolicy.Effective AgentArm = new(false, false, Investigations: true); + + /* Tags: 1 prod (root) > 2 east > 3 rack1; 4 dev (root); 5 and 6 are two roots a server can sit under at once. */ + private static readonly Dictionary Parents = new() + { + [1] = null, [2] = 1, [3] = 2, [4] = null, [5] = null, [6] = null, + }; + + private static StoreConfigProvider.RouteScopeInputs Inputs(bool readable = true) => new( + new HashSet { 7, 8, 9, 10 }, + Parents, + new Dictionary> + { + [7] = new[] { 3 }, /* under rack1: prod > east > rack1 */ + [8] = new[] { 1 }, /* directly under prod */ + [9] = new[] { 4 }, /* dev */ + [10] = new[] { 5, 6 }, /* two roots at the same depth */ + }, + readable); + + private static DarlingThresholdOverrides Load( + (int Server, string? Database, string Key, double Value)[] direct, + (long RowId, int Tag, string Key, double Value)[] tags, + bool readable = true) + { + var overrides = new DarlingThresholdOverrides(); + overrides.Replace(direct, tags.Select(t => (t.RowId, t.Tag, t.Key, t.Value)), Inputs(readable)); + return overrides; + } + + [Fact] + public void Precedence_DatabaseBeatsServerBeatsTagBeatsGlobal() + { + const string Key = "pvs.threshold_percent"; + var o = Load( + [(7, "shop", Key, 60), (7, null, Key, 55)], + [(1, 1, Key, 50)]); + + Assert.Equal(60, o.ResolveWire(Key, "7", "shop", 40)); /* the database override */ + Assert.Equal(55, o.ResolveWire(Key, "7", "ledger", 40)); /* no override for that database: the server's */ + Assert.Equal(55, o.ResolveWire(Key, "7", null, 40)); + Assert.Equal(50, o.ResolveWire(Key, "8", null, 40)); /* no server override: the tag's */ + Assert.Equal(50, o.ResolveWire(Key, "8", "shop", 40)); + Assert.Equal(40, o.ResolveWire(Key, "9", null, 40)); /* a server outside the tag: global */ + Assert.Equal(40, o.ResolveWire("cpu.threshold_percent", "8", null, 40)); + } + + [Fact] + public void TagRule_DeepestTagWins_AndASameDepthTieGoesToTheLowestRowId() + { + const string Key = "cpu.threshold_percent"; + + /* Server 7 sits under rack1 (2 ancestors); prod (0 ancestors) also covers it. The deeper tag wins whatever the row ids. */ + var nested = Load([], [(1, 1, Key, 80), (2, 3, Key, 70), (3, 2, Key, 75)]); + Assert.Equal(70, nested.ResolveWire(Key, "7", null, 90)); + Assert.Equal(80, nested.ResolveWire(Key, "8", null, 90)); /* server 8 is only under prod */ + + /* Server 10 is under two roots of the same depth: the lower row id wins, the way the router takes the lower route id. */ + var tie = Load([], [(20, 5, Key, 65), (10, 6, Key, 85)]); + Assert.Equal(85, tie.ResolveWire(Key, "10", null, 90)); + var reversed = Load([], [(10, 5, Key, 65), (20, 6, Key, 85)]); + Assert.Equal(65, reversed.ResolveWire(Key, "10", null, 90)); + } + + [Fact] + public void ADeletedTag_MatchesNothing_AndUnreadableTagTablesMatchNothing() + { + const string Key = "cpu.threshold_percent"; + var orphan = Load([], [(1, 99, Key, 70)]); /* tag 99 is not in the tree */ + Assert.Equal(90, orphan.ResolveWire(Key, "7", null, 90)); + Assert.Equal(0, orphan.Count); + + var unreadable = Load([], [(1, 1, Key, 70)], readable: false); + Assert.Equal(90, unreadable.ResolveWire(Key, "8", null, 90)); + } + + [Fact] + public void ATagRowWithAnInvalidValue_IsSkipped_AndTheShallowerTagStillApplies() + { + const string Key = "cpu.threshold_percent"; + var o = Load([], [(1, 1, Key, 80), (2, 3, Key, 5000)]); /* 5000 is out of bounds for a percent */ + Assert.Equal(80, o.ResolveWire(Key, "7", null, 90)); + } + + [Fact] + public async Task Tool_SetsAndClearsATagOverride_TheLoaderAndTheServerViewNameTheTag() + { + await RunAsync(async (ds, connection, store, ct) => + { + var set = await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 95, tag: "Production", setBy: "erik"); + Assert.Equal("ok", Status(set)); + Assert.Equal("Production", JsonNode.Parse(set)!["tag"]!.GetValue()); + Assert.Equal("1", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides WHERE scope_tag_id = 1 AND server_id IS NULL", ct)); + Assert.Equal("erik", await ScalarAsync(connection, "SELECT set_by FROM config.config_threshold_overrides WHERE scope_tag_id = 1", ct)); + + /* A second call updates the same row (by tag id this time); the answer carries the undo. */ + var again = await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 92, tag: "1"); + Assert.Equal(95, JsonNode.Parse(again)!["previous_override"]!.GetValue()); + Assert.Equal("1", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides WHERE scope_tag_id = 1", ct)); + + /* A nested tag with its own value: the deeper tag wins for the server under it. */ + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 88, tag: "east"))); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(88, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "7", null, 80)); /* srv-7 is under east */ + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); /* srv-8 is directly under Production */ + Assert.Equal(80, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + + /* A server override beats the tag in the loader and in the view. */ + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, "srv-7", "cpu.threshold_percent", 60))); + await loader.LoadAsync(ds, ct); + Assert.Equal(60, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "7", null, 80)); + + await ExecAsync(connection, "INSERT INTO config.config_alert_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + await ExecAsync(connection, "INSERT INTO config.config_notification (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + JsonNode? Row(JsonNode view, string key) => view["thresholds"]!.AsArray().First(n => n!["key"]!.GetValue() == key); + + var seven = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-7", ct))!["server_view"]!; + Assert.Equal("server_override", Row(seven, "cpu.threshold_percent")!["source"]!.GetValue()); + + var eight = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-8", ct))!["server_view"]!; + var cpu = Row(eight, "cpu.threshold_percent")!; + Assert.Equal("tag_override", cpu["source"]!.GetValue()); + Assert.Equal("Production", cpu["tag"]!.GetValue()); + Assert.Equal(92, cpu["effective_value"]!.GetValue()); + Assert.Equal("global", Row(eight, "blocking.count_threshold")!["source"]!.GetValue()); + Assert.Equal(1, eight["tag_overrides"]!.AsArray().Count); + + var nine = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-9", ct))!["server_view"]!; + Assert.Equal("global", Row(nine, "cpu.threshold_percent")!["source"]!.GetValue()); + + /* Clear works the same way, by name or id. */ + Assert.Equal("cleared", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", null, tag: "east", clear: true))); + Assert.Equal("cleared", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", null, tag: "1", clear: true))); + Assert.Equal("0", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides WHERE scope_tag_id IS NOT NULL", ct)); + }); + } + + [Fact] + public async Task Tool_RefusesTagMisuse_AndLeavesTheTableUntouched() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("scope_conflict", Reason(await Set(ds, store, WritesArm, "srv-7", "cpu.threshold_percent", 90, tag: "Production"))); + Assert.Equal("scope_required", Reason(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 90))); + Assert.Equal("database_scope", Reason(await Set(ds, store, WritesArm, null, "pvs.threshold_percent", 70, database: "shop", tag: "Production"))); + Assert.Equal("tag_not_found", Reason(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 90, tag: "nope"))); + Assert.Equal("tag_writes_arm_only", Reason(await Set(ds, store, AgentArm, null, "cpu.threshold_percent", 90, tag: "Production"))); + Assert.Equal("0", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides", ct)); + + /* A name two tags share is refused until the id is used. */ + await ExecAsync(connection, "INSERT INTO config.server_tags (name, parent_id) VALUES ('Shared', 1), ('Shared', 4)", ct); + Assert.Equal("tag_not_found", Reason(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 90, tag: "shared"))); + }); + } + + [Fact] + public async Task ADeletedTag_ListsAsOrphaned_MatchesNothing_AndCanStillBeCleared() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 95, tag: "dev"))); + await ExecAsync(connection, "DELETE FROM config.server_tags WHERE id = 4", ct); + + var rows = await ThresholdOverrideStore.ListAsync(ds, k => null, ct); + var row = Assert.Single(rows); + Assert.True(row.IsTag); + Assert.True(row.Orphaned); + Assert.Equal(4, row.TagId); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(80, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + + Assert.Equal("cleared", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", null, tag: "4", clear: true))); + Assert.Empty(await ThresholdOverrideStore.ListAsync(ds, k => null, ct)); + }); + } + + [Fact] + public async Task ListShowsTagRowsWithTheirTagName() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 95, tag: "Production"))); + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, "srv-7", "cpu.threshold_percent", 90))); + var rows = await ThresholdOverrideStore.ListAsync(ds, k => null, ct); + Assert.Equal(2, rows.Count); + var tag = rows.Single(r => r.IsTag); + Assert.Equal("Production", tag.ServerName); + Assert.False(tag.Orphaned); + Assert.Equal("srv-7", rows.Single(r => !r.IsTag).ServerName); + + var body = await DarlingWebThresholdOverrides.ListBodyAsync(ds, ct); + Assert.Equal(2, body["overrides"]!.AsArray().Count); + Assert.Contains(body["tags"]!.AsArray(), t => t!["tag_name"]!.GetValue() == "Production"); + }); + } + + [Fact] + public async Task V227_OnAStoreThatHoldsServerOverrides_KeepsThem_AndTakesTagRows() + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to run the threshold override live tests."); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var store = new NpgsqlConnection(scratch.ConnectionString); + await store.OpenAsync(ct); + await PgMigrations.MigrateAsync(store, ct); + + var rung = PgMigrations.Scripts.Single(m => m.Name == "threshold-override-tag-scope").Version; + Assert.Equal(227, rung); + await RungRewindArtifacts.DropSentinelsAboveAsync(store, rung - 1, ct); + await ExecAsync(store, "DELETE FROM darling_schema_version WHERE version >= " + rung.ToString(CultureInfo.InvariantCulture), ct); + await ExecAsync(store, "INSERT INTO config.config_monitored_servers (server_id, name, host) VALUES (7, 'srv-7', 'h7')", ct); + await ExecAsync(store, "INSERT INTO config.config_threshold_overrides (server_id, database_name, threshold_key, value, set_by) VALUES " + + "(7, NULL, 'cpu.threshold_percent', 91, 'erik'), (7, 'shop', 'pvs.threshold_percent', 66, 'erik')", ct); + Assert.Equal("False", await ScalarAsync(store, + "SELECT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'config.config_threshold_overrides'::regclass AND attname = 'scope_tag_id' AND NOT attisdropped)", ct)); + + await PgMigrations.MigrateAsync(store, ct); + Assert.Equal("227", await ScalarAsync(store, "SELECT max(version) FROM darling_schema_version", ct)); + Assert.Equal("2", await ScalarAsync(store, "SELECT count(*) FROM config.config_threshold_overrides WHERE server_id = 7 AND scope_tag_id IS NULL", ct)); + Assert.Equal("91", await ScalarAsync(store, "SELECT value FROM config.config_threshold_overrides WHERE database_name IS NULL", ct)); + + /* A tag row is accepted; a row with neither scope, both scopes or a database on a tag is not; one value per tag and key. */ + await ExecAsync(store, "INSERT INTO config.server_tags (name) VALUES ('Production')", ct); + await ExecAsync(store, "INSERT INTO config.config_threshold_overrides (scope_tag_id, threshold_key, value, set_by) VALUES (1, 'cpu.threshold_percent', 80, 'erik')", ct); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (scope_tag_id, threshold_key, value, set_by) VALUES (1, 'cpu.threshold_percent', 70, 'erik')", ct)); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (threshold_key, value, set_by) VALUES ('cpu.threshold_percent', 70, 'erik')", ct)); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (server_id, scope_tag_id, threshold_key, value, set_by) VALUES (7, 1, 'blocking.count_threshold', 70, 'erik')", ct)); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (scope_tag_id, database_name, threshold_key, value, set_by) VALUES (1, 'shop', 'pvs.threshold_percent', 70, 'erik')", ct)); + + /* A second run changes nothing, and the version stamp is not repeated. */ + await ExecAsync(store, "DELETE FROM darling_schema_version WHERE version >= " + rung.ToString(CultureInfo.InvariantCulture), ct); + await PgMigrations.MigrateAsync(store, ct); + Assert.Equal("3", await ScalarAsync(store, "SELECT count(*) FROM config.config_threshold_overrides", ct)); + } + + private static async Task RunAsync(Func body) + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the threshold override live tests."); + + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await ExecAsync(connection, "INSERT INTO config.config_monitored_servers (server_id, name, host) VALUES (7, 'srv-7', 'h7'), (8, 'srv-8', 'h8'), (9, 'srv-9', 'h9')", ct); + await ExecAsync(connection, "INSERT INTO collect.servers (server_id, server_name, display_name, is_enabled, sql_major_version, created_date, modified_date) " + + "VALUES (7, 'srv-7', 'srv-7', TRUE, 15, now(), now()), (8, 'srv-8', 'srv-8', TRUE, 15, now(), now()), (9, 'srv-9', 'srv-9', TRUE, 15, now(), now())", ct); + /* Tag ids 1 Production, 2 east (under Production), 3 rack1 (under east), 4 dev; srv-7 is under east, srv-8 under Production, srv-9 under dev. */ + await ExecAsync(connection, "INSERT INTO config.server_tags (name, parent_id) VALUES ('Production', NULL), ('east', 1), ('rack1', 2), ('dev', NULL)", ct); + await ExecAsync(connection, "INSERT INTO config.server_tag_map (server_id, tag_id) VALUES (7, 2), (8, 1), (9, 4)", ct); + + var builder = new NpgsqlConnectionStringBuilder(scratch.ConnectionString) { SearchPath = PgSchemaGenerator.SearchPath }; + await using var dataSource = NpgsqlDataSource.Create(builder.ConnectionString); + var store = new InvestigationRequestStore(dataSource, 30, () => new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc)); + await body(dataSource, connection, store, ct); + } + + private static Task Set(NpgsqlDataSource ds, InvestigationRequestStore store, DarlingMcpWritePolicy.Effective policy, + string? server, string key, int? value, string? database = null, bool clear = false, string? setBy = null, string? tag = null) => + DarlingMcpThresholdOverrideTools.SetAsync(ds, policy, store, NullLogger.Instance, server, key, value, database, clear, setBy, null, + TestContext.Current.CancellationToken, null, tag); + + private static string Status(string json) => JsonNode.Parse(json)!["status"]!.GetValue(); + + private static string Reason(string json) => JsonNode.Parse(json)!["reason"]!.GetValue(); + + private static async Task ExecAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + return Convert.ToString(await command.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture) ?? ""; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs index aa54bd0fae..1ea10e46c2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs @@ -14,11 +14,12 @@ using Microsoft.Extensions.Logging; using Npgsql; using PerformanceMonitor.Alerting; +using PerformanceMonitor.Darling.Storage; namespace PerformanceMonitor.Darling.Service; /// -/// The per-server and per-database threshold overrides the alert engine and the PostgreSQL count checks resolve through +/// The per-server, per-database and per-tag threshold overrides the alert engine and the PostgreSQL count checks resolve through /// (#5916): the Darling loader behind . It holds one immutable snapshot of /// config.config_threshold_overrides and answers from it, so the sweep path does no I/O. The service reloads it at /// start and on every control-plane reload (the table is on the config_version beacon), and a failed load keeps the @@ -27,66 +28,101 @@ namespace PerformanceMonitor.Darling.Service; internal sealed class DarlingThresholdOverrides : IAlertThresholdOverrides { internal const string SelectSql = - "SELECT server_id, database_name, threshold_key, value FROM config.config_threshold_overrides"; + "SELECT id, server_id, database_name, scope_tag_id, threshold_key, value FROM config.config_threshold_overrides"; + + /// The direct (server and database) overrides, and the tag override that wins per server and key. + private sealed record Snapshot(Dictionary<(int Server, string Database, string Key), int> Direct, Dictionary<(int Server, string Key), int> Tag); private readonly ILogger? _logger; - private volatile Dictionary<(int Server, string Database, string Key), int> _snapshot = new(); + private volatile Snapshot _snapshot = new(new(), new()); internal DarlingThresholdOverrides(ILogger? logger = null) => _logger = logger; - /// How many overrides are in force. - internal int Count => _snapshot.Count; + /// How many overrides are in force: the server and database ones, and the tag ones that some server resolves to. + internal int Count => _snapshot.Direct.Count + _snapshot.Tag.Count; /// /// Replaces the snapshot with the table's rows. The read is awaited before the swap and nothing here catches, so a /// read that throws leaves the overrides already in force. A row whose key is not a tunable threshold, whose value is /// not an integer inside the key's bounds, or whose database name sits on a server-only key is skipped and logged, - /// never applied: the engine uses the value as-is. + /// never applied: the engine uses the value as-is. Tag overrides (#6926) are resolved with the tag tree and the server-to-tag + /// map, read only when the table holds a tag row; a tag-table read that fails leaves them matching nothing, as a tag-scoped route does. /// internal async Task LoadAsync(NpgsqlDataSource postgres, CancellationToken cancellationToken = default) { - var next = new Dictionary<(int, string, string), int>(); - await using var command = postgres.CreateCommand(SelectSql); - command.CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds; - await using var reader = await command.ExecuteReaderAsync(cancellationToken); - while (await reader.ReadAsync(cancellationToken)) + var direct = new List<(int ServerId, string? Database, string Key, double Value)>(); + var tag = new List<(long RowId, int TagId, string Key, double Value)>(); + await using (var command = postgres.CreateCommand(SelectSql)) { - var serverId = reader.GetInt32(0); - var database = reader.IsDBNull(1) ? string.Empty : reader.GetString(1); - var key = reader.GetString(2); - var raw = reader.GetDouble(3); - if (Accept(key, raw, database, out var value) is { } problem) + command.CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) { - LogSkipped(serverId, key, database, problem); - continue; + var key = reader.GetString(4); + var raw = reader.GetDouble(5); + if (!reader.IsDBNull(3)) + tag.Add((reader.GetInt64(0), reader.GetInt32(3), key, raw)); + else if (!reader.IsDBNull(1)) + direct.Add((reader.GetInt32(1), reader.IsDBNull(2) ? null : reader.GetString(2), key, raw)); } + } - next[(serverId, database, key)] = value; + StoreConfigProvider.RouteScopeInputs? inputs = null; + if (tag.Count > 0) + { + await using var connection = await postgres.OpenConnectionAsync(cancellationToken); + inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, _logger, cancellationToken); } - _snapshot = next; + Replace(direct, tag, inputs); } - private void LogSkipped(int serverId, string key, string database, string problem) => + private void LogSkipped(string scope, string key, string database, string problem) => _logger?.LogWarning( - "Threshold override for server {ServerId} key '{Key}' database '{Database}' was skipped: {Problem}", - serverId, key, database, problem); + "Threshold override for {Scope} key '{Key}' database '{Database}' was skipped: {Problem}", + scope, key, database, problem); /// Replaces the snapshot from rows already in hand (the loader's own tests, and any caller that read the table). A /// skipped row is logged the way logs it. - internal void Replace(IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows) + internal void Replace(IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows) => + Replace(rows, [], null); + + /// The same with tag overrides (#6926). carry the table row id, which breaks a tie between two + /// tags of the same depth, and the registry, tag tree and server-to-tag map the routes expand against + /// (null: the tag tables were not read, and a tag override matches nothing). + internal void Replace( + IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows, + IEnumerable<(long RowId, int TagId, string Key, double Value)> tagRows, + StoreConfigProvider.RouteScopeInputs? inputs) { - var next = new Dictionary<(int, string, string), int>(); + var direct = new Dictionary<(int, string, string), int>(); foreach (var (serverId, database, key, raw) in rows) { var db = database ?? string.Empty; if (Accept(key, raw, db, out var value) is { } problem) - LogSkipped(serverId, key, db, problem); + LogSkipped("server " + serverId.ToString(CultureInfo.InvariantCulture), key, db, problem); else - next[(serverId, db, key)] = value; + direct[(serverId, db, key)] = value; + } + + var accepted = new List(); + foreach (var (rowId, tagId, key, raw) in tagRows) + { + if (Accept(key, raw, string.Empty, out var value) is { } problem) + LogSkipped("tag " + tagId.ToString(CultureInfo.InvariantCulture), key, string.Empty, problem); + else + accepted.Add(new TagOverrideCandidate(rowId, tagId, key, value)); + } + + var tagWinners = new Dictionary<(int, string), int>(); + if (accepted.Count > 0 && inputs is not null) + { + foreach (var (slot, winner) in TagThresholdResolver.Resolve( + accepted, inputs.KnownServerIds, inputs.TagParents, inputs.ServerTags, inputs.TagsReadable)) + tagWinners[slot] = (int)winner.Value; } - _snapshot = next; + _snapshot = new Snapshot(direct, tagWinners); } private static string? Accept(string key, double raw, string database, out int value) @@ -106,18 +142,21 @@ public int Resolve(AlertThresholdKey key, string serverKey, string? database, in /// /// The value of the wire-named threshold for the server (and database): a database override first, then the - /// server's, else . is the store server id as text, the - /// engine's server key. This is the one resolution the PostgreSQL count thresholds use too. + /// server's, then the tag's (#6926: the deepest tag the server is under), else . + /// is the store server id as text, the engine's server key. This is the one resolution the + /// PostgreSQL count thresholds use too. /// internal int ResolveWire(string wire, string serverKey, string? database, int globalValue) { if (!int.TryParse(serverKey, NumberStyles.Integer, CultureInfo.InvariantCulture, out var serverId)) return globalValue; var snapshot = _snapshot; - if (snapshot.Count == 0) + if (snapshot.Direct.Count == 0 && snapshot.Tag.Count == 0) return globalValue; - if (!string.IsNullOrEmpty(database) && snapshot.TryGetValue((serverId, database, wire), out var perDatabase)) + if (!string.IsNullOrEmpty(database) && snapshot.Direct.TryGetValue((serverId, database, wire), out var perDatabase)) return perDatabase; - return snapshot.TryGetValue((serverId, string.Empty, wire), out var perServer) ? perServer : globalValue; + if (snapshot.Direct.TryGetValue((serverId, string.Empty, wire), out var perServer)) + return perServer; + return snapshot.Tag.TryGetValue((serverId, wire), out var perTag) ? perTag : globalValue; } } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs index 9c15d750b5..d8cc0d3eec 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs @@ -42,8 +42,9 @@ internal static string RefusedMessage(bool managedStore, bool adminWritesOff = f + "Darling/tools/provision-roles.sql re-run since the upgrade that added threshold overrides: run it once as a superuser (it is idempotent).", adminWritesOff); - /// The list body: {overrides: [{server_id, server_name, database, key, value, global_value, set_by, request_id, - /// updated_at}], keys: [{key, min, max, database_scope}], servers: [{server_id, server_name}], can_edit, note}. + /// The list body: {overrides: [{server_id, server_name, tag_id, tag_name, orphaned, database, key, value, global_value, set_by, + /// request_id, updated_at}], keys: [{key, min, max, database_scope}], servers: [{server_id, server_name}], tags: [{tag_id, tag_name}], + /// can_edit, note}. A tag override (#6926) has a tag_id, a null server_id and server_name, and orphaned true when its tag was deleted. /// can_edit is whether the store login holds the write grant, so a page can say list-only before anyone tries. internal static async Task ListBodyAsync(NpgsqlDataSource postgres, CancellationToken ct, NpgsqlDataSource? accessSource = null) { @@ -56,8 +57,11 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, { overrides.Add(new JsonObject { - ["server_id"] = r.ServerId, - ["server_name"] = r.ServerName, + ["server_id"] = r.IsTag ? null : r.ServerId, + ["server_name"] = r.IsTag ? null : r.ServerName, + ["tag_id"] = r.TagId, + ["tag_name"] = r.IsTag && !r.Orphaned ? r.ServerName : null, + ["orphaned"] = r.Orphaned, ["database"] = r.Database, ["key"] = r.Key, ["value"] = r.Value, @@ -95,11 +99,21 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, servers.Add(new JsonObject { ["server_id"] = reader.GetInt32(0), ["server_name"] = reader.GetString(1) }); } + var tags = new JsonArray(); + await using (var command = postgres.CreateCommand("SELECT id, name FROM config.server_tags ORDER BY name, id")) + { + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + tags.Add(new JsonObject { ["tag_id"] = reader.GetInt32(0), ["tag_name"] = reader.GetString(1) }); + } + return new JsonObject { ["overrides"] = overrides, ["keys"] = keys, ["servers"] = servers, + ["tags"] = tags, ["can_edit"] = await DarlingMcpThresholdOverrideTools.HoldsGrantAsync(accessSource ?? postgres, ct), ["note"] = note, }; @@ -116,6 +130,7 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, var server = body["server_name"]?.GetValueKind() == JsonValueKind.String ? body["server_name"]!.GetValue() : null; var key = body["key"]?.GetValueKind() == JsonValueKind.String ? body["key"]!.GetValue() : null; var database = body["database_name"]?.GetValueKind() == JsonValueKind.String ? body["database_name"]!.GetValue() : null; + var tag = body["tag"]?.GetValueKind() == JsonValueKind.String ? body["tag"]!.GetValue() : null; var clear = body["clear"]?.GetValueKind() == JsonValueKind.True; int? value = null; @@ -126,13 +141,13 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, value = parsed; } - if (string.IsNullOrWhiteSpace(server)) - return (400, new JsonObject { ["error"] = "server_name is required." }); + if (string.IsNullOrWhiteSpace(server) && string.IsNullOrWhiteSpace(tag)) + return (400, new JsonObject { ["error"] = "server_name or tag is required." }); var json = await DarlingMcpThresholdOverrideTools.SetAsync( postgres, new DarlingMcpWritePolicy.Effective(true, false), new InvestigationRequestStore(postgres, McpCommandDeadlines.ReadSeconds), logger, - server, key ?? string.Empty, value, database, clear, ThresholdOverrideStore.SetByFor(editor), null, ct); + server, key ?? string.Empty, value, database, clear, ThresholdOverrideStore.SetByFor(editor), null, ct, tag: tag); var answer = JsonNode.Parse(json)!.AsObject(); var status = answer["status"]?.GetValue(); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs index 521ad6b075..edab71bae2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.ComponentModel; using System.Globalization; +using System.Linq; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; @@ -67,12 +68,14 @@ public sealed class DarlingMcpThresholdOverrideTools "runs again) as a superuser. Managed and compose stores grant it themselves on start. Call again after the script has run."; [McpServerTool(Name = "set_alert_threshold_override"), Description( - "Sets or clears a threshold for ONE server (or one database on it), leaving the global value alone. Changes the " + + "Sets or clears a threshold for ONE server (or one database on it), or for a server TAG, leaving the global value alone. Changes the " + "monitoring store only; the service picks it up within its reload interval. <> " + "key is a tunable integer threshold named as get_alert_settings names it (for example cpu.threshold_percent, " + "blocking.pg_count_threshold); get_alert_settings with server_name lists the keys, the global value, the effective value " + "and where it comes from. value is an integer inside the same bounds the global setting has. clear: true removes the " + - "override so the global value applies again. database_name applies only to pvs.* and file_growth.rise_mb / " + + "override so the global value applies again. tag (a tag name or id, in place of server_name) holds the value for every server under that tag " + + "and every tag below it: the writes arm only, and no database_name. Precedence is database, then server, then tag (the deepest tag a server " + + "is under, as a notification route resolves it, the older override winning a same-depth tie), then global. database_name applies only to pvs.* and file_growth.rise_mb / " + "file_growth.volume_percent. Refused: a key outside that list, a self_alerts.*, enable, cooldown or route key, an " + "out-of-bounds value. To silence an alert, mute it. An agent cannot set a threshold to the value that turns its alert off (the top of the " + "key's range, or zero for the low_disk.* keys); that value is refused as threshold_disables_alert. A very high count below " + @@ -85,23 +88,24 @@ public static async Task SetAlertThresholdOverride( NpgsqlDataSource postgres, DarlingMcpServedPolicy served, ILogger logger, - [Description("Server name, or the start of it.")] string server_name, - [Description("A tunable threshold, as get_alert_settings names it, e.g. cpu.threshold_percent.")] string key, + [Description("Server name, or the start of it. Omit when tag is given.")] string? server_name = null, + [Description("A tunable threshold, as get_alert_settings names it, e.g. cpu.threshold_percent.")] string key = "", [Description("The override value (integer).")] int? value = null, [Description("Database on that server (pvs.*, file_growth.rise_mb, file_growth.volume_percent). Omit for the whole server.")] string? database_name = null, [Description("true removes the override.")] bool clear = false, [Description("Who made the change (writes arm). 1-64 of A-Za-z0-9._:-. Default mcp.")] string? set_by = null, [Description("Investigations arm only: the agent_action object (request_id, claim_id, agent_id, reason, evidence). Ignored under the writes arm.")] string? agent_action = null, + [Description("A server tag, by name or id, in place of server_name: the value applies to every server under the tag and the tags below it (#6926). Writes arm only; no database_name.")] string? tag = null, StoreOwnerSource? ownerSource = null, CancellationToken cancellationToken = default) => await SetAsync(postgres, served.Policy, new InvestigationRequestStore(postgres, ServiceCommandDeadlines.SerialLoopSeconds, agentActionSource: ownerSource?.Read()), - logger, server_name, key, value, database_name, clear, set_by, agent_action, cancellationToken, served.NamedAgentNames); + logger, server_name, key, value, database_name, clear, set_by, agent_action, cancellationToken, served.NamedAgentNames, tag); /// The tool body over a given store, so a test drives either arm. internal static async Task SetAsync( NpgsqlDataSource postgres, DarlingMcpWritePolicy.Effective policy, InvestigationRequestStore store, ILogger logger, - string server_name, string key, int? value, string? database_name, bool clear, string? set_by, string? agent_action, - CancellationToken cancellationToken, IReadOnlySet? namedAgentNames = null) + string? server_name, string key, int? value, string? database_name, bool clear, string? set_by, string? agent_action, + CancellationToken cancellationToken, IReadOnlySet? namedAgentNames = null, string? tag = null) { try { @@ -110,6 +114,15 @@ internal static async Task SetAsync( DarlingThresholdKeys.TryGet(key, out var entry); var wire = entry.Wire; var database = string.IsNullOrWhiteSpace(database_name) ? null : database_name.Trim(); + var hasTag = !string.IsNullOrWhiteSpace(tag); + if (hasTag && !string.IsNullOrWhiteSpace(server_name)) + return Refused("scope_conflict", "Give a server_name or a tag, not both."); + if (!hasTag && string.IsNullOrWhiteSpace(server_name)) + return Refused("scope_required", "server_name (or tag) is required."); + if (hasTag && database is not null) + return Refused("database_scope", "A tag override holds a server-level value, so it takes no database_name."); + if (hasTag && !policy.Writes) + return Refused("tag_writes_arm_only", "A tag override changes every server under the tag, so it is set on the writes arm, not by an investigation request's agent, whose change is for the request's server only."); if (!clear) { @@ -131,7 +144,10 @@ internal static async Task SetAsync( return Refused("set_by_invalid", "set_by must be 1-64 of A-Za-z0-9._:-."); } - var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name, cancellationToken, NameClash.Refuse); + if (hasTag) + return await SetTagAsync(postgres, tag!, wire, value, clear, person!, cancellationToken); + + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name!, cancellationToken, NameClash.Refuse); if (error is not null) return error; @@ -225,6 +241,52 @@ internal static async Task SetAsync( } } + /// The tag branch of (#6926), writes arm only: resolve the tag, then set or clear its value in one transaction. + /// A tag with no servers under it is allowed (servers are tagged later and take the value then). + private static async Task SetTagAsync( + NpgsqlDataSource postgres, string tagText, string wire, int? value, bool clear, string by, CancellationToken cancellationToken) + { + var (tagId, tagName, problem) = await ThresholdOverrideStore.ResolveTagAsync(postgres, tagText, cancellationToken); + /* A clear by id still works for an override whose tag was deleted (it is listed as orphaned): the row is found by its tag id below. */ + var orphanClear = problem is not null && clear + && int.TryParse(tagText.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var orphanId) && (tagId = orphanId) is not null; + if (problem is not null && !orphanClear) + return Refused("tag_not_found", problem); + if (!await HoldsGrantAsync(postgres, cancellationToken)) + return McpHelpers.Status("precondition", PreconditionMessage); + + double? previous; + await using (var connection = await postgres.OpenConnectionAsync(cancellationToken)) + await using (var transaction = await connection.BeginTransactionAsync(cancellationToken)) + { + previous = await ThresholdOverrideStore.ReadTagValueAsync(connection, transaction, tagId!.Value, wire, cancellationToken); + if (orphanClear && previous is null) + return Refused("tag_not_found", problem!); + if (clear) + await ThresholdOverrideStore.DeleteTagAsync(connection, transaction, tagId.Value, wire, cancellationToken); + else + await ThresholdOverrideStore.UpsertTagAsync(connection, transaction, tagId.Value, wire, value!.Value, by, null, cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + + return JsonSerializer.Serialize(new + { + status = clear ? "cleared" : "ok", + tag = tagName, + tag_id = tagId, + key = wire, + value = clear ? (int?)null : value, + previous_override = previous, + undo = DarlingMcpAlertTools.UndoCall(ToolName, previous is { } was + ? new System.Collections.Generic.Dictionary { ["tag"] = tagId!.Value.ToString(CultureInfo.InvariantCulture), ["key"] = wire, ["value"] = was } + : new System.Collections.Generic.Dictionary { ["tag"] = tagId!.Value.ToString(CultureInfo.InvariantCulture), ["key"] = wire, ["clear"] = true }), + set_by = by, + message = clear + ? "The tag override is removed; servers under the tag fall back to a shallower tag, or the global value." + : "The tag override is stored. The service applies it within its reload interval to every server under the tag that has no server or database override of its own." + }, McpHelpers.JsonOptions); + } + private sealed class Applied { internal double? Previous; @@ -264,8 +326,8 @@ internal static async Task HoldsGrantAsync(NpgsqlDataSource postgres, Canc /// /// The get_alert_settings view for one server (#5916): every tunable threshold with its global value, the value in - /// effect for this server and where that value comes from (global or server_override), and the database - /// overrides the server holds. A read of the overrides table, so it answers what the service will apply after its next reload. + /// effect for this server and where that value comes from (global, tag_override naming the tag, or server_override), + /// and the database overrides the server holds. A read of the overrides table, so it answers what the service will apply after its next reload. /// internal static async Task BuildServerViewAsync( NpgsqlDataSource postgres, DarlingAlertReader.AlertSettingsReadRow s, int serverId, string serverName, @@ -291,6 +353,56 @@ internal static async Task BuildServerViewAsync( note = PreconditionMessage; } + /* #6926: the tag overrides that reach this server, resolved by the notification routes' own tag rule (TagThresholdResolver), the same + one the service loads. A tag read that fails reaches no server: the view then shows what the engine would apply. */ + var tagWinners = new System.Collections.Generic.Dictionary(); + try + { + var candidates = new System.Collections.Generic.List(); + var meta = new System.Collections.Generic.Dictionary<(int Tag, string Key), (string SetBy, DateTime UpdatedAt)>(); + await using (var command = postgres.CreateCommand( + "SELECT id, scope_tag_id, threshold_key, value, set_by, updated_at FROM " + Table + " WHERE scope_tag_id IS NOT NULL")) + { + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var key = reader.GetString(2); + var raw = reader.GetDouble(3); + if (!DarlingThresholdKeys.TryGet(key, out _) || raw != Math.Floor(raw) || raw < int.MinValue || raw > int.MaxValue) + continue; + candidates.Add(new TagOverrideCandidate(reader.GetInt64(0), reader.GetInt32(1), key, raw)); + meta[(reader.GetInt32(1), key)] = (reader.GetString(4), reader.GetDateTime(5)); + } + } + + if (candidates.Count > 0) + { + await using var connection = await postgres.OpenConnectionAsync(cancellationToken); + var inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, null, cancellationToken); + var names = new System.Collections.Generic.Dictionary(); + await using (var command = new NpgsqlCommand("SELECT id, name FROM config.server_tags", connection) { CommandTimeout = McpCommandDeadlines.ReadSeconds }) + await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) + { + while (await reader.ReadAsync(cancellationToken)) + names[reader.GetInt32(0)] = reader.GetString(1); + } + + foreach (var ((server, key), winner) in TagThresholdResolver.Resolve( + candidates, inputs.KnownServerIds, inputs.TagParents, inputs.ServerTags, inputs.TagsReadable)) + { + if (server != serverId) + continue; + var m = meta[(winner.TagId, key)]; + tagWinners[key] = (winner.TagId, names.GetValueOrDefault(winner.TagId, string.Empty), winner.Value, m.SetBy, m.UpdatedAt); + } + } + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.InsufficientPrivilege) + { + note ??= PreconditionMessage; + } + /* One number per key, read the way the engine reads it (#5916): the global value and the value with this server's override applied each go through the same pair clamp the enclosing get_alert_settings payload applies, so an override that inverts a warning/critical pair shows the pair the engine evaluates, not the stored one. */ @@ -305,19 +417,25 @@ internal static async Task BuildServerViewAsync( return null; } + int? TagOverride(string wire) => tagWinners.TryGetValue(wire, out var t) ? (int)t.Value : null; + var thresholds = new System.Collections.Generic.List(); foreach (var entry in DarlingThresholdKeys.Entries) { var global = GlobalValueAsEngineReadsIt(s, entry.Wire); var server = ServerOverride(entry.Wire); - var effective = Evaluated(entry.Wire, w => ServerOverride(w) ?? GlobalValue(s, w)); + var effective = Evaluated(entry.Wire, w => ServerOverride(w) ?? TagOverride(w) ?? GlobalValue(s, w)); + var found = tagWinners.TryGetValue(entry.Wire, out var tw); + var tagHit = server is null && found; thresholds.Add(new { key = entry.Wire, global_value = global, effective_value = effective, - source = server is null ? "global" : "server_override", + source = server is not null ? "server_override" : tagHit ? "tag_override" : "global", + tag = tagHit ? tw.TagName : null, + tag_id = tagHit ? tw.TagId : (int?)null, database_scope = entry.Scope == PerformanceMonitor.Alerting.AlertThresholdScope.ServerAndDatabase }); } @@ -338,10 +456,15 @@ internal static async Task BuildServerViewAsync( { server = serverName, server_id = serverId, - note = note ?? "effective_value is the global value with this server's override applied; a database override (database_overrides) applies to that database only. Set one with set_alert_threshold_override.", + note = note ?? "effective_value is the global value with this server's override applied, else the deepest server tag's (source tag_override, naming the tag; tag_overrides lists the ones that reach this server), else global; a database override (database_overrides) applies to that database only. Set one with set_alert_threshold_override.", thresholds, server_overrides = serverOverrides, - database_overrides = databaseOverrides + database_overrides = databaseOverrides, + tag_overrides = tagWinners.Select(t => (object)new + { + key = t.Key, tag = t.Value.TagName, tag_id = t.Value.TagId, value = t.Value.Value, set_by = t.Value.SetBy, + updated_at = t.Value.UpdatedAt.ToString("o", CultureInfo.InvariantCulture) + }).ToList() }; } diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js index 0abc05db92..ee27ef9ebf 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js @@ -1807,12 +1807,12 @@ async function buildPending(body, generation) { /* ─────────────────────────── threshold overrides (#5916) ─────────────────────────── */ -/* A threshold held for one server (or one database on it) that differs from the global value. The list is GET +/* A threshold held for one server (or one database on it), or for a server tag (#6926), that differs from the global value. The list is GET /api/threshold-overrides; a change is PUT /api/threshold-overrides, accepted only under a store login with the admin role. A managed or compose store connects this page as the viewer role, so there the grid is a list: the form and the Remove buttons are not drawn, and the note says where to change one. The service checks the key and the bounds again. */ export const OVERRIDE_COLUMNS = [ - { key: "server_name", label: "Server" }, + { key: "scope_text", label: "Scope" }, { key: "database_text", label: "Database" }, { key: "key", label: "Key" }, { key: "value", label: "Value" }, @@ -1823,10 +1823,10 @@ export const OVERRIDE_COLUMNS = [ ]; const OVERRIDES_NOTE_EDITING = - "A threshold held for one server, or for one database on it, that differs from the global value. The alert engine uses it for that server only " + - "and applies a change within its reload interval. To silence an alert, mute it."; + "A threshold held for one server, for one database on it, or for a server tag (every server under the tag and the tags below it), that differs from the global value. " + + "A database override wins over a server override, then the deepest tag the server is under, then the global value. A change applies within the alert engine's reload interval. To silence an alert, mute it."; const overridesNoteReadOnly = (managed, adminWritesOff) => - "A threshold held for one server, or for one database on it, that differs from the global value. " + + "A threshold held for one server, for one database on it, or for a server tag, that differs from the global value. " + listOnlyNote(managed, "this page lists them only", "To change an override", adminWritesOff); const OVERRIDE_DENIED = "Changing a threshold override needs"; @@ -1834,8 +1834,12 @@ const OVERRIDE_DENIED = "Changing a threshold override needs"; export function overrideRows(data) { const list = data && Array.isArray(data.overrides) ? data.overrides : []; return list.map((o) => ({ - server_id: o.server_id, - server_name: o.server_name, + server_id: o.server_id == null ? null : o.server_id, + server_name: o.server_name == null ? null : o.server_name, + tag_id: o.tag_id == null ? null : o.tag_id, + tag_name: o.tag_name == null ? null : o.tag_name, + orphaned: o.orphaned === true, + scope_text: o.tag_id == null ? o.server_name : (o.orphaned === true ? "Tag (deleted): orphaned, matches nothing" : "Tag: " + (o.tag_name || "")), database: o.database || null, database_text: o.database || "Whole server", key: o.key, @@ -1851,7 +1855,10 @@ export function overrideRows(data) { inside its bounds, and a database only on a key that takes one. The service applies the same rules again. */ export function validateOverride(keys, form) { const list = Array.isArray(keys) ? keys : []; - if (!form || !String(form.server || "").trim()) return "Choose a server."; + const hasServer = !!(form && String(form.server || "").trim()); + const hasTag = !!(form && String(form.tag || "").trim()); + if (hasServer && hasTag) return "Choose a server or a tag, not both."; + if (!hasServer && !hasTag) return "Choose a server or a tag."; const entry = list.find((k) => k.key === String(form.key || "").trim()); if (!entry) return "Choose a key from the list: only a tunable threshold can be overridden."; const text = String(form.value == null ? "" : form.value).trim(); @@ -1860,6 +1867,9 @@ export function validateOverride(keys, form) { if (!Number.isSafeInteger(n) || n < entry.min || n > entry.max) { return entry.key + " must be a whole number between " + entry.min + " and " + entry.max + ", the same bounds the global setting has."; } + if (hasTag && String(form.database || "").trim()) { + return "A tag override holds a server-level value, so it takes no database."; + } if (String(form.database || "").trim() && entry.database_scope !== true) { return entry.key + " is compared against a server-level figure, so it takes a server override only, not a database."; } @@ -1887,7 +1897,7 @@ export function overrideOutcome(verb, res) { } /* The form's entries and the last answer, in module scope so the 60 s repaint keeps what was typed. */ -const EMPTY_OVERRIDE_FORM = { server: "", key: "", database: "", value: "" }; +const EMPTY_OVERRIDE_FORM = { server: "", tag: "", key: "", database: "", value: "" }; let overrideEntries = { ...EMPTY_OVERRIDE_FORM }; let overrideStatus = null; let overrideBusy = false; @@ -1903,7 +1913,9 @@ async function changeOverride(verb, entry) { overrideBusy = true; let out; try { - const payload = { server_name: entry.server, key: entry.key, database_name: entry.database || null }; + const payload = { key: entry.key, database_name: entry.database || null }; + if (entry.tag) payload.tag = String(entry.tag); + else payload.server_name = entry.server; if (verb === "clear") payload.clear = true; else payload.value = Number(String(entry.value).trim()); out = overrideOutcome(verb, await apiWrite("PUT", "/api/threshold-overrides", payload)); @@ -1916,19 +1928,22 @@ async function changeOverride(verb, entry) { await redrawOverrides(); } -function overrideFormNode(servers, keys) { +function overrideFormNode(servers, keys, tags) { const field = (label, control) => el("label", { class: "tag-field" }, [el("span", { class: "mute-label", text: label }), control]); const choose = (items, textOf) => [el("option", { value: "", text: "(choose)" })].concat(items.map((i) => el("option", { value: textOf(i), text: textOf(i) }))); const server = el("select", { class: "tag-input", "data-field": "server" }, choose(servers, (s) => s.server_name)); + const tag = el("select", { class: "tag-input", "data-field": "tag" }, + [el("option", { value: "", text: "(none)" })].concat((Array.isArray(tags) ? tags : []).map((t) => el("option", { value: String(t.tag_id), text: t.tag_name })))); const key = el("select", { class: "tag-input", "data-field": "key" }, choose(keys, (k) => k.key)); const database = el("input", { type: "text", class: "tag-input", "data-field": "database", placeholder: "blank = whole server" }); const value = el("input", { type: "text", class: "tag-input", "data-field": "value", inputmode: "numeric" }); server.value = overrideEntries.server; + tag.value = overrideEntries.tag; key.value = overrideEntries.key; database.value = overrideEntries.database; value.value = overrideEntries.value; - const sync = () => { overrideEntries = { server: server.value, key: key.value, database: database.value, value: value.value }; }; - for (const c of [server, key, database, value]) c.addEventListener("input", sync); + const sync = () => { overrideEntries = { server: server.value, tag: tag.value, key: key.value, database: database.value, value: value.value }; }; + for (const c of [server, tag, key, database, value]) c.addEventListener("input", sync); const save = el("button", { class: "btn primary", type: "button", text: "Save override", "data-action": "save-override", onClick: async () => { @@ -1943,21 +1958,21 @@ function overrideFormNode(servers, keys) { }, }); return el("div", { class: "tag-form", "data-override-form": "1" }, [ - field("Server:", server), field("Key:", key), field("Database:", database), field("Value:", value), save, + field("Server:", server), field("or Tag:", tag), field("Key:", key), field("Database:", database), field("Value:", value), save, ]); } function overrideActions(row) { const edit = el("button", { - class: "btn", type: "button", text: "Edit", "aria-label": "Edit " + row.key + " for " + row.server_name, + class: "btn", type: "button", text: "Edit", "aria-label": "Edit " + row.key + " for " + row.scope_text, onClick: async () => { - overrideEntries = { server: row.server_name, key: row.key, database: row.database || "", value: String(row.value) }; + overrideEntries = { server: row.tag_id == null ? row.server_name : "", tag: row.tag_id == null ? "" : String(row.tag_id), key: row.key, database: row.database || "", value: String(row.value) }; await redrawOverrides(); }, }); const remove = el("button", { - class: "btn", type: "button", text: "Remove", "aria-label": "Remove " + row.key + " for " + row.server_name, "data-action": "remove-override", - onClick: () => changeOverride("clear", { server: row.server_name, key: row.key, database: row.database }), + class: "btn", type: "button", text: "Remove", "aria-label": "Remove " + row.key + " for " + row.scope_text, "data-action": "remove-override", + onClick: () => changeOverride("clear", { server: row.tag_id == null ? row.server_name : "", tag: row.tag_id == null ? "" : String(row.tag_id), key: row.key, database: row.database }), }); return el("span", { class: "row-actions" }, [edit, remove]); } @@ -1981,11 +1996,11 @@ async function buildOverrides(body, generation) { const keys = Array.isArray(res.data && res.data.keys) ? res.data.keys : []; nodes.push(noticeStrip(canEdit ? OVERRIDES_NOTE_EDITING : overridesNoteReadOnly(session.managed_store === true, session.admin_writes_off === true))); if (res.data && typeof res.data.note === "string" && res.data.note) nodes.push(noticeStrip(res.data.note)); - if (canEdit) nodes.push(overrideFormNode(Array.isArray(res.data.servers) ? res.data.servers : [], keys)); + if (canEdit) nodes.push(overrideFormNode(Array.isArray(res.data.servers) ? res.data.servers : [], keys, res.data.tags)); nodes.push(VIZ.table({ overrides: overrideRows(res.data) }, { rowsKey: "overrides", columns: overrideColumns(canEdit), - emptyText: "No server holds an override: every server uses the global values.", + emptyText: "No server or tag holds an override: every server uses the global values.", })); mount(body, nodes); } diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index c65424e31c..8ca7eb987e 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -2975,8 +2975,52 @@ holds the ordering. */ new Migration(224, "identity-capacity", V224Sql), new Migration(225, "server-aws-rds", V225Sql), new Migration(226, "server-aws-rds-web-or-express", V226Sql), + new Migration(227, "threshold-override-tag-scope", V227Sql), }; + /// + /// V227 (#6926): a threshold override can be held for a server tag. config.config_threshold_overrides gains the nullable + /// scope_tag_id (a config.server_tags id), server_id becomes nullable, and a CHECK says exactly one of the two is set, + /// so a stored row (server set, tag NULL) keeps its meaning and stays valid. A tag row holds no database (CHECK), because a tag value is + /// the server-level value for every server under the tag. A second unique index keeps one value per tag and key, since the V191 index is + /// on server_id and NULLs never collide. + /// + /// No foreign key on the tag, deliberately, as V181 did for a route: an override whose tag was deleted stays as written and + /// shows as orphaned, and the service resolves it to nothing. A cascade would delete the person's setting, and SET NULL would break the + /// exactly-one CHECK. The table's reload trigger (V191) and the tag tables' (V181) already bump the config version, so the service + /// re-reads when a tag moves. No GRANT: the table-level grants of V191 (admin and mcp write, viewer read) cover the new column. Every + /// statement is idempotent. No Lite twin: Lite has no overrides. + /// + private const string V227Sql = @" +ALTER TABLE config.config_threshold_overrides + ADD COLUMN IF NOT EXISTS scope_tag_id integer; + +ALTER TABLE config.config_threshold_overrides + ALTER COLUMN server_id DROP NOT NULL; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ck_config_threshold_overrides_one_scope' + AND conrelid = 'config.config_threshold_overrides'::regclass) THEN + ALTER TABLE config.config_threshold_overrides + ADD CONSTRAINT ck_config_threshold_overrides_one_scope + CHECK ((server_id IS NOT NULL) <> (scope_tag_id IS NOT NULL)); + END IF; + + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ck_config_threshold_overrides_tag_no_database' + AND conrelid = 'config.config_threshold_overrides'::regclass) THEN + ALTER TABLE config.config_threshold_overrides + ADD CONSTRAINT ck_config_threshold_overrides_tag_no_database + CHECK (scope_tag_id IS NULL OR database_name IS NULL); + END IF; +END $$; + +CREATE UNIQUE INDEX IF NOT EXISTS ux_config_threshold_overrides_tag + ON config.config_threshold_overrides (scope_tag_id, threshold_key) + WHERE scope_tag_id IS NOT NULL;"; + /// /// V225 (#6894): collect.servers.is_aws_rds, whether the target is an Amazon RDS for SQL Server instance (the connect query's /// DB_ID('rdsadmin') IS NOT NULL), written on every connect. The collectors that RDS gates off (running jobs, Agent status, Agent alerts) diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs index bfebc69b62..e669ce62e7 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs @@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class StorageVersion { - public const int SchemaVersion = 226; + public const int SchemaVersion = 227; } diff --git a/Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs b/Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs new file mode 100644 index 0000000000..2a6f88ae5c --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using PerformanceMonitor.Notifications; + +namespace PerformanceMonitor.Darling.Storage; + +/// One stored tag override: the table row's id, its tag, the wire key and the value. +public sealed record TagOverrideCandidate(long RowId, int TagId, string Key, double Value); + +/// The tag override that applies to a server for one key: the tag it comes from and its value. +public sealed record TagOverrideWinner(int TagId, double Value); + +/// +/// Which tag override applies to a server (#6926). It is the notification routes' rule, run by the routes' own code: each tag that +/// holds an override is expanded by exactly as a tag-scoped route is (a tag covers the servers +/// mapped to it or to any tag below it, a tag that no longer exists is orphaned and covers nothing, and tag tables that could not be read +/// cover nothing and are not called orphaned). When a server is covered by several tags that hold the same key, the deepest tag wins +/// (, the number of ancestors) and a tie goes to the lowest row id, the way the router's +/// ordering takes the lowest route id after depth. Pure: the service, the MCP view and the tests share this one resolution. +/// +public static class TagThresholdResolver +{ + /// The winning tag override per (server, key) for every server some tag covers. + public static IReadOnlyDictionary<(int ServerId, string Key), TagOverrideWinner> Resolve( + IReadOnlyCollection rows, + IReadOnlyCollection knownServerIds, + IReadOnlyDictionary tagParents, + IReadOnlyDictionary> serverTags, + bool tagsReadable = true) + { + ArgumentNullException.ThrowIfNull(rows); + if (rows.Count == 0) + return new Dictionary<(int ServerId, string Key), TagOverrideWinner>(); + + var tagIds = rows.Select(r => r.TagId).Distinct().ToList(); + var routes = tagIds + .Select(t => new NotificationRoute(t, NotificationRoute.WildcardMatch, "", "", "", "", "", true) { ScopeTagId = t }) + .ToList(); + var expanded = NotificationRouteScopeExpander.Expand(routes, knownServerIds, tagParents, serverTags, tagsReadable); + var byTag = new Dictionary(); + for (var i = 0; i < tagIds.Count; i++) + byTag[tagIds[i]] = expanded[i]; + + var winners = new Dictionary<(int ServerId, string Key), (int Depth, long RowId, TagOverrideWinner Winner)>(); + foreach (var row in rows) + { + var route = byTag[row.TagId]; + if (route.Orphaned || route.ExpandedServerIds is null) + continue; + foreach (var serverId in route.ExpandedServerIds) + { + var slot = (serverId, row.Key); + if (winners.TryGetValue(slot, out var current) + && (current.Depth > route.ScopeTagDepth || (current.Depth == route.ScopeTagDepth && current.RowId <= row.RowId))) + continue; + winners[slot] = (route.ScopeTagDepth, row.RowId, new TagOverrideWinner(row.TagId, row.Value)); + } + } + + return winners.ToDictionary(p => p.Key, p => p.Value.Winner); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs new file mode 100644 index 0000000000..b4d9a5ab65 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using NpgsqlTypes; + +namespace PerformanceMonitor.Darling.Storage; + +/// The tag-scoped statements of (#6926): a threshold held for a server tag, the deepest tag +/// a server is under winning as a notification route's does (). +public static partial class ThresholdOverrideStore +{ + private static NpgsqlCommand TagCommand(NpgsqlConnection connection, NpgsqlTransaction transaction, string sql, int tagId, string wire) + { + var command = new NpgsqlCommand(sql, connection) { Transaction = transaction, CommandTimeout = CommandTimeoutSeconds }; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = tagId }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = wire }); + return command; + } + + /// The tag's stored value for a key, or null. + public static async Task ReadTagValueAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, int tagId, string wire, CancellationToken ct) + { + await using var command = TagCommand(connection, transaction, + "SELECT value FROM " + Table + " WHERE scope_tag_id = $1 AND threshold_key = $2", tagId, wire); + return await command.ExecuteScalarAsync(ct) is double d ? d : null; + } + + public static async Task DeleteTagAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, int tagId, string wire, CancellationToken ct) + { + await using var command = TagCommand(connection, transaction, + "DELETE FROM " + Table + " WHERE scope_tag_id = $1 AND threshold_key = $2", tagId, wire); + await command.ExecuteNonQueryAsync(ct); + } + + public static async Task UpsertTagAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, int tagId, string wire, int value, + string setBy, Guid? requestId, CancellationToken ct) + { + await using var command = TagCommand(connection, transaction, + "INSERT INTO " + Table + " (scope_tag_id, threshold_key, value, set_by, request_id) VALUES ($1, $2, $3, $4, $5) " + + "ON CONFLICT (scope_tag_id, threshold_key) WHERE scope_tag_id IS NOT NULL DO UPDATE " + + "SET value = EXCLUDED.value, set_by = EXCLUDED.set_by, request_id = EXCLUDED.request_id, updated_at = now()", + tagId, wire); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Double, Value = (double)value }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = setBy }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Uuid, Value = (object?)requestId ?? DBNull.Value }); + await command.ExecuteNonQueryAsync(ct); + } + + /// Finds a tag from the text a person or an agent gave: a number is a tag id, anything else is a name matched without regard + /// to case. A name two tags share, or one that matches nothing, comes back as a refusal sentence. + public static async Task<(int? TagId, string? Name, string? Problem)> ResolveTagAsync(NpgsqlDataSource postgres, string text, CancellationToken ct) + { + var wanted = text.Trim(); + if (wanted.Length == 0) + return (null, null, "A tag name or id is required."); + var matches = new List<(int Id, string Name)>(); + await using var command = postgres.CreateCommand( + "SELECT id, name FROM config.server_tags WHERE id::text = $1 OR lower(name) = lower($1) ORDER BY id"); + command.CommandTimeout = CommandTimeoutSeconds; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = wanted }); + await using (var reader = await command.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + matches.Add((reader.GetInt32(0), reader.GetString(1))); + } + + var byId = matches.Where(m => m.Id.ToString(CultureInfo.InvariantCulture) == wanted).ToList(); + if (byId.Count == 1) + return (byId[0].Id, byId[0].Name, null); + if (matches.Count == 1) + return (matches[0].Id, matches[0].Name, null); + return matches.Count == 0 + ? (null, null, $"No server tag is named or numbered '{wanted}'.") + : (null, null, $"More than one server tag is named '{wanted}' (ids {string.Join(", ", matches.Select(m => m.Id))}): use the tag id."); + } + + /// The tag list statement: every tag override with its tag's name. A tag that was deleted has no name and is orphaned. + public const string TagListSql = + "SELECT o.scope_tag_id, t.name, o.threshold_key, o.value, o.set_by, o.request_id, o.updated_at " + + "FROM " + Table + " AS o LEFT JOIN config.server_tags AS t ON t.id = o.scope_tag_id " + + "WHERE o.scope_tag_id IS NOT NULL ORDER BY t.name NULLS LAST, o.scope_tag_id, o.threshold_key"; +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs index 3d11d4b0b6..9eb8a4584a 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs @@ -20,7 +20,12 @@ namespace PerformanceMonitor.Darling.Storage; /// override replaces (null when the store has no global row or no such column yet). public sealed record ThresholdOverrideRow( int ServerId, string ServerName, string? Database, string Key, double Value, double? GlobalValue, - string SetBy, Guid? RequestId, DateTime UpdatedAt); + string SetBy, Guid? RequestId, DateTime UpdatedAt, int? TagId = null, bool Orphaned = false) +{ + /// True for a row held for a tag (#6926): is set, is 0, + /// holds the tag's name (empty when the tag was deleted) and says the tag no longer exists, so the row matches nothing. + public bool IsTag => TagId.HasValue; +} /// /// The statements behind config.config_threshold_overrides (#5916), shared by the set_alert_threshold_override tool, @@ -28,7 +33,7 @@ public sealed record ThresholdOverrideRow( /// and transaction (the tool journals its write in the same transaction) or a data source (the list). Key and bound checks are /// not here: they are DarlingThresholdKeys' (PerformanceMonitor.Alerting), applied by every caller before any write. /// -public static class ThresholdOverrideStore +public static partial class ThresholdOverrideStore { /// The table, schema-qualified; the mcp role holds SELECT, INSERT, UPDATE and DELETE on it (#5920). public const string Table = "config.config_threshold_overrides"; @@ -141,6 +146,28 @@ public static async Task> ListAsync( reader.GetString(5), reader.IsDBNull(6) ? null : reader.GetGuid(6), reader.GetDateTime(7))); } + await reader.DisposeAsync(); + + await using var tagCommand = postgres.CreateCommand(TagListSql); + tagCommand.CommandTimeout = CommandTimeoutSeconds; + await using var tagReader = await tagCommand.ExecuteReaderAsync(ct); + while (await tagReader.ReadAsync(ct)) + { + var key = tagReader.GetString(2); + double? globalValue = null; + if (global is { } g && columnFor(key) is { } column + && g.TryGetProperty(column, out var element) && element.ValueKind == JsonValueKind.Number) + { + globalValue = element.GetDouble(); + } + + var name = tagReader.IsDBNull(1) ? null : tagReader.GetString(1); + rows.Add(new ThresholdOverrideRow( + 0, name ?? string.Empty, null, key, tagReader.GetDouble(3), globalValue, + tagReader.GetString(4), tagReader.IsDBNull(5) ? null : tagReader.GetGuid(5), tagReader.GetDateTime(6), + tagReader.GetInt32(0), name is null)); + } + return rows; } } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs index f5e4df60fc..a99e6db89c 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs @@ -25,10 +25,18 @@ public interface IThresholdOverrideGateway Task SaveThresholdOverrideAsync(int serverId, string? database, string key, int value, string setBy, CancellationToken cancellationToken = default); Task RemoveThresholdOverrideAsync(int serverId, string? database, string key, CancellationToken cancellationToken = default); + + /// Sets a tag's override (#6926): the value for every server under the tag and the tags below it. + Task SaveThresholdOverrideForTagAsync(int tagId, string key, int value, string setBy, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + Task RemoveThresholdOverrideForTagAsync(int tagId, string key, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); } -/// One server the add form can pick. -public sealed record ThresholdOverrideServerChoice(int ServerId, string Name); +/// One scope the add form can pick: a server, or ( set, #6926) a server tag, whose +/// reads Tag: name and whose is 0. +public sealed record ThresholdOverrideServerChoice(int ServerId, string Name, int? TagId = null); /// The grid's row: the stored override as the window shows it. public sealed class ThresholdOverrideViewRow @@ -39,6 +47,11 @@ public sealed class ThresholdOverrideViewRow public int ServerId => Row.ServerId; public string ServerName => Row.ServerName; + public bool IsTag => Row.IsTag; + public int? TagId => Row.TagId; + /// The Scope column (#6926): the server's name, or Tag: name for a tag override. A tag override whose tag was deleted reads + /// Tag: (deleted) - orphaned, matches nothing, as an orphaned notification route is marked. + public string ScopeText => !Row.IsTag ? Row.ServerName : Row.Orphaned ? "Tag: (deleted) - orphaned, matches nothing" : "Tag: " + Row.ServerName; public string? Database => Row.Database; public string DatabaseText => Row.Database ?? "Whole server"; public string Key => Row.Key; @@ -91,6 +104,18 @@ public ThresholdOverridesViewModel(IThresholdOverrideGateway gateway, string log return DarlingThresholdKeys.ValidateValue(entry, value, database)?.Message; } + /// Why a tag entry is refused, or null: a tag must be chosen, the key must be a tunable threshold, the value a whole number + /// inside its bounds, and no database is given, because a tag value is the server-level value for every server under the tag. + public static string? ValidateForTag(int? tagId, string? key, string? valueText, string? databaseText, out int value) + { + value = 0; + if (tagId is null) + return "Choose a tag."; + if (!string.IsNullOrWhiteSpace(databaseText)) + return "A tag override holds a server-level value, so it takes no database. Clear the Database box."; + return Validate(0, key, valueText, null, out value); + } + public async Task> LoadAsync(CancellationToken cancellationToken = default) => (await _gateway.ListThresholdOverridesAsync(cancellationToken)).Select(r => new ThresholdOverrideViewRow(r)).ToList(); @@ -115,6 +140,26 @@ public async Task SaveAsync( return new ThresholdOverrideOutcome(true, $"Override saved: {entry.Wire} = {value.ToString(CultureInfo.CurrentCulture)}. The service applies it within its reload interval."); } + /// Checks a tag entry, then writes it (#6926). An entry that fails the check changes nothing. + public async Task SaveForTagAsync( + int? tagId, string? key, string? valueText, string? databaseText, CancellationToken cancellationToken = default) + { + if (ValidateForTag(tagId, key, valueText, databaseText, out var value) is { } refusal) + return new ThresholdOverrideOutcome(false, refusal); + + DarlingThresholdKeys.TryGet(key, out var entry); + try + { + await _gateway.SaveThresholdOverrideForTagAsync(tagId!.Value, entry.Wire, value, _setBy, cancellationToken); + } + catch (ViewerReadOnlyException ex) + { + return new ThresholdOverrideOutcome(false, ex.Message); + } + + return new ThresholdOverrideOutcome(true, $"Tag override saved: {entry.Wire} = {value.ToString(CultureInfo.CurrentCulture)}. The service applies it within its reload interval to every server under the tag that has no override of its own."); + } + /// Removes the chosen override so the global value applies again. public async Task RemoveAsync(ThresholdOverrideViewRow? row, CancellationToken cancellationToken = default) { @@ -122,13 +167,16 @@ public async Task RemoveAsync(ThresholdOverrideViewRow return new ThresholdOverrideOutcome(false, "Select an override to remove."); try { - await _gateway.RemoveThresholdOverrideAsync(row.ServerId, row.Database, row.Key, cancellationToken); + if (row.TagId is { } tagId) + await _gateway.RemoveThresholdOverrideForTagAsync(tagId, row.Key, cancellationToken); + else + await _gateway.RemoveThresholdOverrideAsync(row.ServerId, row.Database, row.Key, cancellationToken); } catch (ViewerReadOnlyException ex) { return new ThresholdOverrideOutcome(false, ex.Message); } - return new ThresholdOverrideOutcome(true, $"Override removed: {row.Key} on {row.ServerName} uses the global value again."); + return new ThresholdOverrideOutcome(true, $"Override removed: {row.Key} on {row.ScopeText} uses the global value again."); } } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml index d2e7718283..7984e4da4f 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml @@ -30,7 +30,7 @@ @@ -40,7 +40,7 @@ IsReadOnly="True" SelectionChanged="OverridesGrid_SelectionChanged"> - + @@ -51,7 +51,7 @@ - - + + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs index c45baa5ac9..28ea7dcb56 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs @@ -59,9 +59,13 @@ private async Task LoadAsync() try { var servers = await _dataService.GetMonitoredServersAsync(); + var tags = await _dataService.GetServerTagsAsync(); ServerCombo.ItemsSource = servers .OrderBy(s => s.Name, StringComparer.OrdinalIgnoreCase) .Select(s => new ThresholdOverrideServerChoice(s.ServerId, s.Name)) + .Concat(tags + .OrderBy(t => t.Name, StringComparer.OrdinalIgnoreCase) + .Select(t => new ThresholdOverrideServerChoice(0, "Tag: " + t.Name, t.Id))) .ToList(); _rows.Clear(); foreach (var row in await _model.LoadAsync()) @@ -85,7 +89,8 @@ private void OverridesGrid_SelectionChanged(object sender, SelectionChangedEvent } /* Selecting a row loads it into the form, so Save changes it and Remove removes it. */ - ServerCombo.SelectedItem = ServerCombo.Items.OfType().FirstOrDefault(s => s.ServerId == row.ServerId); + ServerCombo.SelectedItem = ServerCombo.Items.OfType() + .FirstOrDefault(s => row.IsTag ? s.TagId == row.TagId : s.TagId is null && s.ServerId == row.ServerId); KeyCombo.SelectedItem = row.Key; DatabaseBox.Text = row.Database ?? ""; ValueBox.Text = row.Row.Value.ToString("0.##", System.Globalization.CultureInfo.CurrentCulture); @@ -98,7 +103,9 @@ private async void Save_Click(object sender, RoutedEventArgs e) try { var server = ServerCombo.SelectedItem as ThresholdOverrideServerChoice; - var outcome = await _model.SaveAsync(server?.ServerId, KeyCombo.SelectedItem as string, ValueBox.Text, DatabaseBox.Text); + var outcome = server?.TagId is { } tagId + ? await _model.SaveForTagAsync(tagId, KeyCombo.SelectedItem as string, ValueBox.Text, DatabaseBox.Text) + : await _model.SaveAsync(server?.ServerId, KeyCombo.SelectedItem as string, ValueBox.Text, DatabaseBox.Text); SetStatus(outcome.Message, !outcome.Ok); if (outcome.Ok) { diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs index a9a422a345..20ae73c0de 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs @@ -44,6 +44,37 @@ public async Task SaveThresholdOverrideAsync( } } + public async Task SaveThresholdOverrideForTagAsync( + int tagId, string key, int value, string setBy, CancellationToken cancellationToken = default) + { + try + { + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken); + await ThresholdOverrideStore.UpsertTagAsync(connection, transaction, tagId, key, value, setBy, null, cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + catch (PostgresException ex) when (ex.SqlState == InsufficientPrivilegeSqlState) + { + throw new ViewerReadOnlyException(ex); + } + } + + public async Task RemoveThresholdOverrideForTagAsync(int tagId, string key, CancellationToken cancellationToken = default) + { + try + { + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken); + await ThresholdOverrideStore.DeleteTagAsync(connection, transaction, tagId, key, cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + catch (PostgresException ex) when (ex.SqlState == InsufficientPrivilegeSqlState) + { + throw new ViewerReadOnlyException(ex); + } + } + public async Task RemoveThresholdOverrideAsync( int serverId, string? database, string key, CancellationToken cancellationToken = default) { diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs index 7b42e0ab90..3d9e58a532 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs @@ -1377,7 +1377,10 @@ per the V71 finding. */ EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds' AND NOT attisdropped), /* V226 (#6893) adds the AWS RDS Web or Express fact to the server registry. Found by catalog like the V225 column above. Named only in this probe line, never in prose, per the V71 finding. */ - EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped)"; + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped), + /* V227 (#6926) adds the tag scope to the threshold overrides. Found by catalog like the V225 column above. Named only in this probe line, never in + prose, per the V71 finding. */ + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.config_threshold_overrides') AND attname = 'scope_tag_id' AND NOT attisdropped)"; /// The store schema version this viewer build requires — the highest migration it knows /// (). The connect-time gate blocks a store below this. @@ -1399,7 +1402,7 @@ per the V71 finding. */ await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201)); + return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201), reader.GetBoolean(202)); } return null; @@ -1424,7 +1427,7 @@ per the V71 finding. */ /// is unit-tested without a live store; any schema bump past the newest arm trips the pinning test that keeps /// this in step with . /// - internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false) + internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false, bool hasThresholdOverrideTagScope = false) { /* V71 (the PostgreSQL blocking-edges rung): a table-existence sentinel and now the newest-first arm. A collector table would ordinarily get no arm at all — see the V63-V69 note below — but the TOP @@ -1633,6 +1636,11 @@ store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through t /* V226 (#6893): the AWS RDS Web or Express fact on the server registry. Column-existence sentinel, newest-first, and the TOP rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 225 and showing a spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + /* V227 (#6926): the tag scope on the threshold overrides. Catalog sentinel, the top rung, newest first. */ + if (hasThresholdOverrideTagScope) + { + return 227; + } if (hasServerAwsRdsWebOrExpress) { return 226; diff --git a/docs/darling/admin.md b/docs/darling/admin.md index df819a8722..1668dc998b 100644 --- a/docs/darling/admin.md +++ b/docs/darling/admin.md @@ -189,6 +189,7 @@ An older build ignores what those migrations added, so it can read some newer da - **V219**: an older build does not know the Query Store self-statement keys ([#6750](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6750)). Rolling back leaves `collect.query_store_self_keys`, `collect.query_store_self_keys_state` and the two triggers on `collect.query_store_text` in place. The older build ignores the tables, and the triggers keep the keys current whenever it writes statement text, so when you roll forward again the Query Store regressions read still hides the monitor's own statement. Nothing needs to be done before you roll back. - **V220**: an older build does not know the Job History statistics index ([#6798](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6798)). Rolling back leaves `idx_job_history_job_success` on `collect.job_history` in place. The older build's Job History read can still use it, and the collector keeps it current on every insert. Nothing needs doing before you roll back. Leave the index in place: the viewer recognizes a V220 store by it, so a store whose index was dropped answers 219 to the viewer while its schema version says 220, the viewer refuses it, and the migrator does not build the index again. - **V221**: an older build does not know the Query Store hash-hourly table ([#6800](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6800)). Rolling back leaves `collect.query_store_hash_hourly`, the day tables `collect.query_store_hash_daily`, `collect.query_store_hash_daily_built` and `collect.query_store_hash_daily_failures`, and the `hash_built_seq` column of `collect.query_store_compose_stamp_built` in place. The older build neither reads nor fills them, so the tables stop growing and their hours and days go stale. Nothing to do before rolling back. If you roll forward again, the next hourly passes rebuild the stale hours, newest first, and the next cleanup removes the hash-hourly rows of any hour the older build retired without deleting them (it deletes by the retention floor, not by the hours it just retired) +- **V227**: an older build does not know a threshold override held for a server tag ([#6926](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6926)). Rolling back leaves the `scope_tag_id` column on `config.config_threshold_overrides` (and `server_id` nullable), and any tag override rows stay. An older build reads the whole overrides table when it loads, and a row with no server stops that read, so it applies **no** threshold override at all (server or database ones included) until the tag rows are gone. Before rolling back past V227, remove every tag override (the Viewer's Threshold Overrides window, the web Admin tab, or `set_alert_threshold_override` with `clear` and a tag), or run `DELETE FROM config.config_threshold_overrides WHERE server_id IS NULL;` as the owner. ### Upgrading from a build that predates the script From 679302b2efc0ca14d9a75f110a3816ea8fbd0303 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:25:55 -0400 Subject: [PATCH 02/10] Settable blocking bands so the colours agree with the alerts (#6927) --- .../Darling.Tests/BlockingBandTiersTests.cs | 81 ++++++++++++++++++ Darling/Darling.Tests/RungRewindArtifacts.cs | 8 ++ .../DarlingAlertSettings.cs | 9 ++ .../DarlingConfig.cs | 21 +++++ .../FleetSweepEngine.cs | 25 ++++-- .../Mcp/DarlingAlertReader.cs | 14 +++- .../Mcp/DarlingFleetReader.cs | 47 +++++++++-- .../Mcp/DarlingHealthReader.cs | 8 +- .../Mcp/DarlingMcpAlertTools.cs | 82 +++++++++++++++++- .../StoreConfigProvider.cs | 18 +++- .../wwwroot/js/pages/admin.js | 3 +- .../wwwroot/js/pages/fleet.js | 6 +- .../BlockingBandStore.cs | 46 ++++++++++ .../PgMigrations.cs | 19 +++++ .../StorageVersion.cs | 2 +- .../MainWindow.xaml.cs | 13 ++- .../SettingsWindow.xaml | 20 +++++ .../SettingsWindow.xaml.cs | 60 ++++++++++++++ .../ViewerDataService.AlertSettings.cs | 29 ++++++- .../ViewerDataService.DailySummary.cs | 1 + .../ViewerDataService.Overview.cs | 21 ++++- .../ViewerDataService.cs | 16 +++- PerformanceMonitor.Common/DailyHealthBand.cs | 9 +- .../ServerHealthBands.cs | 83 +++++++++++++++++-- 24 files changed, 598 insertions(+), 43 deletions(-) create mode 100644 Darling/Darling.Tests/BlockingBandTiersTests.cs create mode 100644 Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs diff --git a/Darling/Darling.Tests/BlockingBandTiersTests.cs b/Darling/Darling.Tests/BlockingBandTiersTests.cs new file mode 100644 index 0000000000..7bd72a3314 --- /dev/null +++ b/Darling/Darling.Tests/BlockingBandTiersTests.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace PerformanceMonitor.Darling.Tests; + +/// #6927: the blocking band reads settable tiers whose defaults are the constants every earlier build used. +public sealed class BlockingBandTiersTests +{ + private static readonly TimeSpan Day = TimeSpan.FromHours(24); + + [Fact] + public void Defaults_EqualTheCompileTimeConstants() + { + var d = BlockingBandThresholds.Default; + Assert.Equal(ServerHealthThresholds.BlockingWarnPerHour, d.WarnPerHour); + Assert.Equal(ServerHealthThresholds.BlockingCriticalPerHour, d.CriticalPerHour); + Assert.Equal(ServerHealthThresholds.BlockingWarnWaitSeconds, d.WarnWaitSeconds); + Assert.Equal(ServerHealthThresholds.BlockingCriticalWaitSeconds, d.CriticalWaitSeconds); + Assert.Equal( + ServerHealthClassifier.BlockingSeverity(1434, 162.0, Day), + ServerHealthClassifier.BlockingSeverity(1434, 162.0, Day, d)); + } + + [Fact] + public void ADayTunedTo100EventsAnd180Seconds_IsNotCritical_ButIsUnderTheDefaults() + { + /* 2026-10-23 in the issue: 1,434 reports over a day is 59.75 an hour here (73.5 on the day's own window), peak block 2.7 minutes. */ + const long reports = 1764; // 73.5 an hour over 24 hours + const double peakSeconds = 162.0; // 2.7 minutes + + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(reports, peakSeconds, Day)); + + var tuned = new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(reports, peakSeconds, Day, tuned)); + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(2400, 100.0, Day, tuned)); // 100 an hour + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(1, 180.0, Day, tuned)); + } + + [Fact] + public void TheDailyClassifier_UsesTheTunedTiers() + { + var signals = new DailyHealthSignals + { + Window = Day, + BlockingReports = 1764, + BlockingEvents = 1764, + PeakBlockWaitMs = 162_000, + }; + + Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(signals)); + var tuned = new DailyHealthThresholds { BlockingBands = new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0) }; + Assert.NotEqual(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(signals, tuned)); + } + + [Fact] + public void RawValuesClampToTheFloorsAndCeilings_AndADefaultStructClampsToTheFloorNotZero() + { + var wild = new BlockingBandThresholds(-5, double.NaN, 0, 1e12); + Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, wild.WarnPerHour); + Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, wild.CriticalPerHour); + Assert.Equal(ServerHealthThresholds.BlockingWaitSecondsFloor, wild.WarnWaitSeconds); + Assert.Equal(ServerHealthThresholds.BlockingWaitSecondsCeiling, wild.CriticalWaitSeconds); + Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, default(BlockingBandThresholds).WarnPerHour); + } + + [Fact] + public void TheFleetTotalsTiles_AreBandedByTheClassifier_NotOnAnyNonZeroCount() + { + var now = new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc); + var empty = DarlingFleetReader.BuildRollup(Array.Empty(), now, now.AddHours(-1), now); + Assert.Equal(HealthSeverity.Healthy, empty.TotalDeadlockSeverity); + + var tuned = DarlingFleetReader.BuildRollup( + new List(), now, now.AddHours(-1), now, + deadlockTiers: new DeadlockRateThresholds(50.0, 100.0), blockingTiers: new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0)); + Assert.Equal(HealthSeverity.Healthy, tuned.TotalDeadlockSeverity); + } +} diff --git a/Darling/Darling.Tests/RungRewindArtifacts.cs b/Darling/Darling.Tests/RungRewindArtifacts.cs index 375b56efdd..eadcce8d4c 100644 --- a/Darling/Darling.Tests/RungRewindArtifacts.cs +++ b/Darling/Darling.Tests/RungRewindArtifacts.cs @@ -577,6 +577,14 @@ ALTER TABLE config.config_alert_settings // so a re-run of V226 does everything again. Safe on a store that never had it. new(226, "hasServerAwsRdsWebOrExpress", @"ALTER TABLE collect.servers DROP COLUMN IF EXISTS is_aws_rds_web_or_express;"), + // V228 (#6927): the four blocking band columns on the alert settings row. The warn per hour column is the probe's newest sentinel. The + // rewind drops all four, so a re-run of V228 adds them again. Safe on a store that never had them. + new(228, "hasBlockingBands", + @"ALTER TABLE config.config_alert_settings + DROP COLUMN IF EXISTS blocking_warn_per_hour, + DROP COLUMN IF EXISTS blocking_critical_per_hour, + DROP COLUMN IF EXISTS blocking_warn_wait_seconds, + DROP COLUMN IF EXISTS blocking_critical_wait_seconds;"), ]; /// diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs index add0198c81..048c648cc8 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs @@ -296,6 +296,15 @@ public IReadOnlyList WebhookChannelProblems _config.Alerts.DeadlockWarnPerHour, _config.Alerts.DeadlockCriticalPerHour); + /// #6927 (V228): the blocking health band's four tiers as ONE value, for the reason + /// is one. The clamp lives on + /// , so the viewer's card and the service's read clamp alike. + public BlockingBandThresholds BlockingBandThresholds => new( + _config.Alerts.BlockingWarnPerHour, + _config.Alerts.BlockingCriticalPerHour, + _config.Alerts.BlockingWarnWaitSeconds, + _config.Alerts.BlockingCriticalWaitSeconds); + public int CollectionFailureThreshold => Math.Clamp(_config.Alerts.CollectionFailureThreshold, 1, 1000); /* #1984: percent clamped like low-disk's (0 = off); the GB floor merely floored at 0 — unlike diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs index 02690d7096..1d765cde4f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs @@ -1105,6 +1105,27 @@ public sealed class AlertsConfig [JsonPropertyName("deadlockCriticalPerHour")] public double DeadlockCriticalPerHour { get; set; } = ServerHealthThresholds.DeadlockCriticalPerHourDefault; + /// #6927 (V228): the blocking health band's WARNING tier, blocked-process reports per hour over the window a card + /// counted them in. Seeded from , named rather than restated, so a store + /// that never sets it bands as every earlier build did. BlockingBandThresholds clamps it on read. + [JsonPropertyName("blockingWarnPerHour")] + public double BlockingWarnPerHour { get; set; } = ServerHealthThresholds.BlockingWarnPerHour; + + /// #6927 (V228): the blocking health band's CRITICAL tier, reports per hour, seeded from + /// . + [JsonPropertyName("blockingCriticalPerHour")] + public double BlockingCriticalPerHour { get; set; } = ServerHealthThresholds.BlockingCriticalPerHour; + + /// #6927 (V228): the blocking band's WARNING bar on the longest single block, seconds, seeded from + /// . + [JsonPropertyName("blockingWarnWaitSeconds")] + public double BlockingWarnWaitSeconds { get; set; } = ServerHealthThresholds.BlockingWarnWaitSeconds; + + /// #6927 (V228): the blocking band's CRITICAL bar on the longest single block, seconds, seeded from + /// . + [JsonPropertyName("blockingCriticalWaitSeconds")] + public double BlockingCriticalWaitSeconds { get; set; } = ServerHealthThresholds.BlockingCriticalWaitSeconds; + [JsonPropertyName("longRunningJobEnabled")] public bool LongRunningJobEnabled { get; set; } = true; diff --git a/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs b/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs index 54d5b52a91..3e219964e0 100644 --- a/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs +++ b/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs @@ -197,7 +197,8 @@ public static FleetSweepComposition Compose( IReadOnlyList previousVerdicts, IReadOnlyList activeWatchItems, FleetSweepInstrumentCounters instruments, - DeadlockRateThresholds deadlockRateTiers) + DeadlockRateThresholds deadlockRateTiers, + BlockingBandThresholds? blockingBandTiers = null) { ArgumentNullException.ThrowIfNull(readings); ArgumentNullException.ThrowIfNull(previousVerdicts); @@ -209,7 +210,12 @@ public static FleetSweepComposition Compose( a caller that kept the old call would compile and silently band on the shipped pair while the Overview card used the store's. Built once, because the banding thresholds must be one configuration for the whole sweep. */ - var banding = new DailyHealthThresholds { DeadlockRates = deadlockRateTiers }; + /* #6927: the blocking band's tiers travel the same way; null is today's constants. */ + var banding = new DailyHealthThresholds + { + DeadlockRates = deadlockRateTiers, + BlockingBands = blockingBandTiers ?? BlockingBandThresholds.Default, + }; var sweepId = nowUtc.Ticks; var inSettleWindow = nowUtc - instruments.ServiceStartedUtc < PostRestartSettleWindow; @@ -785,26 +791,26 @@ The production read fills both from one MAX. */ unit its rate tiers were measured in; BlockingEvents counts WAITS and is what the evidence shows as the count. Classify reads the same pair, so the ledger still cannot page on a count the verdict did not band. */ var bandedBlocking = signals.BlockingReports ?? signals.BlockingEvents; - if (ServerHealthClassifier.BlockingSeverity(bandedBlocking, peakBlockSeconds, signals.Window) + if (ServerHealthClassifier.BlockingSeverity(bandedBlocking, peakBlockSeconds, signals.Window, thresholds.BlockingBands) == HealthSeverity.Critical) { - yield return (FamilyBlocking, BlockingEvidence(signals.BlockingEvents, bandedBlocking, signals.BlockingReports.HasValue, signals.Window, peakBlockSeconds)); + yield return (FamilyBlocking, BlockingEvidence(signals.BlockingEvents, bandedBlocking, signals.BlockingReports.HasValue, signals.Window, peakBlockSeconds, thresholds.BlockingBands)); } } /// The blocking family's evidence (#3539 A3): the arm that banded is the one named. A 60 s /// block is the wait arm's Critical whatever the rate; otherwise the RATE is the value, with the raw /// count as its own member for the operator reconciling against the blocking grid. - private static string BlockingEvidence(long count, long bandedCount, bool bandedOnReports, TimeSpan window, double peakBlockSeconds) + private static string BlockingEvidence(long count, long bandedCount, bool bandedOnReports, TimeSpan window, double peakBlockSeconds, BlockingBandThresholds tiers) { - if (peakBlockSeconds >= ServerHealthThresholds.BlockingCriticalWaitSeconds) + if (peakBlockSeconds >= tiers.CriticalWaitSeconds) { return JsonSerializer.Serialize(new { derivation = "summary-scoring critical trigger under alerts_enabled: false — not an alert-engine replay", trigger = "longest single block in the sweep span, seconds", value = peakBlockSeconds, - threshold = ServerHealthThresholds.BlockingCriticalWaitSeconds, + threshold = tiers.CriticalWaitSeconds, blocking_count = count, }); } @@ -819,7 +825,7 @@ rate is present by construction. */ ? "blocked-process reports per hour over the sweep span" : "blocking events per hour over the sweep span", value = ratePerHour, - threshold = ServerHealthThresholds.BlockingCriticalPerHour, + threshold = tiers.CriticalPerHour, blocking_count = count, }); } @@ -924,6 +930,7 @@ public static async Task RunAsync( — an engine-seam read, so a fault here loudly costs this sweep slot rather than quietly banding the fleet on the shipped pair. */ var deadlockRateTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken).ConfigureAwait(false); + var blockingBandTiers = await BlockingBandStore.ReadAsync(postgres, McpCommandDeadlines.ReadSeconds, cancellationToken).ConfigureAwait(false); var nowUtc = DateTime.UtcNow; var spanStartUtc = ComputeSpanStart(nowUtc, interval, previousRun); @@ -942,7 +949,7 @@ at a time and a herd all at once — and the sweep is a background errand racing var composition = Compose( nowUtc, spanStartUtc, alertsEnabled, servers.Count, readings, - previousRun, previousVerdicts, activeItems, ReadInstrumentCounters(), deadlockRateTiers); + previousRun, previousVerdicts, activeItems, ReadInstrumentCounters(), deadlockRateTiers, blockingBandTiers); await FleetSweepStore.RecordSweepAsync( postgres, composition.Run, composition.Verdicts, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs index 846a2a2dcc..b33da5ef24 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs @@ -542,7 +542,12 @@ homes decided (DarlingAlertSettings.ResolveUncorroboratedRoute takes this beside /* #6875 (V224): the Identity Capacity switch (boolean NOT NULL, default off) and its warning and critical bars (integer NOT NULL, 70 and 90). APPENDED, same reason. */ bool IdentityCapacityEnabled, int IdentityCapacityWarningPercent, - int IdentityCapacityCriticalPercent); + int IdentityCapacityCriticalPercent, + /* #6927 (V228): the four blocking band tiers (double precision NOT NULL, 5, 20 reports per hour and 10, 60 seconds). */ + double BlockingWarnPerHour, + double BlockingCriticalPerHour, + double BlockingWarnWaitSeconds, + double BlockingCriticalWaitSeconds); /// The single global alert-settings row (id=1) — the viewer's AlertSettingsSelectSql. The /// columns are read in the SAME order the service reads them (StoreConfigProvider), and @@ -582,7 +587,8 @@ homes decided (DarlingAlertSettings.ResolveUncorroboratedRoute takes this beside excluded_volumes, client_timeouts_warning_multiple, client_timeouts_critical_multiple, agent_alert_gaps_enabled, backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, - identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent + identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, + blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1"; @@ -653,7 +659,9 @@ FROM config_alert_settings /* #6874 (V222): the Agent Alert Gaps switch at 79. #6873 (V223): the three database protection switches at 80-82. */ reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), /* #6875 (V224): the Identity Capacity switch and bars at 83-85. */ - reader.GetBoolean(83), reader.GetInt32(84), reader.GetInt32(85)); + reader.GetBoolean(83), reader.GetInt32(84), reader.GetInt32(85), + /* #6927 (V228): the four blocking band tiers at 86-89. */ + reader.GetDouble(86), reader.GetDouble(87), reader.GetDouble(88), reader.GetDouble(89)); } /* ─────────────────────── delivery cooldown (a SECOND config table) ─────────────────────── */ diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index ea394f5ddb..5fcb31de44 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -930,6 +930,8 @@ the production photo this answers. */ servers on the old pair and the rest on the new one — a mixed reading no configuration ever held, and the band counts and the worst-first ranking would be derived from it. */ var deadlockTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken); + /* #6927 (V228): the blocking band's tiers, read once per roll-up for the same reason. */ + var blockingTiers = await BlockingBandStore.ReadAsync(postgres, McpCommandDeadlines.ReadSeconds, cancellationToken); var identityCapacity = await ReadIdentityCapacityAsync(postgres, now, cancellationToken); var cards = new List(servers.Count); @@ -967,11 +969,11 @@ server the read reported with an empty window reads Offline or never-collected b cards.Add(BuildCard( server, c, pg, m, mp, t, b, deadlock, pgDeadlock, collected.LastCollection, collectors, serverTags, now, - windowEndUtc - windowStartUtc, deadlockTiers, collected.RegisteredAt, collected.OlderCollection, serverAgentAlertGaps, serverDatabaseProtection, serverIdentityCapacity)); + windowEndUtc - windowStartUtc, deadlockTiers, collected.RegisteredAt, collected.OlderCollection, serverAgentAlertGaps, serverDatabaseProtection, serverIdentityCapacity, blockingTiers)); } return BuildRollup(cards, now, windowStartUtc, windowEndUtc, worstCount, tagForest, collectionHealthAgeSeconds, - await ReadCollectionStateAsync(postgres, logger, cancellationToken)); + await ReadCollectionStateAsync(postgres, logger, cancellationToken), deadlockTiers, blockingTiers); } /// Builds one pre-banded card from a server's raw cross-server reads (pure — the reduction the WPF @@ -1093,7 +1095,8 @@ internal static FleetServerCard BuildCard( DateTime? olderCollection = null, IReadOnlyList? agentAlertGaps = null, IReadOnlyList? databaseProtectionOverdue = null, - FleetIdentityCapacity? identityCapacity = null) + FleetIdentityCapacity? identityCapacity = null, + BlockingBandThresholds? blockingTiers = null) { /* Lite's XE-preferred / DMV-fallback, per server: XE when it has any row this window, else the DMV @@ -1188,6 +1191,8 @@ because cpuForAlert there is percent of an allocation that moves. The source is arm, one metric over. */ DeadlockWindow = deadlockWindow, DeadlockRateThresholds = deadlockTiers, + /* #6927: the blocking tiers travel with the count and the window, for the reason the deadlock ones do. */ + BlockingBandThresholds = blockingTiers, TotalThreads = threads.TotalThreads, AvailableThreads = availableThreads, ThreadsWaitingForCpu = threads.RunnableTasks, @@ -1268,7 +1273,7 @@ deliberately not surfaced. */ ? ServerHealthClassifier.BlockingRatePerHour(blockingForBand.Value, deadlockWindow) : null, BlockingWindow = deadlockWindow, - BlockingSeverity = ServerHealthClassifier.BlockingSeverity(blockingForBand, maxBlockedSeconds, deadlockWindow), + BlockingSeverity = ServerHealthClassifier.BlockingSeverity(blockingForBand, maxBlockedSeconds, deadlockWindow, blockingTiers), DeadlockCount = deadlockCount, DeadlockLastSeen = deadlockLastSeen, DeadlockMeasured = deadlockMeasured, @@ -1281,6 +1286,7 @@ over. Null exactly when the PostgreSQL arm took no difference (#3539). */ : null, DeadlockWindow = deadlockWindow, DeadlockRateThresholds = deadlockTiers, + BlockingBandThresholds = blockingTiers, /* #6875: set only when a column is at the warning bar or past it. The Band below is not read from it. */ IdentityCapacity = identityCapacity, DeadlockSeverity = ServerHealthClassifier.DeadlockSeverity( @@ -1323,7 +1329,9 @@ public static FleetOverviewResult BuildRollup( int worstCount = DefaultWorstCount, IReadOnlyList? tags = null, int collectionHealthAgeSeconds = 0, - FleetCollectionState? collection = null) + FleetCollectionState? collection = null, + DeadlockRateThresholds? deadlockTiers = null, + BlockingBandThresholds? blockingTiers = null) { var healthy = 0; var warning = 0; @@ -1332,6 +1340,8 @@ public static FleetOverviewResult BuildRollup( var failures = 0; long totalBlocking = 0; long totalDeadlocks = 0; + long? totalBlockingReports = null; + var maxBlockedSeconds = 0.0; var deadlockSourcesRead = 0; var deadlockPostgresTargets = 0; var deadlockCollectorsSilent = 0; @@ -1375,6 +1385,14 @@ attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on totalBlocking += card.BlockingCount; totalDeadlocks += card.DeadlockCount; + /* #6927: the blocking tile bands on the reports the cards band on, and the longest block anywhere. A PostgreSQL card + has no blocking source, so it adds nothing and does not make the tile measured. */ + var cardMetrics = card.ToHealthMetrics(); + if (cardMetrics.BlockingCount.HasValue) + { + totalBlockingReports = (totalBlockingReports ?? 0) + cardMetrics.BlockingCount.Value; + maxBlockedSeconds = Math.Max(maxBlockedSeconds, cardMetrics.MaxBlockedSeconds); + } } var problems = cards @@ -1410,6 +1428,12 @@ attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on ServersWithCollectionFailures = failures, TotalBlockingEvents = totalBlocking, TotalDeadlocks = totalDeadlocks, + /* #6927: the two totals tiles band through the same classifier and the store's tiers as the cards, over the window the totals + were read in, instead of the page colouring them on any non-zero count. */ + TotalBlockingSeverity = ServerHealthClassifier.BlockingSeverity( + totalBlockingReports, maxBlockedSeconds, windowEndUtc - windowStartUtc, blockingTiers), + TotalDeadlockSeverity = ServerHealthClassifier.DeadlockSeverity( + totalDeadlocks, windowEndUtc - windowStartUtc, deadlockTiers ?? DeadlockRateThresholds.Default), DeadlockCoverage = new FleetDeadlockCoverage { ServersRead = deadlockSourcesRead, @@ -2950,6 +2974,11 @@ public sealed class FleetServerCard [JsonIgnore] public DeadlockRateThresholds DeadlockRateThresholds { get; init; } + /// The blocking tiers this card banded on (#6927), or null for today's constants. Carried, not serialized, for + /// , so the worst-first score re-bands on the card's own tiers. + [JsonIgnore] + public BlockingBandThresholds? BlockingBandThresholds { get; init; } + /// This server's deadlock-source collector band over the trailing seven days of collection /// health (#3017) — the fact that explains a of zero. The collector is /// deadlocks on SQL Server and pg_database_stats on PostgreSQL (#3539). Null when that @@ -3080,6 +3109,7 @@ was taken - see DeadlockMeasured. */ dot used the store's. */ DeadlockWindow = DeadlockWindow, DeadlockRateThresholds = DeadlockRateThresholds, + BlockingBandThresholds = BlockingBandThresholds, TotalThreads = TotalThreads, AvailableThreads = AvailableThreads, ThreadsWaitingForCpu = ThreadsWaitingForCpu, @@ -3145,6 +3175,13 @@ public sealed class FleetOverviewResult [JsonPropertyName("total_blocking_events")] public long TotalBlockingEvents { get; init; } [JsonPropertyName("total_deadlocks")] public long TotalDeadlocks { get; init; } + /// The Blocking totals tile's band (#6927): the fleet's blocked-process reports per hour and longest block against the + /// store's blocking tiers, through the same classifier the cards use. Unknown when no card has a blocking source. + [JsonPropertyName("total_blocking_severity")] public HealthSeverity TotalBlockingSeverity { get; init; } + + /// The Deadlocks totals tile's band (#6927): the fleet's deadlocks per hour against the store's deadlock tiers. + [JsonPropertyName("total_deadlock_severity")] public HealthSeverity TotalDeadlockSeverity { get; init; } + /// How much of the fleet actually read a deadlock source for, with /// the causes named (#3017). Never null — a total with no denominator beside it is the defect. [JsonPropertyName("deadlock_coverage")] public FleetDeadlockCoverage DeadlockCoverage { get; init; } = new(); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs index ef80e8d186..8026178ac3 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs @@ -275,6 +275,10 @@ public sealed record DailySummaryReadRow( /// defaults holds anyway. public DeadlockRateThresholds RateTiers { get; init; } = DeadlockRateThresholds.Default; + /// The blocking band's tiers the day is banded on (#6927), the store's values for the same reason + /// is. + public BlockingBandThresholds BlockingTiers { get; init; } = BlockingBandThresholds.Default; + /// The clock the still-forming day's window clamps against (#3525 review): anchored /// reads hand their resolved window end so a backdated as_of clamps against its own "now"; /// unanchored reads (the explicit-date tool, the viewer path) band against the wall clock. @@ -337,7 +341,7 @@ have not happened yet (review finding on #3525). The fleet sweep does NOT read t }; public DailyHealthBand HealthBand => - DailyHealthBandCalculator.Classify(ToSignals(), new DailyHealthThresholds { DeadlockRates = RateTiers }); + DailyHealthBandCalculator.Classify(ToSignals(), new DailyHealthThresholds { DeadlockRates = RateTiers, BlockingBands = BlockingTiers }); /// Human label for the band ("Healthy" / "Warning" / "Critical" / "No Data"). public string OverallHealth => DailyHealthBandCalculator.Label(HealthBand); @@ -496,6 +500,7 @@ a floor moves back only on a backfill (stale, it under-claims, so a window stays DarlingFleetReader's own hoist argument: a settings write mid-read must not band some days on the old pair and the rest on the new one. */ var rateTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken); + var blockingTiers = await BlockingBandStore.ReadAsync(postgres, McpCommandDeadlines.ReadSeconds, cancellationToken); /* #3541 A9: the retention horizon, from the store's effective retention and the READER's wall clock. The clock is deliberately NOT the caller's anchor — a purge is a wall-clock event and a backdated @@ -592,6 +597,7 @@ twice McpCommandDeadlines.ReadSeconds. */ var results = rawResults.Select(row => row with { RateTiers = rateTiers, + BlockingTiers = blockingTiers, ReferenceUtc = referenceUtc ?? DateTime.UtcNow, DataState = DailySummaryRetention.StateFor(row.SummaryDate, row.CollectionRuns, row.SignalSourcesPresent, horizon), RetentionHorizon = horizon, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs index b2686932c9..ff5a1824ea 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs @@ -477,7 +477,12 @@ invite tuning one and expecting the other to move. */ health_bands = new { deadlock_warn_per_hour = s.DeadlockWarnPerHour, - deadlock_critical_per_hour = s.DeadlockCriticalPerHour + deadlock_critical_per_hour = s.DeadlockCriticalPerHour, + /* #6927 (V228): the blocking band, beside the deadlock one: reports per hour and the longest single block in seconds. */ + blocking_warn_per_hour = s.BlockingWarnPerHour, + blocking_critical_per_hour = s.BlockingCriticalPerHour, + blocking_warn_wait_seconds = s.BlockingWarnWaitSeconds, + blocking_critical_wait_seconds = s.BlockingCriticalWaitSeconds }, poison_wait = new { @@ -1205,6 +1210,11 @@ internal static JsonObject BuildMuteRuleWithState(MuteRule r, DateTime nowUtc) "per hour is the tightest setting that is still a rate, so no value here can restore the 'any deadlock " + "is Critical' reading these tiers replaced. Setting critical BELOW warn is accepted and means every " + "banded rate is Critical. " + + "health_bands.blocking_warn_per_hour and health_bands.blocking_critical_per_hour are the blocking band's tiers in blocked-process reports per HOUR " + + "(accepting 1 to 10000), and health_bands.blocking_warn_wait_seconds and health_bands.blocking_critical_wait_seconds are its bars on the longest " + + "single block in seconds (accepting 1 to 86400). They default to 5 and 20 reports per hour and 10 and 60 seconds, and they decide the colour " + + "of the fleet cards, the Overview tab, the Daily Health Calendar and get_fleet_overview, so set them to match the blocking alert's own " + + "threshold. Unlike the deadlock tiers, each blocking pair must keep critical at or above warn, and a write that would leave it below is refused. " + "file_growth.rise_mb is megabytes per HOUR averaged over file_growth.lookback_minutes (a rate — the same 10240 is 10 GB/hr on any lookback; the engine scales it to the window), so shortening the lookback does not tighten the rise gate and lengthening it does not loosen it; only the rate does. " + "Two keys govern the PostgreSQL versions of the two count alerts and are NOT the same numbers as their SQL Server neighbours: deadlocks.pg_count_threshold and blocking.pg_count_threshold, both accepting 1 upward. They sit inside those groups rather than a section of their own so both engines' figures are visible together, but tuning deadlocks.count_threshold does NOT move the PostgreSQL gate and tuning deadlocks.pg_count_threshold does NOT move the SQL Server one. The enabled switch in each group DOES govern both engines. They are separate because the two engines count with different instruments - SQL Server's figure is captured deadlock graphs, the PostgreSQL one is deadlocks parsed from the server log (get_pg_deadlocks) - and an operator tuning one should not silently move the other. Both engines' fleet cards now band deadlocks through the SAME health_bands.deadlock_warn_per_hour tiers (the PostgreSQL card differences the server's own pg_stat_database.deadlocks counter over the window), so the move - raising a fire gate to meet the band's Warning bar so a page and an amber dot describe the same server - is available on either knob. On the blocking side the SQL Server count is engine-recorded blocked-process reports and the PostgreSQL one is distinct root blockers in a periodic SAMPLE of pg_stat_activity. Both PostgreSQL keys are ignored on a store with no PostgreSQL targets. " + "The fleet_sweep group is NOT an alert family and the alerts_enabled master switch does not govern it: " + @@ -1384,6 +1394,29 @@ Widening that allowlist to admit a transaction would have let a real target comm } } + /* #6927: the same lone-bar rule for the two blocking band pairs. */ + if (updates.Any(u => u.Column is BlockingWarnPerHourColumn or BlockingCriticalPerHourColumn or BlockingWarnWaitColumn or BlockingCriticalWaitColumn)) + { + const string blockingBandSql = "SELECT blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1"; + await using var blockingBandCommand = new NpgsqlCommand(blockingBandSql, connection) { Transaction = transaction }; + blockingBandCommand.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var blockingBandReader = await blockingBandCommand.ExecuteReaderAsync(); + if (await blockingBandReader.ReadAsync()) + { + if (blockingBandReader.GetDouble(1) < blockingBandReader.GetDouble(0)) + { + return Outcome("invalid", BlockingRateBarsOrderMessage + + $" (stored after this write: warn {blockingBandReader.GetDouble(0).ToString("0.###", CultureInfo.InvariantCulture)}, critical {blockingBandReader.GetDouble(1).ToString("0.###", CultureInfo.InvariantCulture)}; nothing was written.)"); + } + + if (blockingBandReader.GetDouble(3) < blockingBandReader.GetDouble(2)) + { + return Outcome("invalid", BlockingWaitBarsOrderMessage + + $" (stored after this write: warn {blockingBandReader.GetDouble(2).ToString("0.###", CultureInfo.InvariantCulture)}, critical {blockingBandReader.GetDouble(3).ToString("0.###", CultureInfo.InvariantCulture)}; nothing was written.)"); + } + } + } + /* #5640: the same lone-bar rule for the Client Timeouts pair. */ if (updates.Any(u => u.Column == ClientTimeoutsWarningColumn || u.Column == ClientTimeoutsCriticalColumn)) { @@ -2586,6 +2619,14 @@ private static string AlreadyExists(MuteRule existing) => "'client_timeouts.critical_multiple' must not be lower than 'client_timeouts.warning_multiple' (the Critical bar can never sit below the Warning bar)."; private const string ClientTimeoutsBarsOrderMessage = "'client_timeouts.critical_per_minute' must not be lower than 'client_timeouts.warning_per_minute' (the Critical bar can never sit below the Warning bar)."; + private const string BlockingWarnPerHourColumn = "blocking_warn_per_hour"; + private const string BlockingCriticalPerHourColumn = "blocking_critical_per_hour"; + private const string BlockingWarnWaitColumn = "blocking_warn_wait_seconds"; + private const string BlockingCriticalWaitColumn = "blocking_critical_wait_seconds"; + private const string BlockingRateBarsOrderMessage = + "'health_bands.blocking_critical_per_hour' must not be lower than 'health_bands.blocking_warn_per_hour' (the Critical bar can never sit below the Warning bar)."; + private const string BlockingWaitBarsOrderMessage = + "'health_bands.blocking_critical_wait_seconds' must not be lower than 'health_bands.blocking_warn_wait_seconds' (the Critical bar can never sit below the Warning bar)."; private const string RunnableQueueBarsOrderMessage = "'runnable_queue.critical_per_scheduler' must not be lower than 'runnable_queue.warning_per_scheduler' (the Critical bar can never sit below the Warning bar)."; @@ -2963,6 +3004,27 @@ threshold that cannot reach a bare-count band. */ ServerHealthThresholds.DeadlockRatePerHourFloor, ServerHealthThresholds.DeadlockRatePerHourCeiling); break; + /* #6927 (V228): the blocking band, same named bounds BlockingBandThresholds clamps to. */ + case "blocking_warn_per_hour": + AddDouble("blocking_warn_per_hour", n, "health_bands.blocking_warn_per_hour", + ServerHealthThresholds.BlockingRatePerHourFloor, + ServerHealthThresholds.BlockingRatePerHourCeiling); + break; + case "blocking_critical_per_hour": + AddDouble("blocking_critical_per_hour", n, "health_bands.blocking_critical_per_hour", + ServerHealthThresholds.BlockingRatePerHourFloor, + ServerHealthThresholds.BlockingRatePerHourCeiling); + break; + case "blocking_warn_wait_seconds": + AddDouble("blocking_warn_wait_seconds", n, "health_bands.blocking_warn_wait_seconds", + ServerHealthThresholds.BlockingWaitSecondsFloor, + ServerHealthThresholds.BlockingWaitSecondsCeiling); + break; + case "blocking_critical_wait_seconds": + AddDouble("blocking_critical_wait_seconds", n, "health_bands.blocking_critical_wait_seconds", + ServerHealthThresholds.BlockingWaitSecondsFloor, + ServerHealthThresholds.BlockingWaitSecondsCeiling); + break; default: error = $"Unknown field 'health_bands.{k}'."; break; } }); @@ -3375,6 +3437,24 @@ is a Postgres error (multiple assignments to the same column) -- so refusing it error = RunnableQueueBarsOrderMessage; } + /* #6927: both bars of a blocking band pair in one body must already be in order; a lone bar is compared with the stored other one + inside UpdateAlertSettings' transaction (the builder has no store). */ + if (error == null + && updates.FirstOrDefault(u => u.Column == BlockingWarnPerHourColumn) is { Param: NpgsqlParameter blockWarnRate } + && updates.FirstOrDefault(u => u.Column == BlockingCriticalPerHourColumn) is { Param: NpgsqlParameter blockCriticalRate } + && blockCriticalRate.TypedValue < blockWarnRate.TypedValue) + { + error = BlockingRateBarsOrderMessage; + } + + if (error == null + && updates.FirstOrDefault(u => u.Column == BlockingWarnWaitColumn) is { Param: NpgsqlParameter blockWarnWait } + && updates.FirstOrDefault(u => u.Column == BlockingCriticalWaitColumn) is { Param: NpgsqlParameter blockCriticalWait } + && blockCriticalWait.TypedValue < blockWarnWait.TypedValue) + { + error = BlockingWaitBarsOrderMessage; + } + /* #5640: both Client Timeouts bars in one body must already be in order; a lone bar is compared with the stored other one. */ if (error == null && updates.FirstOrDefault(u => u.Column == ClientTimeoutsWarningColumn) is { Param: NpgsqlParameter timeoutWarningParam } diff --git a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs index 965aa93a3a..0c6883ad1f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs +++ b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs @@ -1431,11 +1431,12 @@ INSERT INTO config_alert_settings ( excluded_volumes, client_timeouts_warning_multiple, client_timeouts_critical_multiple, agent_alert_gaps_enabled, backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, - identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent) + identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, + blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds) VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46, $47, $48, $49, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, - $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86) + $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86, $87, $88, $89, $90) ON CONFLICT (id) DO NOTHING", connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; command.Parameters.AddWithValue(a.Enabled); command.Parameters.AddWithValue(a.CpuEnabled); @@ -1565,6 +1566,11 @@ apply and get_alert_settings reports back. */ command.Parameters.AddWithValue(a.IdentityCapacityEnabled); command.Parameters.AddWithValue(a.IdentityCapacityWarningPercent); command.Parameters.AddWithValue(a.IdentityCapacityCriticalPercent); + /* #6927, bound last in the order the V228 columns were appended. Seeded RAW like every sibling: the clamp lives on BlockingBandThresholds. */ + command.Parameters.AddWithValue(a.BlockingWarnPerHour); + command.Parameters.AddWithValue(a.BlockingCriticalPerHour); + command.Parameters.AddWithValue(a.BlockingWarnWaitSeconds); + command.Parameters.AddWithValue(a.BlockingCriticalWaitSeconds); await command.ExecuteNonQueryAsync(ct); } @@ -2204,7 +2210,8 @@ internal static string NormalizePlanXmlCompression(string? value) => excluded_volumes, client_timeouts_warning_multiple, client_timeouts_critical_multiple, agent_alert_gaps_enabled, backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, - identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent + identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, + blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1", connection) { CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds }; using var reader = await command.ExecuteReaderAsync(ct); if (!await reader.ReadAsync(ct)) @@ -2375,6 +2382,11 @@ selected but not read here -- or read but not selected -- would reset the list o IdentityCapacityEnabled = reader.GetBoolean(83), IdentityCapacityWarningPercent = reader.GetInt32(84), IdentityCapacityCriticalPercent = reader.GetInt32(85), + /* #6927 (V228): the four blocking band tiers at 86-89 (same reachability rule as every appended knob). */ + BlockingWarnPerHour = reader.GetDouble(86), + BlockingCriticalPerHour = reader.GetDouble(87), + BlockingWarnWaitSeconds = reader.GetDouble(88), + BlockingCriticalWaitSeconds = reader.GetDouble(89), }; var analysis = new AnalysisConfig { diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js index 0abc05db92..e2fe27cce3 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js @@ -196,7 +196,8 @@ export const GROUP_FIELDS = { cpu: ["enabled", "threshold_percent", "mode"], blocking: ["enabled", "count_threshold", "wait_threshold_seconds", "pg_count_threshold"], deadlocks: ["enabled", "count_threshold", "pg_count_threshold"], - health_bands: ["deadlock_warn_per_hour", "deadlock_critical_per_hour"], + health_bands: ["deadlock_warn_per_hour", "deadlock_critical_per_hour", "blocking_warn_per_hour", "blocking_critical_per_hour", + "blocking_warn_wait_seconds", "blocking_critical_wait_seconds"], poison_wait: ["enabled", "threshold_ms"], runnable_queue: ["enabled", "warning_per_scheduler", "critical_per_scheduler"], client_timeouts: ["enabled", "warning_per_minute", "critical_per_minute", "warning_multiple", "critical_multiple"], diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js index 1035e40e02..63fd351bcc 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js @@ -681,7 +681,9 @@ export function rollup(d) { not a confirmed incident either — warning is the "interrogate this number" class without claiming a deadlock happened. A real deadlock keeps critical whatever the coverage: a measured incident outranks an incomplete denominator, and the sub-line under it already says the count may be short. */ - const deadlockClass = d.total_deadlocks > 0 ? "critical" : deadlockSub && deadlockSub.partial ? "warning" : ""; + /* #6927: banded by the service through the same classifier and the store's tiers as the cards, not on any non-zero count. */ + const totalClass = (sev) => (sev === "Critical" ? "critical" : sev === "Warning" ? "warning" : ""); + const deadlockClass = totalClass(d.total_deadlock_severity) || (deadlockSub && deadlockSub.partial ? "warning" : ""); /* Two fixed groups (server-band counts | event counts) split by a divider; a non-zero blocking / deadlock total takes a severity color. */ @@ -695,7 +697,7 @@ export function rollup(d) { ]), el("div", { class: "rollup-divider" }), el("div", { class: "rollup-group" }, [ - tile(d.total_blocking_events, "Blocking (recent)", d.total_blocking_events > 0 ? "warning" : ""), + tile(d.total_blocking_events, "Blocking (recent)", totalClass(d.total_blocking_severity)), /* The API's own note rides along as the tile's title — the causes and the two windows, in the words the service already renders for get_fleet_overview, so the hover is DETAIL on a signal that is already visible rather than the only place the signal exists. */ diff --git a/Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs b/Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs new file mode 100644 index 0000000000..c854472ffc --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Storage; + +/// +/// The one read of the blocking health band's four tiers (#6927, V228) from the store's singleton settings row. It lives in Storage because +/// every Darling reader of the band needs the same values: the fleet cards and get_fleet_overview, the fleet sweep, the Daily Health +/// Calendar and get_daily_health in the service, and the Overview tab, the calendar and the fleet grid in the viewer. Values come back +/// RAW and clamps on read. A store with no row yet bands on today's constants, which is what such a +/// store would seed. +/// +public static class BlockingBandStore +{ + /// The singleton row's four tiers. No parameters: this is the current configuration, not history. + public const string ReadSql = @" +SELECT blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds +FROM config_alert_settings +WHERE id = 1"; + + /// Reads the tiers, or when the row is absent. + public static async Task ReadAsync( + NpgsqlDataSource postgres, int commandTimeoutSeconds, CancellationToken cancellationToken) + { + await using var command = postgres.CreateCommand(ReadSql); + command.CommandTimeout = commandTimeoutSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return new BlockingBandThresholds( + reader.GetDouble(0), reader.GetDouble(1), reader.GetDouble(2), reader.GetDouble(3)); + } + + return BlockingBandThresholds.Default; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index c65424e31c..ed1dd2faf9 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -2975,6 +2975,7 @@ holds the ordering. */ new Migration(224, "identity-capacity", V224Sql), new Migration(225, "server-aws-rds", V225Sql), new Migration(226, "server-aws-rds-web-or-express", V226Sql), + new Migration(228, "blocking-band-tiers", V228Sql), }; /// @@ -3002,6 +3003,24 @@ ALTER TABLE collect.servers ALTER TABLE collect.servers ADD COLUMN IF NOT EXISTS is_aws_rds_web_or_express boolean;"; + /// + /// V228 (#6927): the blocking health band's four tiers on the alert settings row, beside the V120 deadlock ones. They carry the + /// figures ServerHealthThresholds.BlockingWarnPerHour, BlockingCriticalPerHour, BlockingWarnWaitSeconds and + /// BlockingCriticalWaitSeconds have always been, so a store that never sets them bands exactly as before. They are literals here + /// because SQL text cannot name a C# constant. No CHECK ordering the two tiers of a pair, for the reason V120 gives: the writers refuse + /// a pair with critical below warn, and a hand-edited row reads as written. Idempotent: every column is added with IF NOT EXISTS, so a + /// store that already has them is left as it is. Table-level grants already cover new columns on this table, so no GRANT is added. + /// + private const string V228Sql = @" +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_warn_per_hour double precision NOT NULL DEFAULT 5.0; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_critical_per_hour double precision NOT NULL DEFAULT 20.0; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_warn_wait_seconds double precision NOT NULL DEFAULT 10.0; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_critical_wait_seconds double precision NOT NULL DEFAULT 60.0;"; + /// /// V189 (#5867): collect.query_store_compose_stamp_failures, the failure record of the Query Store compose builder: one row /// per hour whose last build threw, with its consecutive failures, last error and next attempt, so a build that cannot finish backs diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs index bfebc69b62..0db43132aa 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs @@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class StorageVersion { - public const int SchemaVersion = 226; + public const int SchemaVersion = 228; } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs index 98886ade29..46d0714e12 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs @@ -1669,6 +1669,17 @@ is exactly why DarlingFleetReader.GetFleetOverviewAsync hoists its own copy of t deadlockTiers = DeadlockRateThresholds.Default; } + /* #6927: the blocking band's tiers, hoisted like the deadlock ones above and for the same reasons. */ + BlockingBandThresholds blockingTiers; + try + { + blockingTiers = await service.GetBlockingBandThresholdsAsync(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + blockingTiers = BlockingBandThresholds.Default; + } + var perLane = await Task.WhenAll(lanes.Select(async lane => { var found = new List(lane.Count); @@ -1678,7 +1689,7 @@ is exactly why DarlingFleetReader.GetFleetOverviewAsync hoists its own copy of t try { var summary = await service.GetServerSummaryAsync( - server.ServerId, server.DisplayName, deadlockTiers); + server.ServerId, server.DisplayName, deadlockTiers, blockingTiers: blockingTiers); summary.ServerName = server.ServerName; /* #3029: the engine discriminator comes from the REGISTRY row, which already carries it (servers.engine_kind, via ManagedServersSql / ServersSql) — the per-server summary diff --git a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml index 0ca93eac7f..e05447a6b1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml @@ -445,6 +445,26 @@ + + + + + + + + + + + + + + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs index 025787b455..7b4fe4dd7d 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs @@ -923,6 +923,10 @@ and the alert text agree about what the number looks like. */ card agree about what the number looks like. */ BandDeadlockWarnPerHourBox.Text = r.DeadlockWarnPerHour.ToString("0.0", CultureInfo.InvariantCulture); BandDeadlockCriticalPerHourBox.Text = r.DeadlockCriticalPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnPerHourBox.Text = r.BlockingWarnPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalPerHourBox.Text = r.BlockingCriticalPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnWaitSecondsBox.Text = r.BlockingWarnWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalWaitSecondsBox.Text = r.BlockingCriticalWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); AlertPvsCheckBox.IsChecked = r.PvsEnabled; AlertPvsThresholdPercentBox.Text = r.PvsThresholdPercent.ToString(CultureInfo.InvariantCulture); AlertPvsFloorGbBox.Text = r.PvsFloorGb.ToString(CultureInfo.InvariantCulture); @@ -967,6 +971,23 @@ seeded from the row like every other alert-engine control. */ UpdateAlertControlStates(); } + /// Parses one blocking band box (#6927): InvariantCulture, within the named bounds the classifier clamps to and the MCP writer + /// accepts. A bad value adds an error naming the box and the range, and leaves null. + private static void BlockingBandBox( + System.Windows.Controls.TextBox box, double min, double max, string label, SettingsValidationErrors errors, out double? value) + { + if (double.TryParse(box.Text, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) && parsed >= min && parsed <= max) + { + value = parsed; + return; + } + + value = null; + errors.Add( + $"{label} must be a number between {min.ToString("0.0", CultureInfo.InvariantCulture)} and {max.ToString("0.0", CultureInfo.InvariantCulture)}.", + box); + } + /// Builds the store row from the alert + analysis controls, validating the numeric fields. Adds /// range errors to ; a bad field keeps its default rather than throwing. internal AlertSettingsRow BuildAlertRowFromControls(SettingsValidationErrors errors) @@ -1162,6 +1183,37 @@ not a pair to reject. */ && dlCritical >= ServerHealthThresholds.DeadlockRatePerHourFloor && dlCritical <= ServerHealthThresholds.DeadlockRatePerHourCeiling) row.DeadlockCriticalPerHour = dlCritical; + /* #6927: the blocking band, validated against the SAME named bounds BlockingBandThresholds clamps to and the MCP writer accepts, + InvariantCulture on the parse. Unlike the deadlock pair, a pair with critical below warn is refused, the rule update_alert_settings + applies, so the window cannot save a pair the tool would refuse. */ + BlockingBandBox(BandBlockingWarnPerHourBox, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling, + "Blocking warn per hour", errors, out var blWarnRate); + BlockingBandBox(BandBlockingCriticalPerHourBox, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling, + "Blocking critical per hour", errors, out var blCriticalRate); + BlockingBandBox(BandBlockingWarnWaitSecondsBox, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling, + "Blocking warn seconds", errors, out var blWarnWait); + BlockingBandBox(BandBlockingCriticalWaitSecondsBox, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling, + "Blocking critical seconds", errors, out var blCriticalWait); + if (blWarnRate.HasValue && blCriticalRate.HasValue) + { + if (blCriticalRate.Value < blWarnRate.Value) + errors.Add("Blocking critical per hour must not be lower than blocking warn per hour.", BandBlockingCriticalPerHourBox); + else + { + row.BlockingWarnPerHour = blWarnRate.Value; + row.BlockingCriticalPerHour = blCriticalRate.Value; + } + } + if (blWarnWait.HasValue && blCriticalWait.HasValue) + { + if (blCriticalWait.Value < blWarnWait.Value) + errors.Add("Blocking critical seconds must not be lower than blocking warn seconds.", BandBlockingCriticalWaitSecondsBox); + else + { + row.BlockingWarnWaitSeconds = blWarnWait.Value; + row.BlockingCriticalWaitSeconds = blCriticalWait.Value; + } + } if (int.TryParse(AlertPvsThresholdPercentBox.Text, out var pvsPct) && pvsPct is >= 0 and <= 100) row.PvsThresholdPercent = pvsPct; if (int.TryParse(AlertPvsFloorGbBox.Text, out var pvsFloor) && pvsFloor >= 0) @@ -1292,6 +1344,14 @@ the grid moved would arm the alert past the point it exists to precede. */ ServerHealthThresholds.DeadlockWarnPerHourDefault.ToString("0.0", CultureInfo.InvariantCulture); BandDeadlockCriticalPerHourBox.Text = ServerHealthThresholds.DeadlockCriticalPerHourDefault.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnPerHourBox.Text = + ServerHealthThresholds.BlockingWarnPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalPerHourBox.Text = + ServerHealthThresholds.BlockingCriticalPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnWaitSecondsBox.Text = + ServerHealthThresholds.BlockingWarnWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalWaitSecondsBox.Text = + ServerHealthThresholds.BlockingCriticalWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); AnalysisNotifyCooldownBox.Text = "360"; AlertPvsThresholdPercentBox.Text = "40"; AlertPvsFloorGbBox.Text = "1"; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs index f92c77fb31..f97ae27e73 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs @@ -99,7 +99,9 @@ the same reason and under the same connect-time store gate. */ /* #6873 (V223): the three database protection alert switches (backup, log backup, CHECKDB overdue), APPENDED, same gate. */ "backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, " + /* #6875 (V224): the Identity Capacity alert's switch (off by default) and its two bars, APPENDED for the same reason and under the same gate. */ - "identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent"; + "identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, " + + /* #6927 (V228): the blocking health band's four tiers, APPENDED for the same reason and under the same connect-time store gate. */ + "blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds"; /// The single global alert-settings row (id=1), for the Settings window prefill + the migrate-in /// defaults check. Column order matches . @@ -116,7 +118,7 @@ INSERT INTO config_alert_settings (id, " + AlertSettingsColumns + @", modified_a VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46, $47, $48, $49, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, - $63, $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86, + $63, $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86, $87, $88, $89, $90, (now() AT TIME ZONE 'UTC')) ON CONFLICT (id) DO UPDATE SET enabled = EXCLUDED.enabled, @@ -205,6 +207,10 @@ ON CONFLICT (id) DO UPDATE SET identity_capacity_enabled = EXCLUDED.identity_capacity_enabled, identity_capacity_warning_percent = EXCLUDED.identity_capacity_warning_percent, identity_capacity_critical_percent = EXCLUDED.identity_capacity_critical_percent, + blocking_warn_per_hour = EXCLUDED.blocking_warn_per_hour, + blocking_critical_per_hour = EXCLUDED.blocking_critical_per_hour, + blocking_warn_wait_seconds = EXCLUDED.blocking_warn_wait_seconds, + blocking_critical_wait_seconds = EXCLUDED.blocking_critical_wait_seconds, modified_at = (now() AT TIME ZONE 'UTC')"; /// The two cpu_mode values the service honors (it compares case-insensitively against @@ -344,6 +350,11 @@ parameter keeps its type on the INSERT arm. */ command.Parameters.Add(new NpgsqlParameter { TypedValue = r.IdentityCapacityEnabled }); // $84 (#6875, V224) command.Parameters.Add(new NpgsqlParameter { TypedValue = identityWarning }); // $85 (#6875, V224) command.Parameters.Add(new NpgsqlParameter { TypedValue = Math.Clamp(r.IdentityCapacityCriticalPercent, identityWarning, 100) }); // $86 (#6875, V224) + /* #6927 (V228): the blocking band tiers, stored raw like the deadlock pair above them. */ + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingWarnPerHour }); // $87 (#6927, V228) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingCriticalPerHour }); // $88 (#6927, V228) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingWarnWaitSeconds }); // $89 (#6927, V228) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingCriticalWaitSeconds }); // $90 (#6927, V228) } private static AlertSettingsRow ReadAlertSettingsRow(NpgsqlDataReader reader) => new() @@ -457,6 +468,11 @@ parameter keeps its type on the INSERT arm. */ IdentityCapacityEnabled = reader.GetBoolean(83), IdentityCapacityWarningPercent = reader.GetInt32(84), IdentityCapacityCriticalPercent = reader.GetInt32(85), + /* #6927 (V228) the four blocking band tiers appended at ordinals 86-89. */ + BlockingWarnPerHour = reader.GetDouble(86), + BlockingCriticalPerHour = reader.GetDouble(87), + BlockingWarnWaitSeconds = reader.GetDouble(88), + BlockingCriticalWaitSeconds = reader.GetDouble(89), }; /// Maps the Settings window's CPU-mode combo tag ("Total"/"SqlOnly") to the store value. @@ -614,6 +630,11 @@ is MB per HOUR averaged over the lookback (#3539 A8c) -- the Settings row's labe public int IdentityCapacityWarningPercent { get; set; } = 70; /// #6875 (V224): percent used at which an identity column is critical. public int IdentityCapacityCriticalPercent { get; set; } = 90; + /// #6927 (V228): the blocking health band's tiers, today's constants by default. + public double BlockingWarnPerHour { get; set; } = ServerHealthThresholds.BlockingWarnPerHour; + public double BlockingCriticalPerHour { get; set; } = ServerHealthThresholds.BlockingCriticalPerHour; + public double BlockingWarnWaitSeconds { get; set; } = ServerHealthThresholds.BlockingWarnWaitSeconds; + public double BlockingCriticalWaitSeconds { get; set; } = ServerHealthThresholds.BlockingCriticalWaitSeconds; public bool LongRunningQueryEnabled { get; set; } = true; public int LongRunningQueryThresholdMinutes { get; set; } = 30; public bool TempDbSpaceEnabled { get; set; } = true; @@ -749,6 +770,10 @@ public bool ValueEquals(AlertSettingsRow other) && IdentityCapacityEnabled == other.IdentityCapacityEnabled && IdentityCapacityWarningPercent == other.IdentityCapacityWarningPercent && IdentityCapacityCriticalPercent == other.IdentityCapacityCriticalPercent + && Math.Abs(BlockingWarnPerHour - other.BlockingWarnPerHour) < 0.0001 + && Math.Abs(BlockingCriticalPerHour - other.BlockingCriticalPerHour) < 0.0001 + && Math.Abs(BlockingWarnWaitSeconds - other.BlockingWarnWaitSeconds) < 0.0001 + && Math.Abs(BlockingCriticalWaitSeconds - other.BlockingCriticalWaitSeconds) < 0.0001 && LongRunningQueryEnabled == other.LongRunningQueryEnabled && LongRunningQueryThresholdMinutes == other.LongRunningQueryThresholdMinutes && TempDbSpaceEnabled == other.TempDbSpaceEnabled diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs index ccbebc3da0..e2381642ac 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs @@ -124,6 +124,7 @@ public async Task> GetDailySummaryRangeAsync( var banding = new DailyHealthThresholds { DeadlockRates = await GetDeadlockRateThresholdsAsync(cancellationToken), + BlockingBands = await GetBlockingBandThresholdsAsync(cancellationToken), }; /* #3653: the horizon, read once per range like the banding tiers above — a schedule save mid-read must diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs index 37453b9057..e5031f2bc6 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs @@ -290,7 +290,8 @@ public async Task GetServerSummaryAsync( int serverId, string displayName, DeadlockRateThresholds? deadlockTiers = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + BlockingBandThresholds? blockingTiers = null) { var nowUtc = DateTime.UtcNow; /* #3368: the window's LENGTH is named once and the start derived from it, so the card's deadlock @@ -586,6 +587,8 @@ know the engine (the loader stamps IsPostgres afterwards), and one arm is always DeadlockWindow = window, BlockingWindow = window, DeadlockRateThresholds = deadlockTiers, + /* #6927: the store's blocking tiers, or null for today's constants. */ + BlockingBandThresholds = blockingTiers, TotalThreads = totalThreads, CurrentWorkers = currentWorkers, ThreadsWaitingForCpu = threadsWaitingForCpu, @@ -622,6 +625,11 @@ FROM config_alert_settings /// Values come back RAW; clamps on read, so a hand-edited row /// cannot drive a nonsense threshold. /// + /// The store's blocking-band tiers (#6927), or today's constants when the singleton row is absent. The same read + /// the service uses (). + public Task GetBlockingBandThresholdsAsync(CancellationToken cancellationToken = default) => + BlockingBandStore.ReadAsync(_dataSource, ViewerCommandDeadlines.CurrentInteractiveReadSeconds, cancellationToken); + public async Task GetDeadlockRateThresholdsAsync(CancellationToken cancellationToken = default) { await using var command = _dataSource.CreateCommand(DeadlockRateThresholdSql); @@ -1328,10 +1336,10 @@ decision the service's fleet card uses so the two cannot disagree about whether : null; /// Blocking band (#3539 A3) — a 60 s block Critical whatever the rate; else blocking events per - /// hour over the window against the shared tiers (20/hr Critical, 5/hr Warning); a 10 s block Warning; - /// no source Unknown. + /// hour over the window against the store's tiers (#6927, 20/hr Critical, 5/hr Warning by default); a 10 s block + /// Warning by default; no source Unknown. public HealthSeverity BlockingSeverity => - ServerHealthClassifier.BlockingSeverity(BlockingCountForBand, MaxBlockedSeconds, BlockingWindow); + ServerHealthClassifier.BlockingSeverity(BlockingCountForBand, MaxBlockedSeconds, BlockingWindow, BlockingBandThresholds); /// /// The window and cover (#3368) — the @@ -1349,6 +1357,10 @@ decision the service's fleet card uses so the two cannot disagree about whether /// about which numbers are in force. public DeadlockRateThresholds? DeadlockRateThresholds { get; set; } + /// The store's blocking-band tiers (#6927), or null to band on today's constants, set by the read from + /// config_alert_settings so this card and the service's fleet card agree. + public BlockingBandThresholds? BlockingBandThresholds { get; set; } + /// Deadlocks per HOUR over — what the band evaluates (#3368), or /// null when the window is too short to normalise. Rendered beside the count so the dot's reason is /// legible. @@ -1424,6 +1436,7 @@ no difference (#3539). */ contradicting itself, which is the shape #3281 fixed on the CPU arm. */ DeadlockWindow = DeadlockWindow, DeadlockRateThresholds = DeadlockRateThresholds, + BlockingBandThresholds = BlockingBandThresholds, TotalThreads = TotalThreads, AvailableThreads = AvailableThreads, ThreadsWaitingForCpu = ThreadsWaitingForCpu, diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs index 7b42e0ab90..8436247a50 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs @@ -1377,7 +1377,10 @@ per the V71 finding. */ EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds' AND NOT attisdropped), /* V226 (#6893) adds the AWS RDS Web or Express fact to the server registry. Found by catalog like the V225 column above. Named only in this probe line, never in prose, per the V71 finding. */ - EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped)"; + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped), + /* V228 (#6927) adds the four settable blocking band columns to the alert settings row. Found by catalog like the V226 column above. The + warn per hour column is the sentinel, the first of the four. Named only in this probe line, never in prose, per the V71 finding. */ + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.config_alert_settings') AND attname = 'blocking_warn_per_hour' AND NOT attisdropped)"; /// The store schema version this viewer build requires — the highest migration it knows /// (). The connect-time gate blocks a store below this. @@ -1399,7 +1402,7 @@ per the V71 finding. */ await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201)); + return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201), reader.GetBoolean(202)); } return null; @@ -1424,7 +1427,7 @@ per the V71 finding. */ /// is unit-tested without a live store; any schema bump past the newest arm trips the pinning test that keeps /// this in step with . /// - internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false) + internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false, bool hasBlockingBands = false) { /* V71 (the PostgreSQL blocking-edges rung): a table-existence sentinel and now the newest-first arm. A collector table would ordinarily get no arm at all — see the V63-V69 note below — but the TOP @@ -1633,6 +1636,13 @@ store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through t /* V226 (#6893): the AWS RDS Web or Express fact on the server registry. Column-existence sentinel, newest-first, and the TOP rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 225 and showing a spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + /* V228 (#6927): the settable blocking band columns on the alert settings row. Column-existence sentinel, newest-first, and the TOP + rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 226 and showing a + spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + if (hasBlockingBands) + { + return 228; + } if (hasServerAwsRdsWebOrExpress) { return 226; diff --git a/PerformanceMonitor.Common/DailyHealthBand.cs b/PerformanceMonitor.Common/DailyHealthBand.cs index adb9176412..47a7dfbee0 100644 --- a/PerformanceMonitor.Common/DailyHealthBand.cs +++ b/PerformanceMonitor.Common/DailyHealthBand.cs @@ -211,6 +211,13 @@ public sealed record DailyHealthThresholds /// public DeadlockRateThresholds DeadlockRates { get; init; } = DeadlockRateThresholds.Default; + /// + /// The blocking band's tiers the day bands with (#6927) — the SAME store-backed values the Overview + /// card's blocking dot reads, so a day cell and the card cannot disagree about what a blocking rate or + /// a peak block means. Defaults to today's constants; Lite has no such knobs and bands on them. + /// + public BlockingBandThresholds BlockingBands { get; init; } = BlockingBandThresholds.Default; + /// The high-CPU sample count at or above which is Critical: the /// greater of and /// × the window's hours. An undeclared window @@ -275,7 +282,7 @@ arm are documented on ServerHealthThresholds. Delegating rather than restating i undeclared span) is the fallback this classifier wants: a non-zero count reads Warning, a zero count (Unknown) stays out of the blocking trigger. */ var blockingSeverity = ServerHealthClassifier.BlockingSeverity( - signals.BlockingReports ?? signals.BlockingEvents, signals.PeakBlockWaitMs / 1000.0, signals.Window); + signals.BlockingReports ?? signals.BlockingEvents, signals.PeakBlockWaitMs / 1000.0, signals.Window, t.BlockingBands); var cpuSeverity = HighCpuSeverity(signals.HighCpuEvents, signals.Window, t); var collectionSeverity = CollectionErrorSeverity(signals.CollectionErrors, signals.CollectionRuns); diff --git a/PerformanceMonitor.Common/ServerHealthBands.cs b/PerformanceMonitor.Common/ServerHealthBands.cs index c277dc33d3..7e53d036ff 100644 --- a/PerformanceMonitor.Common/ServerHealthBands.cs +++ b/PerformanceMonitor.Common/ServerHealthBands.cs @@ -448,6 +448,69 @@ public static class ServerHealthThresholds /// reach it first regardless. /// public const double DeadlockRatePerHourCeiling = 1000.0; + + /* ── the settable blocking band (#6927) ── */ + + /// + /// The FLOOR the settable blocking rate tiers clamp to (#6927): one event per hour, for the reason + /// gives. It keeps a tier a rate, and it keeps zero and + /// negative values, which would band every window Critical, out of the classifier. + /// + public const double BlockingRatePerHourFloor = 1.0; + + /// The CEILING the settable blocking rate tiers clamp to (#6927), so a tier stays a threshold and + /// not an undisclosed off switch. The worst measured server-hour held 232 reports, so 10,000 is far + /// above anything real. + public const double BlockingRatePerHourCeiling = 10000.0; + + /// The FLOOR the settable blocking wait tiers clamp to, seconds (#6927). A tier of zero would + /// band every block Critical. + public const double BlockingWaitSecondsFloor = 1.0; + + /// The CEILING the settable blocking wait tiers clamp to, seconds (#6927): one day. + public const double BlockingWaitSecondsCeiling = 86400.0; + } + + /// + /// The four settable tiers of the blocking band (#6927): events per hour and the longest single block in + /// seconds, each with a Warning and a Critical bar. The same shape as : + /// the store holds what the operator set, these accessors clamp on read, and a default struct + /// (every field zero) clamps to the floors rather than to zero. + /// + /// Defaults are today's constants ( and + /// its three siblings), so a store that never set them bands exactly as every build before this one + /// did. Lite has no store and keeps the constants. + /// + /// Like the deadlock pair, the critical bar is not floored at the warning bar here, so a hand-edited + /// pair reads as written. The writers refuse a pair with critical below warn. + /// + public readonly record struct BlockingBandThresholds( + double WarnPerHourRaw, + double CriticalPerHourRaw, + double WarnWaitSecondsRaw, + double CriticalWaitSecondsRaw) + { + /// Warning bar, events per hour, clamped. + public double WarnPerHour => Clamp(WarnPerHourRaw, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling); + + /// Critical bar, events per hour, clamped independently of the warning bar. + public double CriticalPerHour => Clamp(CriticalPerHourRaw, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling); + + /// Warning bar, longest block in seconds, clamped. + public double WarnWaitSeconds => Clamp(WarnWaitSecondsRaw, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling); + + /// Critical bar, longest block in seconds, clamped independently of the warning bar. + public double CriticalWaitSeconds => Clamp(CriticalWaitSecondsRaw, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling); + + /// Today's constants: what a store at its V228 column defaults hands the band. + public static BlockingBandThresholds Default => new( + ServerHealthThresholds.BlockingWarnPerHour, + ServerHealthThresholds.BlockingCriticalPerHour, + ServerHealthThresholds.BlockingWarnWaitSeconds, + ServerHealthThresholds.BlockingCriticalWaitSeconds); + + private static double Clamp(double value, double floor, double ceiling) => + double.IsNaN(value) ? floor : Math.Clamp(value, floor, ceiling); } /// @@ -582,6 +645,12 @@ public readonly record struct ServerHealthMetrics /// public DeadlockRateThresholds? DeadlockRateThresholds { get; init; } + /// The blocking band's four tiers (#6927), or null for today's constants + /// (). A Darling path with a store hands the + /// config_alert_settings values in so the card, the Overview tab, the calendar and the fleet + /// sweep band on what the operator set. Lite has no such knobs and leaves it null. + public BlockingBandThresholds? BlockingBandThresholds { get; init; } + /// Worker-thread ceiling (max_workers_count), or null with no scheduler snapshot (e.g. Azure SQL DB). public int? TotalThreads { get; init; } @@ -887,7 +956,8 @@ public static HealthSeverity MemorySeverity(bool? hasMemoryPressure) public static HealthSeverity BlockingSeverity( long? blockingCountOrNullWhenUnmeasured, double maxBlockedSeconds, - TimeSpan window) + TimeSpan window, + BlockingBandThresholds? thresholds = null) { if (!blockingCountOrNullWhenUnmeasured.HasValue) { @@ -895,19 +965,20 @@ public static HealthSeverity BlockingSeverity( } var blockingCount = blockingCountOrNullWhenUnmeasured.Value; + var tiers = thresholds ?? BlockingBandThresholds.Default; - if (maxBlockedSeconds >= ServerHealthThresholds.BlockingCriticalWaitSeconds) + if (maxBlockedSeconds >= tiers.CriticalWaitSeconds) { return HealthSeverity.Critical; } var ratePerHour = BlockingRatePerHour(blockingCount, window); - if (ratePerHour.HasValue && ratePerHour.Value >= ServerHealthThresholds.BlockingCriticalPerHour) + if (ratePerHour.HasValue && ratePerHour.Value >= tiers.CriticalPerHour) { return HealthSeverity.Critical; } - if (maxBlockedSeconds >= ServerHealthThresholds.BlockingWarnWaitSeconds) + if (maxBlockedSeconds >= tiers.WarnWaitSeconds) { return HealthSeverity.Warning; } @@ -917,7 +988,7 @@ public static HealthSeverity BlockingSeverity( return blockingCount > 0 ? HealthSeverity.Warning : HealthSeverity.Unknown; } - return ratePerHour.Value >= ServerHealthThresholds.BlockingWarnPerHour + return ratePerHour.Value >= tiers.WarnPerHour ? HealthSeverity.Warning : HealthSeverity.Healthy; } @@ -1140,7 +1211,7 @@ public static IEnumerable MetricSeverities(ServerHealthMetrics m yield return CpuSeverity(m.CpuPercentForAlert, m.CapacityUtilizationPercent, m.CpuSource); yield return ThreadsSeverity(m.TotalThreads, m.AvailableThreads, m.ThreadsWaitingForCpu, m.RequestsWaitingForThreads); yield return MemorySeverity(m.HasMemoryPressure); - yield return BlockingSeverity(m.BlockingCount, m.MaxBlockedSeconds, m.BlockingWindow); + yield return BlockingSeverity(m.BlockingCount, m.MaxBlockedSeconds, m.BlockingWindow, m.BlockingBandThresholds); yield return DeadlockSeverity( m.DeadlockCount, m.DeadlockWindow, From 5ec3b92772f51171f397fb894512c0945117765e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:47:20 -0400 Subject: [PATCH 03/10] Carry the blocking tiers on the Overview card item, and fix the settings row sample (#6927) --- Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs | 3 ++- Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs | 4 +++- .../ViewerDataService.Overview.cs | 5 +---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs b/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs index 2b0ee84ee0..92ff8853e8 100644 --- a/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs @@ -1333,7 +1333,8 @@ dropped the columns and fell back to the engine defaults still match. Warning is ClientTimeoutsWarningMultiple: 2, ClientTimeoutsCriticalMultiple: 7, AgentAlertGapsEnabled: true, BackupOverdueEnabled: true, LogBackupOverdueEnabled: true, CheckDbOverdueEnabled: true, - IdentityCapacityEnabled: true, IdentityCapacityWarningPercent: 72, IdentityCapacityCriticalPercent: 92); + IdentityCapacityEnabled: true, IdentityCapacityWarningPercent: 72, IdentityCapacityCriticalPercent: 92, + BlockingWarnPerHour: 6.5, BlockingCriticalPerHour: 22.5, BlockingWarnWaitSeconds: 11.5, BlockingCriticalWaitSeconds: 62.5); [Fact] public void AlertSettingsSql_ReadsSingleGlobalRow() diff --git a/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs index 46d0714e12..7dd680f253 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs @@ -1689,7 +1689,9 @@ is exactly why DarlingFleetReader.GetFleetOverviewAsync hoists its own copy of t try { var summary = await service.GetServerSummaryAsync( - server.ServerId, server.DisplayName, deadlockTiers, blockingTiers: blockingTiers); + server.ServerId, server.DisplayName, deadlockTiers); + /* #6927: the store's blocking tiers; the card's blocking band and its re-band read them from the item. */ + summary.BlockingBandThresholds = blockingTiers; summary.ServerName = server.ServerName; /* #3029: the engine discriminator comes from the REGISTRY row, which already carries it (servers.engine_kind, via ManagedServersSql / ServersSql) — the per-server summary diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs index e5031f2bc6..297ff2b1b0 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs @@ -290,8 +290,7 @@ public async Task GetServerSummaryAsync( int serverId, string displayName, DeadlockRateThresholds? deadlockTiers = null, - CancellationToken cancellationToken = default, - BlockingBandThresholds? blockingTiers = null) + CancellationToken cancellationToken = default) { var nowUtc = DateTime.UtcNow; /* #3368: the window's LENGTH is named once and the start derived from it, so the card's deadlock @@ -587,8 +586,6 @@ know the engine (the loader stamps IsPostgres afterwards), and one arm is always DeadlockWindow = window, BlockingWindow = window, DeadlockRateThresholds = deadlockTiers, - /* #6927: the store's blocking tiers, or null for today's constants. */ - BlockingBandThresholds = blockingTiers, TotalThreads = totalThreads, CurrentWorkers = currentWorkers, ThreadsWaitingForCpu = threadsWaitingForCpu, From 208e3fc0dff1e6662e1c641e2883a8587fe56474 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:52:02 -0400 Subject: [PATCH 04/10] Mark the daily classifier test day as having data (#6927) --- Darling/Darling.Tests/BlockingBandTiersTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Darling/Darling.Tests/BlockingBandTiersTests.cs b/Darling/Darling.Tests/BlockingBandTiersTests.cs index 7bd72a3314..e8d98cf0d0 100644 --- a/Darling/Darling.Tests/BlockingBandTiersTests.cs +++ b/Darling/Darling.Tests/BlockingBandTiersTests.cs @@ -44,6 +44,7 @@ public void TheDailyClassifier_UsesTheTunedTiers() { var signals = new DailyHealthSignals { + HasData = true, Window = Day, BlockingReports = 1764, BlockingEvents = 1764, From dc8a5ab824695f3866464c7c59b105dedf98e1f4 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:07:26 -0400 Subject: [PATCH 05/10] Band each fleet totals tile by its worst card, and test the V228 upgrade and the settings round trip (#6927) The fleet total was banded as one rate against the per-server tiers, so ten servers at 3 events an hour (none even Warning) made the tile Critical. The roll-up now takes the worst card band per signal, in the same fields. Adds live tests for the V228 upgrade on a store with a settings row and for update_alert_settings and get_alert_settings with the critical-below-warn refusal, and moves the settings column pins from 86 to 90. --- .../BlockingBandSettingsLiveTests.cs | 152 ++++++++++++++++++ .../Darling.Tests/BlockingBandTiersTests.cs | 34 +++- .../ClientTimeoutsSettingsSurfaceTests.cs | 14 +- .../Mcp/DarlingFleetReader.cs | 31 ++-- 4 files changed, 199 insertions(+), 32 deletions(-) create mode 100644 Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs diff --git a/Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs b/Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs new file mode 100644 index 0000000000..6f835bcf51 --- /dev/null +++ b/Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// The blocking band tiers (#6927, V228) on a real PostgreSQL store: an upgraded store whose settings row already exists keeps +/// today's defaults and its other values, and the tiers round trip through update_alert_settings and get_alert_settings +/// with a critical bar below its warn bar refused and nothing written. Each fact mints its own scratch database. +/// +/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Each fact mints its own scratch database through + ScratchPostgres and never touches the shared store's tables. */ +public sealed class BlockingBandSettingsLiveTests +{ + private const string TiersSql = "SELECT blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1"; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task AnUpgradedStoreWithASettingsRow_KeepsTodaysDefaults_AndItsOtherValues() + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the V228 upgrade test."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + + await PgMigrations.MigrateAsync(connection, ct); + await SeedAsync(scratch.ConnectionString, ct); + + /* Roll the store back to look like V227: the settings row exists and carries an operator's own deadlock tier, the four + columns do not. */ + await ExecAsync(connection, ct, "UPDATE config.config_alert_settings SET deadlock_warn_per_hour = 7.0, deadlock_critical_per_hour = 30.0 WHERE id = 1"); + foreach (var column in new[] { "blocking_warn_per_hour", "blocking_critical_per_hour", "blocking_warn_wait_seconds", "blocking_critical_wait_seconds" }) + { + await ExecAsync(connection, ct, "ALTER TABLE config.config_alert_settings DROP COLUMN " + column); + } + + await ExecAsync(connection, ct, "DELETE FROM darling_schema_version WHERE version >= 228"); + + var applied = await PgMigrations.MigrateAsync(connection, ct); + Assert.Equal(PgMigrations.Scripts.Count(m => m.Version >= 228), applied); + + Assert.Equal((5.0, 20.0, 10.0, 60.0), await ReadTiersAsync(connection, ct)); + + /* The band reads today's constants from the upgraded row, and the operator's deadlock tier did not move. */ + await using var dataSource = NpgsqlDataSource.Create(scratch.ConnectionString); + Assert.Equal(BlockingBandThresholds.Default, await BlockingBandStore.ReadAsync(dataSource, 10, ct)); + using var deadlock = new NpgsqlCommand("SELECT deadlock_warn_per_hour, deadlock_critical_per_hour FROM config_alert_settings WHERE id = 1", connection); + using var reader = await deadlock.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct)); + Assert.Equal(7.0, reader.GetDouble(0)); + Assert.Equal(30.0, reader.GetDouble(1)); + } + + [Fact] + public async Task UpdateAlertSettings_SavesTheFourTiers_GetAlertSettingsReadsThemBack_AndACriticalBelowWarnIsRefused() + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the blocking band settings test."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await SeedAsync(scratch.ConnectionString, ct); + + await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString); + + /* The seeded row reads today's constants. */ + AssertBands(await DarlingMcpAlertTools.GetAlertSettings(postgres), 5, 20, 10, 60); + + /* A valid body saves, and a fresh read carries it. */ + var saved = await DarlingMcpAlertTools.UpdateAlertSettings(postgres, + "{\"health_bands\":{\"blocking_warn_per_hour\":50,\"blocking_critical_per_hour\":100,\"blocking_warn_wait_seconds\":60,\"blocking_critical_wait_seconds\":180}}"); + Assert.Equal("updated", DarlingMcpTestData.StatusOf(saved)); + Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct)); + AssertBands(await DarlingMcpAlertTools.GetAlertSettings(postgres), 50, 100, 60, 180); + Assert.Equal(new BlockingBandThresholds(50, 100, 60, 180), await BlockingBandStore.ReadAsync(postgres, 10, ct)); + + /* A critical bar below its warn bar is refused with nothing written: in one body, and as a lone value against the stored other one. */ + foreach (var body in new[] + { + "{\"health_bands\":{\"blocking_warn_per_hour\":80,\"blocking_critical_per_hour\":30}}", + "{\"health_bands\":{\"blocking_critical_per_hour\":40}}", + "{\"health_bands\":{\"blocking_warn_wait_seconds\":200}}", + "{\"health_bands\":{\"blocking_warn_wait_seconds\":90,\"blocking_critical_wait_seconds\":30}}", + }) + { + var refused = await DarlingMcpAlertTools.UpdateAlertSettings(postgres, body); + Assert.Equal("invalid", DarlingMcpTestData.StatusOf(refused)); + Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct)); + } + + /* Out of range is refused too. */ + Assert.Equal("invalid", DarlingMcpTestData.StatusOf(await DarlingMcpAlertTools.UpdateAlertSettings(postgres, + "{\"health_bands\":{\"blocking_warn_per_hour\":0}}"))); + Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct)); + } + + private static void AssertBands(string settings, double warnPerHour, double criticalPerHour, double warnWait, double criticalWait) + { + Assert.False(McpHelpers.IsErrorEnvelope(settings), settings); + using var doc = JsonDocument.Parse(settings); + var bands = doc.RootElement.GetProperty("health_bands"); + Assert.Equal(warnPerHour, bands.GetProperty("blocking_warn_per_hour").GetDouble()); + Assert.Equal(criticalPerHour, bands.GetProperty("blocking_critical_per_hour").GetDouble()); + Assert.Equal(warnWait, bands.GetProperty("blocking_warn_wait_seconds").GetDouble()); + Assert.Equal(criticalWait, bands.GetProperty("blocking_critical_wait_seconds").GetDouble()); + } + + private static async Task<(double WarnPerHour, double CriticalPerHour, double WarnWait, double CriticalWait)> ReadTiersAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var command = new NpgsqlCommand(TiersSql, connection); + using var reader = await command.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct), "config_alert_settings has no singleton row"); + return (reader.GetDouble(0), reader.GetDouble(1), reader.GetDouble(2), reader.GetDouble(3)); + } + + private static async Task ExecAsync(NpgsqlConnection connection, CancellationToken ct, string sql) + { + using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(ct); + } + + /// Seeds the singleton rows through the product's own path, so this test need not track their other NOT NULL columns. + private static async Task SeedAsync(string connectionString, CancellationToken ct) + { + await using var dataSource = NpgsqlDataSource.Create(connectionString); + await new StoreConfigProvider(dataSource).SeedIfEmptyAsync(new DarlingConfig(), ct); + } +} diff --git a/Darling/Darling.Tests/BlockingBandTiersTests.cs b/Darling/Darling.Tests/BlockingBandTiersTests.cs index e8d98cf0d0..7f0e52cfe1 100644 --- a/Darling/Darling.Tests/BlockingBandTiersTests.cs +++ b/Darling/Darling.Tests/BlockingBandTiersTests.cs @@ -67,16 +67,40 @@ public void RawValuesClampToTheFloorsAndCeilings_AndADefaultStructClampsToTheFlo Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, default(BlockingBandThresholds).WarnPerHour); } + private static FleetServerCard CardAt(int id, HealthSeverity blocking, HealthSeverity deadlock) => new() + { + ServerId = id, + DisplayName = "s" + id, + ServerName = "s" + id, + BlockingCount = 3, + BlockingSeverity = blocking, + DeadlockSeverity = deadlock, + }; + [Fact] - public void TheFleetTotalsTiles_AreBandedByTheClassifier_NotOnAnyNonZeroCount() + public void TheFleetTotalsTiles_TakeTheWorstCardBand_SoTenServersAtThreeAnHourAreNotCritical() { var now = new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc); var empty = DarlingFleetReader.BuildRollup(Array.Empty(), now, now.AddHours(-1), now); + Assert.Equal(HealthSeverity.Healthy, empty.TotalBlockingSeverity); Assert.Equal(HealthSeverity.Healthy, empty.TotalDeadlockSeverity); - var tuned = DarlingFleetReader.BuildRollup( - new List(), now, now.AddHours(-1), now, - deadlockTiers: new DeadlockRateThresholds(50.0, 100.0), blockingTiers: new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0)); - Assert.Equal(HealthSeverity.Healthy, tuned.TotalDeadlockSeverity); + /* Ten servers each at 3 reports an hour: no card is even Warning at the default 5 an hour, and the fleet sum of 30 an hour + must not band as one server's rate. */ + var window = TimeSpan.FromHours(1); + var cards = new List(); + for (var i = 1; i <= 10; i++) + { + var sev = ServerHealthClassifier.BlockingSeverity(3, 0.0, window, BlockingBandThresholds.Default); + Assert.Equal(HealthSeverity.Healthy, sev); + cards.Add(CardAt(i, sev, HealthSeverity.Healthy)); + } + Assert.Equal(30, DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now).TotalBlockingEvents); + Assert.Equal(HealthSeverity.Healthy, DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now).TotalBlockingSeverity); + + cards[4] = CardAt(5, HealthSeverity.Critical, HealthSeverity.Warning); + var one = DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now); + Assert.Equal(HealthSeverity.Critical, one.TotalBlockingSeverity); + Assert.Equal(HealthSeverity.Warning, one.TotalDeadlockSeverity); } } diff --git a/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs b/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs index cfd8abc4ef..325bb6f96b 100644 --- a/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs +++ b/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs @@ -23,7 +23,7 @@ namespace Darling.Tests; /// #5640 (Lane E1): the Darling Viewer's Client Timeouts settings, the V179 twin of /// RunnableQueueSettingsSurfaceTests. The row carries the engine's defaults (ON, 10, 60), compares and saves /// all three columns through the engine's clamp, and the Settings window names the alert. V179 is the newest append, so -/// this class owns the settings row's total width (86). The viewer needs no per-version gate for the columns: it refuses a +/// this class owns the settings row's total width (90). The viewer needs no per-version gate for the columns: it refuses a /// store below StorageVersion.SchemaVersion at connect, as it did for V177's. /// public sealed class ClientTimeoutsSettingsSurfaceTests @@ -72,7 +72,7 @@ public void TheUpsertAndTheSelect_CarryTheThreeColumns_AtTheEnd() /// /// V179's three columns sit at the FIXED ordinals 73, 74 and 75 (parameters $74 to $76) on every positional - /// settings-row surface, and the row is 86 columns wide (V200 appended the excluded volumes at 76, V211 the two multiples at 77 and 78, V222 the Agent alert gaps switch at 79, V223 the three database protection switches at 80 to 82, V224 the three identity capacity columns at 83 to 85): the viewer's select and upsert, the service's own read and + /// settings-row surface, and the row is 90 columns wide (V228 appended the four blocking band tiers at 86 to 89, V200 appended the excluded volumes at 76, V211 the two multiples at 77 and 78, V222 the Agent alert gaps switch at 79, V223 the three database protection switches at 80 to 82, V224 the three identity capacity columns at 83 to 85): the viewer's select and upsert, the service's own read and /// its seed, and the MCP read with its positional record. This is the test that was red while the seed had 76 columns /// and the viewer 73. /// @@ -82,7 +82,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co var viewerColumns = ViewerDataService.AlertSettingsSelectSql .Replace("SELECT ", "", StringComparison.Ordinal).Split(" FROM ")[0] .Split(',').Select(c => c.Trim()).ToList(); - Assert.Equal(86, viewerColumns.Count); + Assert.Equal(90, viewerColumns.Count); Assert.Equal(Expected, viewerColumns.Skip(73).Take(3).ToArray()); var service = File.ReadAllText(Path.Combine(RepoRoot(), "Darling", "PerformanceMonitor.Darling.Service", "StoreConfigProvider.cs")); @@ -91,7 +91,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co var select = service[start..]; select = select[(select.IndexOf("SELECT", StringComparison.Ordinal) + 6)..select.IndexOf("FROM config_alert_settings WHERE id = 1", StringComparison.Ordinal)]; var serviceColumns = select.Split(',').Select(c => c.Trim()).Where(c => c.Length > 0).ToList(); - Assert.Equal(86, serviceColumns.Count); + Assert.Equal(90, serviceColumns.Count); Assert.Equal(Expected, serviceColumns.Skip(73).Take(3).ToArray()); Assert.Contains("ClientTimeoutsEnabled = reader.GetBoolean(73),", service, StringComparison.Ordinal); Assert.Contains("ClientTimeoutsWarningPerMinute = reader.GetInt32(74),", service, StringComparison.Ordinal); @@ -107,10 +107,10 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co var mcpColumns = DarlingAlertReader.AlertSettingsSelectSql .Split("FROM config_alert_settings")[0].Replace("SELECT", "", StringComparison.Ordinal) .Split(',').Select(c => c.Trim()).Where(c => c.Length > 0).ToList(); - Assert.Equal(86, mcpColumns.Count); + Assert.Equal(90, mcpColumns.Count); Assert.Equal(Expected, mcpColumns.Skip(73).Take(3).ToArray()); var parameters = typeof(DarlingAlertReader.AlertSettingsReadRow).GetConstructors().Single().GetParameters(); - Assert.Equal(86, parameters.Length); + Assert.Equal(90, parameters.Length); Assert.Equal(new[] { "ClientTimeoutsEnabled", "ClientTimeoutsWarningPerMinute", "ClientTimeoutsCriticalPerMinute" }, parameters.Skip(73).Take(3).Select(p => p.Name!).ToArray()); Assert.Equal(typeof(bool), parameters[73].ParameterType); @@ -120,7 +120,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co /* The viewer's bind supplies $74 to $76 at indexes 73 to 75, in column order (V200's excluded_volumes follows at 76). */ var row = new AlertSettingsRow { ClientTimeoutsEnabled = true, ClientTimeoutsWarningPerMinute = 5, ClientTimeoutsCriticalPerMinute = 11 }; var command = Bind(row); - Assert.Equal(86, command.Parameters.Count); + Assert.Equal(90, command.Parameters.Count); Assert.Equal(true, command.Parameters[73].Value); Assert.Equal(5, command.Parameters[74].Value); Assert.Equal(11, command.Parameters[75].Value); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index 5fcb31de44..fb4609d8be 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -973,7 +973,7 @@ server the read reported with an empty window reads Offline or never-collected b } return BuildRollup(cards, now, windowStartUtc, windowEndUtc, worstCount, tagForest, collectionHealthAgeSeconds, - await ReadCollectionStateAsync(postgres, logger, cancellationToken), deadlockTiers, blockingTiers); + await ReadCollectionStateAsync(postgres, logger, cancellationToken)); } /// Builds one pre-banded card from a server's raw cross-server reads (pure — the reduction the WPF @@ -1329,9 +1329,7 @@ public static FleetOverviewResult BuildRollup( int worstCount = DefaultWorstCount, IReadOnlyList? tags = null, int collectionHealthAgeSeconds = 0, - FleetCollectionState? collection = null, - DeadlockRateThresholds? deadlockTiers = null, - BlockingBandThresholds? blockingTiers = null) + FleetCollectionState? collection = null) { var healthy = 0; var warning = 0; @@ -1340,8 +1338,11 @@ public static FleetOverviewResult BuildRollup( var failures = 0; long totalBlocking = 0; long totalDeadlocks = 0; - long? totalBlockingReports = null; - var maxBlockedSeconds = 0.0; + /* #6927: each totals tile takes the worst band among the cards for that signal, the classification the cards already show. + A fleet total is not a per-server rate: ten servers at 3 an hour each would otherwise band the sum of 30 against the + per-server tiers and turn Critical with no card even Warning. */ + var blockingTileSeverity = HealthSeverity.Healthy; + var deadlockTileSeverity = HealthSeverity.Healthy; var deadlockSourcesRead = 0; var deadlockPostgresTargets = 0; var deadlockCollectorsSilent = 0; @@ -1385,14 +1386,8 @@ attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on totalBlocking += card.BlockingCount; totalDeadlocks += card.DeadlockCount; - /* #6927: the blocking tile bands on the reports the cards band on, and the longest block anywhere. A PostgreSQL card - has no blocking source, so it adds nothing and does not make the tile measured. */ - var cardMetrics = card.ToHealthMetrics(); - if (cardMetrics.BlockingCount.HasValue) - { - totalBlockingReports = (totalBlockingReports ?? 0) + cardMetrics.BlockingCount.Value; - maxBlockedSeconds = Math.Max(maxBlockedSeconds, cardMetrics.MaxBlockedSeconds); - } + if (card.BlockingSeverity > blockingTileSeverity) blockingTileSeverity = card.BlockingSeverity; + if (card.DeadlockSeverity > deadlockTileSeverity) deadlockTileSeverity = card.DeadlockSeverity; } var problems = cards @@ -1428,12 +1423,8 @@ attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on ServersWithCollectionFailures = failures, TotalBlockingEvents = totalBlocking, TotalDeadlocks = totalDeadlocks, - /* #6927: the two totals tiles band through the same classifier and the store's tiers as the cards, over the window the totals - were read in, instead of the page colouring them on any non-zero count. */ - TotalBlockingSeverity = ServerHealthClassifier.BlockingSeverity( - totalBlockingReports, maxBlockedSeconds, windowEndUtc - windowStartUtc, blockingTiers), - TotalDeadlockSeverity = ServerHealthClassifier.DeadlockSeverity( - totalDeadlocks, windowEndUtc - windowStartUtc, deadlockTiers ?? DeadlockRateThresholds.Default), + TotalBlockingSeverity = blockingTileSeverity, + TotalDeadlockSeverity = deadlockTileSeverity, DeadlockCoverage = new FleetDeadlockCoverage { ServersRead = deadlockSourcesRead, From 735da3965ef91c348886fa88a5671d18b3c61cb3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:50:01 -0400 Subject: [PATCH 06/10] Put the cancellation token last on GetSweepPressureAsync (CA1068) and use Assert.Single in the tag scope test --- Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs | 2 +- Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs | 4 ++-- .../ViewerDataService.CollectionHealth.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs index 1c5c98400b..b97cef96a9 100644 --- a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -165,7 +165,7 @@ await RunAsync(async (ds, connection, store, ct) => Assert.Equal("Production", cpu["tag"]!.GetValue()); Assert.Equal(92, cpu["effective_value"]!.GetValue()); Assert.Equal("global", Row(eight, "blocking.count_threshold")!["source"]!.GetValue()); - Assert.Equal(1, eight["tag_overrides"]!.AsArray().Count); + Assert.Single(eight["tag_overrides"]!.AsArray()); var nine = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-9", ct))!["server_view"]!; Assert.Equal("global", Row(nine, "cpu.threshold_percent")!["source"]!.GetValue()); diff --git a/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs b/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs index 59fec5ed1f..bbdb559253 100644 --- a/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs +++ b/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs @@ -77,7 +77,7 @@ public async Task TheViewersSweepPressureAndHeaviestCollectors_AreTheFiguresTheW var web = doc.RootElement.GetProperty("sweep_pressure"); await using var viewer = new ViewerDataService(scratch.ConnectionString); - var report = await viewer.GetSweepPressureAsync(ServerId, ct); + var report = await viewer.GetSweepPressureAsync(ServerId, cancellationToken: ct); Assert.NotNull(report); var pressure = report.Pressure; @@ -113,7 +113,7 @@ public async Task TheViewersSweepPressureAndHeaviestCollectors_AreTheFiguresTheW SweepPressurePanel.StatRows(pressure).Select(r => r.Label)); /* A server with no collection log rows has no report, as the tool answers it "unavailable". */ - Assert.Null(await viewer.GetSweepPressureAsync(EmptyServerId, ct)); + Assert.Null(await viewer.GetSweepPressureAsync(EmptyServerId, cancellationToken: ct)); bodySucceeded = true; } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs index 97cb7b2228..f0752874d1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs @@ -518,7 +518,7 @@ public async Task> GetCollectionHealthAsync(int serverI /// pre-gate runs are left out of both panels, as get_collection_health computes its figures from the running rows only, so the /// panel and the tool name the same peak collector for the same server. Null leaves every row in. public async Task GetSweepPressureAsync( - int serverId, CancellationToken cancellationToken = default, IReadOnlySet? engineGated = null) + int serverId, IReadOnlySet? engineGated = null, CancellationToken cancellationToken = default) { var scheduleOverrides = await ReadScheduleOverridesForHealthAsync(cancellationToken); var costs = new List<(CollectorHealthRow Row, double Avg, double P95, double Max)>(); From 3dee3305c98d4ad2b50ada268ae3f1173a0a85cf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:53:43 -0400 Subject: [PATCH 07/10] Keep tag overrides on a failed tag read and match the loader's accept rule in the server view (#6926) LoadAsync now throws when the tag tables could not be read, so the worker keeps the whole snapshot in force instead of installing one with no tag winners until the next config change. get_alert_settings filtered tag rows through its own known-key and integer checks. It now calls the loader's Accept, so a deeper tag whose value is outside the key's bounds is skipped there as it is in the engine, and the view names the tag the engine applies. --- .../ThresholdOverrideTagScopeTests.cs | 53 +++++++++++++++++++ .../DarlingThresholdOverrides.cs | 9 +++- .../Mcp/DarlingMcpThresholdOverrideTools.cs | 3 +- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs index b97cef96a9..7aeb652800 100644 --- a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -218,6 +218,59 @@ await RunAsync(async (ds, connection, store, ct) => }); } + [Fact] + public async Task AFailedTagTableRead_KeepsTheSnapshotInForce_NotAGlobalOne() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 92, tag: "Production"))); + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, "srv-9", "cpu.threshold_percent", 70))); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); + Assert.Equal(3, loader.Count); /* srv-9's own row, and the Production override for srv-7 and srv-8 */ + + /* The server-to-tag map cannot be read (a reset or a timeout stands in as a missing table), and a server override changes. */ + await ExecAsync(connection, "ALTER TABLE config.server_tag_map RENAME TO server_tag_map_away", ct); + await ExecAsync(connection, "UPDATE config.config_threshold_overrides SET value = 71 WHERE server_id = 9", ct); + await Assert.ThrowsAnyAsync(() => loader.LoadAsync(ds, ct)); + + /* Nothing was replaced: the tag override still applies and the changed server row is not half-loaded. */ + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); + Assert.Equal(70, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + Assert.Equal(3, loader.Count); /* srv-9's own row, and the Production override for srv-7 and srv-8 */ + + await ExecAsync(connection, "ALTER TABLE config.server_tag_map_away RENAME TO server_tag_map", ct); + await loader.LoadAsync(ds, ct); + Assert.Equal(71, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); + }); + } + + [Fact] + public async Task TheServerView_SkipsATagRowOutsideTheBounds_AsTheLoaderDoes() + { + await RunAsync(async (ds, connection, store, ct) => + { + /* srv-7 is under east (2) under Production (1). The deeper tag holds a value outside a percent's bounds (written by direct SQL). */ + await ExecAsync(connection, "INSERT INTO config.config_threshold_overrides (scope_tag_id, threshold_key, value, set_by) VALUES " + + "(1, 'cpu.threshold_percent', 85, 'erik'), (2, 'cpu.threshold_percent', 5000, 'erik')", ct); + await ExecAsync(connection, "INSERT INTO config.config_alert_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + await ExecAsync(connection, "INSERT INTO config.config_notification (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(85, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "7", null, 80)); + + var view = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-7", ct))!["server_view"]!; + var cpu = view["thresholds"]!.AsArray().First(n => n!["key"]!.GetValue() == "cpu.threshold_percent")!; + Assert.Equal("tag_override", cpu["source"]!.GetValue()); + Assert.Equal("Production", cpu["tag"]!.GetValue()); + Assert.Equal(85, cpu["effective_value"]!.GetValue()); + }); + } + [Fact] public async Task ListShowsTagRowsWithTheirTagName() { diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs index 1ea10e46c2..cb4906e908 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs @@ -46,7 +46,7 @@ private sealed record Snapshot(Dictionary<(int Server, string Database, string K /// read that throws leaves the overrides already in force. A row whose key is not a tunable threshold, whose value is /// not an integer inside the key's bounds, or whose database name sits on a server-only key is skipped and logged, /// never applied: the engine uses the value as-is. Tag overrides (#6926) are resolved with the tag tree and the server-to-tag - /// map, read only when the table holds a tag row; a tag-table read that fails leaves them matching nothing, as a tag-scoped route does. + /// map, read only when the table holds a tag row; a tag-table read that fails throws, so the whole snapshot already in force stays (it is not replaced by one with no tag overrides). /// internal async Task LoadAsync(NpgsqlDataSource postgres, CancellationToken cancellationToken = default) { @@ -72,6 +72,10 @@ internal async Task LoadAsync(NpgsqlDataSource postgres, CancellationToken cance { await using var connection = await postgres.OpenConnectionAsync(cancellationToken); inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, _logger, cancellationToken); + /* ReadRouteScopeInputsAsync swallows a failed tag read, and installing a snapshot with no tag winners would drop every + tag override to the global value until the next config change. Throw so the worker keeps the snapshot in force. */ + if (!inputs.TagsReadable) + throw new InvalidOperationException("The server tag tables could not be read, so the tag overrides cannot be resolved."); } Replace(direct, tag, inputs); @@ -125,7 +129,8 @@ internal void Replace( _snapshot = new Snapshot(direct, tagWinners); } - private static string? Accept(string key, double raw, string database, out int value) + /// Why a stored value cannot be applied (null: it can). The MCP server view filters tag rows through this too, so it never names a tag the engine skips. + internal static string? Accept(string key, double raw, string database, out int value) { value = 0; if (!DarlingThresholdKeys.TryGet(key, out var entry)) diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs index edab71bae2..f27e14ca86 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs @@ -369,7 +369,8 @@ internal static async Task BuildServerViewAsync( { var key = reader.GetString(2); var raw = reader.GetDouble(3); - if (!DarlingThresholdKeys.TryGet(key, out _) || raw != Math.Floor(raw) || raw < int.MinValue || raw > int.MaxValue) + /* The loader's own accept rule (known key, integer, inside the key's bounds), so a row the engine skips never names a tag here. */ + if (DarlingThresholdOverrides.Accept(key, raw, string.Empty, out _) is not null) continue; candidates.Add(new TagOverrideCandidate(reader.GetInt64(0), reader.GetInt32(1), key, raw)); meta[(reader.GetInt32(1), key)] = (reader.GetString(4), reader.GetDateTime(5)); From 9e89adc1969607ddfcad3c51b6ff3290accde1a6 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:00:29 -0400 Subject: [PATCH 08/10] Remove the unused DarlingAlertSettings.BlockingBandThresholds property (#6927) Nothing read it: the service, the viewer and the MCP tools all read the tiers through BlockingBandStore. The config fields it wrapped are still bound by the seed insert and the store load, so they stay. --- .../DarlingAlertSettings.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs index 048c648cc8..add0198c81 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs @@ -296,15 +296,6 @@ public IReadOnlyList WebhookChannelProblems _config.Alerts.DeadlockWarnPerHour, _config.Alerts.DeadlockCriticalPerHour); - /// #6927 (V228): the blocking health band's four tiers as ONE value, for the reason - /// is one. The clamp lives on - /// , so the viewer's card and the service's read clamp alike. - public BlockingBandThresholds BlockingBandThresholds => new( - _config.Alerts.BlockingWarnPerHour, - _config.Alerts.BlockingCriticalPerHour, - _config.Alerts.BlockingWarnWaitSeconds, - _config.Alerts.BlockingCriticalWaitSeconds); - public int CollectionFailureThreshold => Math.Clamp(_config.Alerts.CollectionFailureThreshold, 1, 1000); /* #1984: percent clamped like low-disk's (0 = off); the GB floor merely floored at 0 — unlike From 9dd1a4677d154d9feebce6e0e3b572243ab7c5bf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:30:15 -0400 Subject: [PATCH 09/10] Declare the V227 rung in the data-moving census and build the tag-name read from the store source (#6926) V227 adds a nullable column, two CHECKs and a partial unique index to the operator-written config.config_threshold_overrides table, a handful of rows, so it is declared SetsTheFloor false and the lock-wait multiple stays 5. The tag-name read in the override tool now uses postgres.CreateCommand like its sibling reads, which is the store receiver the read-deadline census recognises. --- .../Darling.Tests/MigrationDataMovingRungCensusPins.cs | 10 ++++++++++ .../Mcp/DarlingMcpThresholdOverrideTools.cs | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs b/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs index 2c222b64ce..5bd785e1c8 100644 --- a/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs +++ b/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs @@ -359,6 +359,16 @@ public sealed class MigrationDataMovingRungCensusPins + "singleton (id = 1): turns the Client Timeouts alert on by default (#5736). V147's shape on the same kind of table. The " + "UPDATE's WHERE client_timeouts_enabled = FALSE touches at most one row, the SET DEFAULT is catalog-only, and neither reads " + "any collected data, so the rung spends none of the lock-wait budget and the multiple stays 5."), + new( + 227, + SetsTheFloor: false, + "one CREATE UNIQUE INDEX and two ADD CONSTRAINT ... CHECK on config.config_threshold_overrides (#6926, created V191), " + + "which hold the new tag scope. The table is an operator-written config table: one row per threshold somebody chose to " + + "override for a server, so a store holds a handful of rows and a very large one a few hundred. The CHECKs are validated " + + "against every stored row and the partial index (WHERE scope_tag_id IS NOT NULL) indexes nothing, because the column the " + + "same rung adds is null on every stored row, so each statement reads the table once and takes milliseconds. The ADD " + + "COLUMN is nullable with no default and the DROP NOT NULL on server_id is catalog-only, so neither rewrites a row. It does " + + "not move the lock-wait floor, so the multiple stays 5."), ]; /// diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs index f27e14ca86..cf8748de8d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs @@ -382,9 +382,10 @@ internal static async Task BuildServerViewAsync( await using var connection = await postgres.OpenConnectionAsync(cancellationToken); var inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, null, cancellationToken); var names = new System.Collections.Generic.Dictionary(); - await using (var command = new NpgsqlCommand("SELECT id, name FROM config.server_tags", connection) { CommandTimeout = McpCommandDeadlines.ReadSeconds }) - await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) + await using (var command = postgres.CreateCommand("SELECT id, name FROM config.server_tags")) { + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) names[reader.GetInt32(0)] = reader.GetString(1); } From fd3bda92a6a9b55003d7709550a80f0ba7f65d1e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:52:03 -0400 Subject: [PATCH 10/10] Let the V227 live test follow the newest schema version instead of pinning 227 --- Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs index 7aeb652800..553cdd8a60 100644 --- a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -313,7 +313,7 @@ await ExecAsync(store, "INSERT INTO config.config_threshold_overrides (server_id "SELECT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'config.config_threshold_overrides'::regclass AND attname = 'scope_tag_id' AND NOT attisdropped)", ct)); await PgMigrations.MigrateAsync(store, ct); - Assert.Equal("227", await ScalarAsync(store, "SELECT max(version) FROM darling_schema_version", ct)); + Assert.Equal(StorageVersion.SchemaVersion.ToString(CultureInfo.InvariantCulture), await ScalarAsync(store, "SELECT max(version) FROM darling_schema_version", ct)); Assert.Equal("2", await ScalarAsync(store, "SELECT count(*) FROM config.config_threshold_overrides WHERE server_id = 7 AND scope_tag_id IS NULL", ct)); Assert.Equal("91", await ScalarAsync(store, "SELECT value FROM config.config_threshold_overrides WHERE database_name IS NULL", ct));