From 4b19caf730fb02546da7615434b4683498710d54 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:18:05 -0400 Subject: [PATCH 1/4] Hold threshold overrides for a server tag (#6926) A threshold override can now be set for a server tag. Precedence is database, then server, then the deepest tag the server is under (the notification routes' own rule), then global. Adds V227, the MCP tag argument, the get_alert_settings tag source, the Viewer and web Admin tag rows, and tests. --- Darling/Darling.Tests/RungRewindArtifacts.cs | 10 + .../ThresholdOverrideTagScopeTests.cs | 329 ++++++++++++++++++ .../DarlingThresholdOverrides.cs | 105 ++++-- .../DarlingWebThresholdOverrides.cs | 29 +- .../Mcp/DarlingMcpThresholdOverrideTools.cs | 151 +++++++- .../wwwroot/js/pages/admin.js | 55 +-- .../PgMigrations.cs | 44 +++ .../StorageVersion.cs | 2 +- .../TagThresholdResolver.cs | 71 ++++ .../ThresholdOverrideStore.TagScope.cs | 94 +++++ .../ThresholdOverrideStore.cs | 31 +- .../ThresholdOverridesViewModel.cs | 56 ++- .../ThresholdOverridesWindow.xaml | 10 +- .../ThresholdOverridesWindow.xaml.cs | 11 +- .../ViewerDataService.ThresholdOverrides.cs | 31 ++ .../ViewerDataService.cs | 14 +- docs/darling/admin.md | 1 + 17 files changed, 953 insertions(+), 91 deletions(-) create mode 100644 Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs create mode 100644 Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs create mode 100644 Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs diff --git a/Darling/Darling.Tests/RungRewindArtifacts.cs b/Darling/Darling.Tests/RungRewindArtifacts.cs index 375b56efdd..a4273f778b 100644 --- a/Darling/Darling.Tests/RungRewindArtifacts.cs +++ b/Darling/Darling.Tests/RungRewindArtifacts.cs @@ -577,6 +577,16 @@ ALTER TABLE config.config_alert_settings // so a re-run of V226 does everything again. Safe on a store that never had it. new(226, "hasServerAwsRdsWebOrExpress", @"ALTER TABLE collect.servers DROP COLUMN IF EXISTS is_aws_rds_web_or_express;"), + // V227 (#6926): the tag scope on the threshold overrides. The column is the probe's newest sentinel. The rewind removes the tag rows, + // the two constraints and the tag index, puts server_id back to NOT NULL and drops the column, so a re-run of V227 does everything + // again. Safe on a store that never had it. + new(227, "hasThresholdOverrideTagScope", + @"DELETE FROM config.config_threshold_overrides WHERE server_id IS NULL; +ALTER TABLE config.config_threshold_overrides DROP CONSTRAINT IF EXISTS ck_config_threshold_overrides_one_scope; +ALTER TABLE config.config_threshold_overrides DROP CONSTRAINT IF EXISTS ck_config_threshold_overrides_tag_no_database; +DROP INDEX IF EXISTS config.ux_config_threshold_overrides_tag; +ALTER TABLE config.config_threshold_overrides ALTER COLUMN server_id SET NOT NULL; +ALTER TABLE config.config_threshold_overrides DROP COLUMN IF EXISTS scope_tag_id;"), ]; /// diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs new file mode 100644 index 0000000000..1c5c98400b --- /dev/null +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Npgsql; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/* #1776 own-store: every live fact mints its own scratch database through ScratchPostgres and never touches another test's rows, + so the class is deliberately NOT [Collection("live-postgres")]. */ + +/// +/// #6926, a threshold override held for a server tag. The precedence (database, then server, then tag, then global), the tag rule +/// (the notification routes' own: the deepest tag a server is under, the lowest row id on a same-depth tie, a deleted tag matching +/// nothing) and the whole path through a live store: set_alert_threshold_override with a tag, get_alert_settings +/// naming the tag, the list marking an orphan, and V227 on a store that already holds server overrides. +/// +public sealed class ThresholdOverrideTagScopeTests +{ + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private static readonly DarlingMcpWritePolicy.Effective WritesArm = new(true, false); + private static readonly DarlingMcpWritePolicy.Effective AgentArm = new(false, false, Investigations: true); + + /* Tags: 1 prod (root) > 2 east > 3 rack1; 4 dev (root); 5 and 6 are two roots a server can sit under at once. */ + private static readonly Dictionary Parents = new() + { + [1] = null, [2] = 1, [3] = 2, [4] = null, [5] = null, [6] = null, + }; + + private static StoreConfigProvider.RouteScopeInputs Inputs(bool readable = true) => new( + new HashSet { 7, 8, 9, 10 }, + Parents, + new Dictionary> + { + [7] = new[] { 3 }, /* under rack1: prod > east > rack1 */ + [8] = new[] { 1 }, /* directly under prod */ + [9] = new[] { 4 }, /* dev */ + [10] = new[] { 5, 6 }, /* two roots at the same depth */ + }, + readable); + + private static DarlingThresholdOverrides Load( + (int Server, string? Database, string Key, double Value)[] direct, + (long RowId, int Tag, string Key, double Value)[] tags, + bool readable = true) + { + var overrides = new DarlingThresholdOverrides(); + overrides.Replace(direct, tags.Select(t => (t.RowId, t.Tag, t.Key, t.Value)), Inputs(readable)); + return overrides; + } + + [Fact] + public void Precedence_DatabaseBeatsServerBeatsTagBeatsGlobal() + { + const string Key = "pvs.threshold_percent"; + var o = Load( + [(7, "shop", Key, 60), (7, null, Key, 55)], + [(1, 1, Key, 50)]); + + Assert.Equal(60, o.ResolveWire(Key, "7", "shop", 40)); /* the database override */ + Assert.Equal(55, o.ResolveWire(Key, "7", "ledger", 40)); /* no override for that database: the server's */ + Assert.Equal(55, o.ResolveWire(Key, "7", null, 40)); + Assert.Equal(50, o.ResolveWire(Key, "8", null, 40)); /* no server override: the tag's */ + Assert.Equal(50, o.ResolveWire(Key, "8", "shop", 40)); + Assert.Equal(40, o.ResolveWire(Key, "9", null, 40)); /* a server outside the tag: global */ + Assert.Equal(40, o.ResolveWire("cpu.threshold_percent", "8", null, 40)); + } + + [Fact] + public void TagRule_DeepestTagWins_AndASameDepthTieGoesToTheLowestRowId() + { + const string Key = "cpu.threshold_percent"; + + /* Server 7 sits under rack1 (2 ancestors); prod (0 ancestors) also covers it. The deeper tag wins whatever the row ids. */ + var nested = Load([], [(1, 1, Key, 80), (2, 3, Key, 70), (3, 2, Key, 75)]); + Assert.Equal(70, nested.ResolveWire(Key, "7", null, 90)); + Assert.Equal(80, nested.ResolveWire(Key, "8", null, 90)); /* server 8 is only under prod */ + + /* Server 10 is under two roots of the same depth: the lower row id wins, the way the router takes the lower route id. */ + var tie = Load([], [(20, 5, Key, 65), (10, 6, Key, 85)]); + Assert.Equal(85, tie.ResolveWire(Key, "10", null, 90)); + var reversed = Load([], [(10, 5, Key, 65), (20, 6, Key, 85)]); + Assert.Equal(65, reversed.ResolveWire(Key, "10", null, 90)); + } + + [Fact] + public void ADeletedTag_MatchesNothing_AndUnreadableTagTablesMatchNothing() + { + const string Key = "cpu.threshold_percent"; + var orphan = Load([], [(1, 99, Key, 70)]); /* tag 99 is not in the tree */ + Assert.Equal(90, orphan.ResolveWire(Key, "7", null, 90)); + Assert.Equal(0, orphan.Count); + + var unreadable = Load([], [(1, 1, Key, 70)], readable: false); + Assert.Equal(90, unreadable.ResolveWire(Key, "8", null, 90)); + } + + [Fact] + public void ATagRowWithAnInvalidValue_IsSkipped_AndTheShallowerTagStillApplies() + { + const string Key = "cpu.threshold_percent"; + var o = Load([], [(1, 1, Key, 80), (2, 3, Key, 5000)]); /* 5000 is out of bounds for a percent */ + Assert.Equal(80, o.ResolveWire(Key, "7", null, 90)); + } + + [Fact] + public async Task Tool_SetsAndClearsATagOverride_TheLoaderAndTheServerViewNameTheTag() + { + await RunAsync(async (ds, connection, store, ct) => + { + var set = await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 95, tag: "Production", setBy: "erik"); + Assert.Equal("ok", Status(set)); + Assert.Equal("Production", JsonNode.Parse(set)!["tag"]!.GetValue()); + Assert.Equal("1", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides WHERE scope_tag_id = 1 AND server_id IS NULL", ct)); + Assert.Equal("erik", await ScalarAsync(connection, "SELECT set_by FROM config.config_threshold_overrides WHERE scope_tag_id = 1", ct)); + + /* A second call updates the same row (by tag id this time); the answer carries the undo. */ + var again = await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 92, tag: "1"); + Assert.Equal(95, JsonNode.Parse(again)!["previous_override"]!.GetValue()); + Assert.Equal("1", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides WHERE scope_tag_id = 1", ct)); + + /* A nested tag with its own value: the deeper tag wins for the server under it. */ + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 88, tag: "east"))); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(88, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "7", null, 80)); /* srv-7 is under east */ + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); /* srv-8 is directly under Production */ + Assert.Equal(80, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + + /* A server override beats the tag in the loader and in the view. */ + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, "srv-7", "cpu.threshold_percent", 60))); + await loader.LoadAsync(ds, ct); + Assert.Equal(60, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "7", null, 80)); + + await ExecAsync(connection, "INSERT INTO config.config_alert_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + await ExecAsync(connection, "INSERT INTO config.config_notification (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + JsonNode? Row(JsonNode view, string key) => view["thresholds"]!.AsArray().First(n => n!["key"]!.GetValue() == key); + + var seven = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-7", ct))!["server_view"]!; + Assert.Equal("server_override", Row(seven, "cpu.threshold_percent")!["source"]!.GetValue()); + + var eight = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-8", ct))!["server_view"]!; + var cpu = Row(eight, "cpu.threshold_percent")!; + Assert.Equal("tag_override", cpu["source"]!.GetValue()); + Assert.Equal("Production", cpu["tag"]!.GetValue()); + Assert.Equal(92, cpu["effective_value"]!.GetValue()); + Assert.Equal("global", Row(eight, "blocking.count_threshold")!["source"]!.GetValue()); + Assert.Equal(1, eight["tag_overrides"]!.AsArray().Count); + + var nine = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-9", ct))!["server_view"]!; + Assert.Equal("global", Row(nine, "cpu.threshold_percent")!["source"]!.GetValue()); + + /* Clear works the same way, by name or id. */ + Assert.Equal("cleared", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", null, tag: "east", clear: true))); + Assert.Equal("cleared", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", null, tag: "1", clear: true))); + Assert.Equal("0", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides WHERE scope_tag_id IS NOT NULL", ct)); + }); + } + + [Fact] + public async Task Tool_RefusesTagMisuse_AndLeavesTheTableUntouched() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("scope_conflict", Reason(await Set(ds, store, WritesArm, "srv-7", "cpu.threshold_percent", 90, tag: "Production"))); + Assert.Equal("scope_required", Reason(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 90))); + Assert.Equal("database_scope", Reason(await Set(ds, store, WritesArm, null, "pvs.threshold_percent", 70, database: "shop", tag: "Production"))); + Assert.Equal("tag_not_found", Reason(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 90, tag: "nope"))); + Assert.Equal("tag_writes_arm_only", Reason(await Set(ds, store, AgentArm, null, "cpu.threshold_percent", 90, tag: "Production"))); + Assert.Equal("0", await ScalarAsync(connection, "SELECT count(*) FROM config.config_threshold_overrides", ct)); + + /* A name two tags share is refused until the id is used. */ + await ExecAsync(connection, "INSERT INTO config.server_tags (name, parent_id) VALUES ('Shared', 1), ('Shared', 4)", ct); + Assert.Equal("tag_not_found", Reason(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 90, tag: "shared"))); + }); + } + + [Fact] + public async Task ADeletedTag_ListsAsOrphaned_MatchesNothing_AndCanStillBeCleared() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 95, tag: "dev"))); + await ExecAsync(connection, "DELETE FROM config.server_tags WHERE id = 4", ct); + + var rows = await ThresholdOverrideStore.ListAsync(ds, k => null, ct); + var row = Assert.Single(rows); + Assert.True(row.IsTag); + Assert.True(row.Orphaned); + Assert.Equal(4, row.TagId); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(80, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + + Assert.Equal("cleared", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", null, tag: "4", clear: true))); + Assert.Empty(await ThresholdOverrideStore.ListAsync(ds, k => null, ct)); + }); + } + + [Fact] + public async Task ListShowsTagRowsWithTheirTagName() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 95, tag: "Production"))); + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, "srv-7", "cpu.threshold_percent", 90))); + var rows = await ThresholdOverrideStore.ListAsync(ds, k => null, ct); + Assert.Equal(2, rows.Count); + var tag = rows.Single(r => r.IsTag); + Assert.Equal("Production", tag.ServerName); + Assert.False(tag.Orphaned); + Assert.Equal("srv-7", rows.Single(r => !r.IsTag).ServerName); + + var body = await DarlingWebThresholdOverrides.ListBodyAsync(ds, ct); + Assert.Equal(2, body["overrides"]!.AsArray().Count); + Assert.Contains(body["tags"]!.AsArray(), t => t!["tag_name"]!.GetValue() == "Production"); + }); + } + + [Fact] + public async Task V227_OnAStoreThatHoldsServerOverrides_KeepsThem_AndTakesTagRows() + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to run the threshold override live tests."); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var store = new NpgsqlConnection(scratch.ConnectionString); + await store.OpenAsync(ct); + await PgMigrations.MigrateAsync(store, ct); + + var rung = PgMigrations.Scripts.Single(m => m.Name == "threshold-override-tag-scope").Version; + Assert.Equal(227, rung); + await RungRewindArtifacts.DropSentinelsAboveAsync(store, rung - 1, ct); + await ExecAsync(store, "DELETE FROM darling_schema_version WHERE version >= " + rung.ToString(CultureInfo.InvariantCulture), ct); + await ExecAsync(store, "INSERT INTO config.config_monitored_servers (server_id, name, host) VALUES (7, 'srv-7', 'h7')", ct); + await ExecAsync(store, "INSERT INTO config.config_threshold_overrides (server_id, database_name, threshold_key, value, set_by) VALUES " + + "(7, NULL, 'cpu.threshold_percent', 91, 'erik'), (7, 'shop', 'pvs.threshold_percent', 66, 'erik')", ct); + Assert.Equal("False", await ScalarAsync(store, + "SELECT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'config.config_threshold_overrides'::regclass AND attname = 'scope_tag_id' AND NOT attisdropped)", ct)); + + await PgMigrations.MigrateAsync(store, ct); + Assert.Equal("227", await ScalarAsync(store, "SELECT max(version) FROM darling_schema_version", ct)); + Assert.Equal("2", await ScalarAsync(store, "SELECT count(*) FROM config.config_threshold_overrides WHERE server_id = 7 AND scope_tag_id IS NULL", ct)); + Assert.Equal("91", await ScalarAsync(store, "SELECT value FROM config.config_threshold_overrides WHERE database_name IS NULL", ct)); + + /* A tag row is accepted; a row with neither scope, both scopes or a database on a tag is not; one value per tag and key. */ + await ExecAsync(store, "INSERT INTO config.server_tags (name) VALUES ('Production')", ct); + await ExecAsync(store, "INSERT INTO config.config_threshold_overrides (scope_tag_id, threshold_key, value, set_by) VALUES (1, 'cpu.threshold_percent', 80, 'erik')", ct); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (scope_tag_id, threshold_key, value, set_by) VALUES (1, 'cpu.threshold_percent', 70, 'erik')", ct)); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (threshold_key, value, set_by) VALUES ('cpu.threshold_percent', 70, 'erik')", ct)); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (server_id, scope_tag_id, threshold_key, value, set_by) VALUES (7, 1, 'blocking.count_threshold', 70, 'erik')", ct)); + await Assert.ThrowsAsync(() => ExecAsync(store, + "INSERT INTO config.config_threshold_overrides (scope_tag_id, database_name, threshold_key, value, set_by) VALUES (1, 'shop', 'pvs.threshold_percent', 70, 'erik')", ct)); + + /* A second run changes nothing, and the version stamp is not repeated. */ + await ExecAsync(store, "DELETE FROM darling_schema_version WHERE version >= " + rung.ToString(CultureInfo.InvariantCulture), ct); + await PgMigrations.MigrateAsync(store, ct); + Assert.Equal("3", await ScalarAsync(store, "SELECT count(*) FROM config.config_threshold_overrides", ct)); + } + + private static async Task RunAsync(Func body) + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the threshold override live tests."); + + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await ExecAsync(connection, "INSERT INTO config.config_monitored_servers (server_id, name, host) VALUES (7, 'srv-7', 'h7'), (8, 'srv-8', 'h8'), (9, 'srv-9', 'h9')", ct); + await ExecAsync(connection, "INSERT INTO collect.servers (server_id, server_name, display_name, is_enabled, sql_major_version, created_date, modified_date) " + + "VALUES (7, 'srv-7', 'srv-7', TRUE, 15, now(), now()), (8, 'srv-8', 'srv-8', TRUE, 15, now(), now()), (9, 'srv-9', 'srv-9', TRUE, 15, now(), now())", ct); + /* Tag ids 1 Production, 2 east (under Production), 3 rack1 (under east), 4 dev; srv-7 is under east, srv-8 under Production, srv-9 under dev. */ + await ExecAsync(connection, "INSERT INTO config.server_tags (name, parent_id) VALUES ('Production', NULL), ('east', 1), ('rack1', 2), ('dev', NULL)", ct); + await ExecAsync(connection, "INSERT INTO config.server_tag_map (server_id, tag_id) VALUES (7, 2), (8, 1), (9, 4)", ct); + + var builder = new NpgsqlConnectionStringBuilder(scratch.ConnectionString) { SearchPath = PgSchemaGenerator.SearchPath }; + await using var dataSource = NpgsqlDataSource.Create(builder.ConnectionString); + var store = new InvestigationRequestStore(dataSource, 30, () => new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc)); + await body(dataSource, connection, store, ct); + } + + private static Task Set(NpgsqlDataSource ds, InvestigationRequestStore store, DarlingMcpWritePolicy.Effective policy, + string? server, string key, int? value, string? database = null, bool clear = false, string? setBy = null, string? tag = null) => + DarlingMcpThresholdOverrideTools.SetAsync(ds, policy, store, NullLogger.Instance, server, key, value, database, clear, setBy, null, + TestContext.Current.CancellationToken, null, tag); + + private static string Status(string json) => JsonNode.Parse(json)!["status"]!.GetValue(); + + private static string Reason(string json) => JsonNode.Parse(json)!["reason"]!.GetValue(); + + private static async Task ExecAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + return Convert.ToString(await command.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture) ?? ""; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs index aa54bd0fae..1ea10e46c2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs @@ -14,11 +14,12 @@ using Microsoft.Extensions.Logging; using Npgsql; using PerformanceMonitor.Alerting; +using PerformanceMonitor.Darling.Storage; namespace PerformanceMonitor.Darling.Service; /// -/// The per-server and per-database threshold overrides the alert engine and the PostgreSQL count checks resolve through +/// The per-server, per-database and per-tag threshold overrides the alert engine and the PostgreSQL count checks resolve through /// (#5916): the Darling loader behind . It holds one immutable snapshot of /// config.config_threshold_overrides and answers from it, so the sweep path does no I/O. The service reloads it at /// start and on every control-plane reload (the table is on the config_version beacon), and a failed load keeps the @@ -27,66 +28,101 @@ namespace PerformanceMonitor.Darling.Service; internal sealed class DarlingThresholdOverrides : IAlertThresholdOverrides { internal const string SelectSql = - "SELECT server_id, database_name, threshold_key, value FROM config.config_threshold_overrides"; + "SELECT id, server_id, database_name, scope_tag_id, threshold_key, value FROM config.config_threshold_overrides"; + + /// The direct (server and database) overrides, and the tag override that wins per server and key. + private sealed record Snapshot(Dictionary<(int Server, string Database, string Key), int> Direct, Dictionary<(int Server, string Key), int> Tag); private readonly ILogger? _logger; - private volatile Dictionary<(int Server, string Database, string Key), int> _snapshot = new(); + private volatile Snapshot _snapshot = new(new(), new()); internal DarlingThresholdOverrides(ILogger? logger = null) => _logger = logger; - /// How many overrides are in force. - internal int Count => _snapshot.Count; + /// How many overrides are in force: the server and database ones, and the tag ones that some server resolves to. + internal int Count => _snapshot.Direct.Count + _snapshot.Tag.Count; /// /// Replaces the snapshot with the table's rows. The read is awaited before the swap and nothing here catches, so a /// read that throws leaves the overrides already in force. A row whose key is not a tunable threshold, whose value is /// not an integer inside the key's bounds, or whose database name sits on a server-only key is skipped and logged, - /// never applied: the engine uses the value as-is. + /// never applied: the engine uses the value as-is. Tag overrides (#6926) are resolved with the tag tree and the server-to-tag + /// map, read only when the table holds a tag row; a tag-table read that fails leaves them matching nothing, as a tag-scoped route does. /// internal async Task LoadAsync(NpgsqlDataSource postgres, CancellationToken cancellationToken = default) { - var next = new Dictionary<(int, string, string), int>(); - await using var command = postgres.CreateCommand(SelectSql); - command.CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds; - await using var reader = await command.ExecuteReaderAsync(cancellationToken); - while (await reader.ReadAsync(cancellationToken)) + var direct = new List<(int ServerId, string? Database, string Key, double Value)>(); + var tag = new List<(long RowId, int TagId, string Key, double Value)>(); + await using (var command = postgres.CreateCommand(SelectSql)) { - var serverId = reader.GetInt32(0); - var database = reader.IsDBNull(1) ? string.Empty : reader.GetString(1); - var key = reader.GetString(2); - var raw = reader.GetDouble(3); - if (Accept(key, raw, database, out var value) is { } problem) + command.CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) { - LogSkipped(serverId, key, database, problem); - continue; + var key = reader.GetString(4); + var raw = reader.GetDouble(5); + if (!reader.IsDBNull(3)) + tag.Add((reader.GetInt64(0), reader.GetInt32(3), key, raw)); + else if (!reader.IsDBNull(1)) + direct.Add((reader.GetInt32(1), reader.IsDBNull(2) ? null : reader.GetString(2), key, raw)); } + } - next[(serverId, database, key)] = value; + StoreConfigProvider.RouteScopeInputs? inputs = null; + if (tag.Count > 0) + { + await using var connection = await postgres.OpenConnectionAsync(cancellationToken); + inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, _logger, cancellationToken); } - _snapshot = next; + Replace(direct, tag, inputs); } - private void LogSkipped(int serverId, string key, string database, string problem) => + private void LogSkipped(string scope, string key, string database, string problem) => _logger?.LogWarning( - "Threshold override for server {ServerId} key '{Key}' database '{Database}' was skipped: {Problem}", - serverId, key, database, problem); + "Threshold override for {Scope} key '{Key}' database '{Database}' was skipped: {Problem}", + scope, key, database, problem); /// Replaces the snapshot from rows already in hand (the loader's own tests, and any caller that read the table). A /// skipped row is logged the way logs it. - internal void Replace(IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows) + internal void Replace(IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows) => + Replace(rows, [], null); + + /// The same with tag overrides (#6926). carry the table row id, which breaks a tie between two + /// tags of the same depth, and the registry, tag tree and server-to-tag map the routes expand against + /// (null: the tag tables were not read, and a tag override matches nothing). + internal void Replace( + IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows, + IEnumerable<(long RowId, int TagId, string Key, double Value)> tagRows, + StoreConfigProvider.RouteScopeInputs? inputs) { - var next = new Dictionary<(int, string, string), int>(); + var direct = new Dictionary<(int, string, string), int>(); foreach (var (serverId, database, key, raw) in rows) { var db = database ?? string.Empty; if (Accept(key, raw, db, out var value) is { } problem) - LogSkipped(serverId, key, db, problem); + LogSkipped("server " + serverId.ToString(CultureInfo.InvariantCulture), key, db, problem); else - next[(serverId, db, key)] = value; + direct[(serverId, db, key)] = value; + } + + var accepted = new List(); + foreach (var (rowId, tagId, key, raw) in tagRows) + { + if (Accept(key, raw, string.Empty, out var value) is { } problem) + LogSkipped("tag " + tagId.ToString(CultureInfo.InvariantCulture), key, string.Empty, problem); + else + accepted.Add(new TagOverrideCandidate(rowId, tagId, key, value)); + } + + var tagWinners = new Dictionary<(int, string), int>(); + if (accepted.Count > 0 && inputs is not null) + { + foreach (var (slot, winner) in TagThresholdResolver.Resolve( + accepted, inputs.KnownServerIds, inputs.TagParents, inputs.ServerTags, inputs.TagsReadable)) + tagWinners[slot] = (int)winner.Value; } - _snapshot = next; + _snapshot = new Snapshot(direct, tagWinners); } private static string? Accept(string key, double raw, string database, out int value) @@ -106,18 +142,21 @@ public int Resolve(AlertThresholdKey key, string serverKey, string? database, in /// /// The value of the wire-named threshold for the server (and database): a database override first, then the - /// server's, else . is the store server id as text, the - /// engine's server key. This is the one resolution the PostgreSQL count thresholds use too. + /// server's, then the tag's (#6926: the deepest tag the server is under), else . + /// is the store server id as text, the engine's server key. This is the one resolution the + /// PostgreSQL count thresholds use too. /// internal int ResolveWire(string wire, string serverKey, string? database, int globalValue) { if (!int.TryParse(serverKey, NumberStyles.Integer, CultureInfo.InvariantCulture, out var serverId)) return globalValue; var snapshot = _snapshot; - if (snapshot.Count == 0) + if (snapshot.Direct.Count == 0 && snapshot.Tag.Count == 0) return globalValue; - if (!string.IsNullOrEmpty(database) && snapshot.TryGetValue((serverId, database, wire), out var perDatabase)) + if (!string.IsNullOrEmpty(database) && snapshot.Direct.TryGetValue((serverId, database, wire), out var perDatabase)) return perDatabase; - return snapshot.TryGetValue((serverId, string.Empty, wire), out var perServer) ? perServer : globalValue; + if (snapshot.Direct.TryGetValue((serverId, string.Empty, wire), out var perServer)) + return perServer; + return snapshot.Tag.TryGetValue((serverId, wire), out var perTag) ? perTag : globalValue; } } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs index 9c15d750b5..d8cc0d3eec 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs @@ -42,8 +42,9 @@ internal static string RefusedMessage(bool managedStore, bool adminWritesOff = f + "Darling/tools/provision-roles.sql re-run since the upgrade that added threshold overrides: run it once as a superuser (it is idempotent).", adminWritesOff); - /// The list body: {overrides: [{server_id, server_name, database, key, value, global_value, set_by, request_id, - /// updated_at}], keys: [{key, min, max, database_scope}], servers: [{server_id, server_name}], can_edit, note}. + /// The list body: {overrides: [{server_id, server_name, tag_id, tag_name, orphaned, database, key, value, global_value, set_by, + /// request_id, updated_at}], keys: [{key, min, max, database_scope}], servers: [{server_id, server_name}], tags: [{tag_id, tag_name}], + /// can_edit, note}. A tag override (#6926) has a tag_id, a null server_id and server_name, and orphaned true when its tag was deleted. /// can_edit is whether the store login holds the write grant, so a page can say list-only before anyone tries. internal static async Task ListBodyAsync(NpgsqlDataSource postgres, CancellationToken ct, NpgsqlDataSource? accessSource = null) { @@ -56,8 +57,11 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, { overrides.Add(new JsonObject { - ["server_id"] = r.ServerId, - ["server_name"] = r.ServerName, + ["server_id"] = r.IsTag ? null : r.ServerId, + ["server_name"] = r.IsTag ? null : r.ServerName, + ["tag_id"] = r.TagId, + ["tag_name"] = r.IsTag && !r.Orphaned ? r.ServerName : null, + ["orphaned"] = r.Orphaned, ["database"] = r.Database, ["key"] = r.Key, ["value"] = r.Value, @@ -95,11 +99,21 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, servers.Add(new JsonObject { ["server_id"] = reader.GetInt32(0), ["server_name"] = reader.GetString(1) }); } + var tags = new JsonArray(); + await using (var command = postgres.CreateCommand("SELECT id, name FROM config.server_tags ORDER BY name, id")) + { + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + tags.Add(new JsonObject { ["tag_id"] = reader.GetInt32(0), ["tag_name"] = reader.GetString(1) }); + } + return new JsonObject { ["overrides"] = overrides, ["keys"] = keys, ["servers"] = servers, + ["tags"] = tags, ["can_edit"] = await DarlingMcpThresholdOverrideTools.HoldsGrantAsync(accessSource ?? postgres, ct), ["note"] = note, }; @@ -116,6 +130,7 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, var server = body["server_name"]?.GetValueKind() == JsonValueKind.String ? body["server_name"]!.GetValue() : null; var key = body["key"]?.GetValueKind() == JsonValueKind.String ? body["key"]!.GetValue() : null; var database = body["database_name"]?.GetValueKind() == JsonValueKind.String ? body["database_name"]!.GetValue() : null; + var tag = body["tag"]?.GetValueKind() == JsonValueKind.String ? body["tag"]!.GetValue() : null; var clear = body["clear"]?.GetValueKind() == JsonValueKind.True; int? value = null; @@ -126,13 +141,13 @@ internal static async Task ListBodyAsync(NpgsqlDataSource postgres, value = parsed; } - if (string.IsNullOrWhiteSpace(server)) - return (400, new JsonObject { ["error"] = "server_name is required." }); + if (string.IsNullOrWhiteSpace(server) && string.IsNullOrWhiteSpace(tag)) + return (400, new JsonObject { ["error"] = "server_name or tag is required." }); var json = await DarlingMcpThresholdOverrideTools.SetAsync( postgres, new DarlingMcpWritePolicy.Effective(true, false), new InvestigationRequestStore(postgres, McpCommandDeadlines.ReadSeconds), logger, - server, key ?? string.Empty, value, database, clear, ThresholdOverrideStore.SetByFor(editor), null, ct); + server, key ?? string.Empty, value, database, clear, ThresholdOverrideStore.SetByFor(editor), null, ct, tag: tag); var answer = JsonNode.Parse(json)!.AsObject(); var status = answer["status"]?.GetValue(); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs index 521ad6b075..edab71bae2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.ComponentModel; using System.Globalization; +using System.Linq; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; @@ -67,12 +68,14 @@ public sealed class DarlingMcpThresholdOverrideTools "runs again) as a superuser. Managed and compose stores grant it themselves on start. Call again after the script has run."; [McpServerTool(Name = "set_alert_threshold_override"), Description( - "Sets or clears a threshold for ONE server (or one database on it), leaving the global value alone. Changes the " + + "Sets or clears a threshold for ONE server (or one database on it), or for a server TAG, leaving the global value alone. Changes the " + "monitoring store only; the service picks it up within its reload interval. <> " + "key is a tunable integer threshold named as get_alert_settings names it (for example cpu.threshold_percent, " + "blocking.pg_count_threshold); get_alert_settings with server_name lists the keys, the global value, the effective value " + "and where it comes from. value is an integer inside the same bounds the global setting has. clear: true removes the " + - "override so the global value applies again. database_name applies only to pvs.* and file_growth.rise_mb / " + + "override so the global value applies again. tag (a tag name or id, in place of server_name) holds the value for every server under that tag " + + "and every tag below it: the writes arm only, and no database_name. Precedence is database, then server, then tag (the deepest tag a server " + + "is under, as a notification route resolves it, the older override winning a same-depth tie), then global. database_name applies only to pvs.* and file_growth.rise_mb / " + "file_growth.volume_percent. Refused: a key outside that list, a self_alerts.*, enable, cooldown or route key, an " + "out-of-bounds value. To silence an alert, mute it. An agent cannot set a threshold to the value that turns its alert off (the top of the " + "key's range, or zero for the low_disk.* keys); that value is refused as threshold_disables_alert. A very high count below " + @@ -85,23 +88,24 @@ public static async Task SetAlertThresholdOverride( NpgsqlDataSource postgres, DarlingMcpServedPolicy served, ILogger logger, - [Description("Server name, or the start of it.")] string server_name, - [Description("A tunable threshold, as get_alert_settings names it, e.g. cpu.threshold_percent.")] string key, + [Description("Server name, or the start of it. Omit when tag is given.")] string? server_name = null, + [Description("A tunable threshold, as get_alert_settings names it, e.g. cpu.threshold_percent.")] string key = "", [Description("The override value (integer).")] int? value = null, [Description("Database on that server (pvs.*, file_growth.rise_mb, file_growth.volume_percent). Omit for the whole server.")] string? database_name = null, [Description("true removes the override.")] bool clear = false, [Description("Who made the change (writes arm). 1-64 of A-Za-z0-9._:-. Default mcp.")] string? set_by = null, [Description("Investigations arm only: the agent_action object (request_id, claim_id, agent_id, reason, evidence). Ignored under the writes arm.")] string? agent_action = null, + [Description("A server tag, by name or id, in place of server_name: the value applies to every server under the tag and the tags below it (#6926). Writes arm only; no database_name.")] string? tag = null, StoreOwnerSource? ownerSource = null, CancellationToken cancellationToken = default) => await SetAsync(postgres, served.Policy, new InvestigationRequestStore(postgres, ServiceCommandDeadlines.SerialLoopSeconds, agentActionSource: ownerSource?.Read()), - logger, server_name, key, value, database_name, clear, set_by, agent_action, cancellationToken, served.NamedAgentNames); + logger, server_name, key, value, database_name, clear, set_by, agent_action, cancellationToken, served.NamedAgentNames, tag); /// The tool body over a given store, so a test drives either arm. internal static async Task SetAsync( NpgsqlDataSource postgres, DarlingMcpWritePolicy.Effective policy, InvestigationRequestStore store, ILogger logger, - string server_name, string key, int? value, string? database_name, bool clear, string? set_by, string? agent_action, - CancellationToken cancellationToken, IReadOnlySet? namedAgentNames = null) + string? server_name, string key, int? value, string? database_name, bool clear, string? set_by, string? agent_action, + CancellationToken cancellationToken, IReadOnlySet? namedAgentNames = null, string? tag = null) { try { @@ -110,6 +114,15 @@ internal static async Task SetAsync( DarlingThresholdKeys.TryGet(key, out var entry); var wire = entry.Wire; var database = string.IsNullOrWhiteSpace(database_name) ? null : database_name.Trim(); + var hasTag = !string.IsNullOrWhiteSpace(tag); + if (hasTag && !string.IsNullOrWhiteSpace(server_name)) + return Refused("scope_conflict", "Give a server_name or a tag, not both."); + if (!hasTag && string.IsNullOrWhiteSpace(server_name)) + return Refused("scope_required", "server_name (or tag) is required."); + if (hasTag && database is not null) + return Refused("database_scope", "A tag override holds a server-level value, so it takes no database_name."); + if (hasTag && !policy.Writes) + return Refused("tag_writes_arm_only", "A tag override changes every server under the tag, so it is set on the writes arm, not by an investigation request's agent, whose change is for the request's server only."); if (!clear) { @@ -131,7 +144,10 @@ internal static async Task SetAsync( return Refused("set_by_invalid", "set_by must be 1-64 of A-Za-z0-9._:-."); } - var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name, cancellationToken, NameClash.Refuse); + if (hasTag) + return await SetTagAsync(postgres, tag!, wire, value, clear, person!, cancellationToken); + + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name!, cancellationToken, NameClash.Refuse); if (error is not null) return error; @@ -225,6 +241,52 @@ internal static async Task SetAsync( } } + /// The tag branch of (#6926), writes arm only: resolve the tag, then set or clear its value in one transaction. + /// A tag with no servers under it is allowed (servers are tagged later and take the value then). + private static async Task SetTagAsync( + NpgsqlDataSource postgres, string tagText, string wire, int? value, bool clear, string by, CancellationToken cancellationToken) + { + var (tagId, tagName, problem) = await ThresholdOverrideStore.ResolveTagAsync(postgres, tagText, cancellationToken); + /* A clear by id still works for an override whose tag was deleted (it is listed as orphaned): the row is found by its tag id below. */ + var orphanClear = problem is not null && clear + && int.TryParse(tagText.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var orphanId) && (tagId = orphanId) is not null; + if (problem is not null && !orphanClear) + return Refused("tag_not_found", problem); + if (!await HoldsGrantAsync(postgres, cancellationToken)) + return McpHelpers.Status("precondition", PreconditionMessage); + + double? previous; + await using (var connection = await postgres.OpenConnectionAsync(cancellationToken)) + await using (var transaction = await connection.BeginTransactionAsync(cancellationToken)) + { + previous = await ThresholdOverrideStore.ReadTagValueAsync(connection, transaction, tagId!.Value, wire, cancellationToken); + if (orphanClear && previous is null) + return Refused("tag_not_found", problem!); + if (clear) + await ThresholdOverrideStore.DeleteTagAsync(connection, transaction, tagId.Value, wire, cancellationToken); + else + await ThresholdOverrideStore.UpsertTagAsync(connection, transaction, tagId.Value, wire, value!.Value, by, null, cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + + return JsonSerializer.Serialize(new + { + status = clear ? "cleared" : "ok", + tag = tagName, + tag_id = tagId, + key = wire, + value = clear ? (int?)null : value, + previous_override = previous, + undo = DarlingMcpAlertTools.UndoCall(ToolName, previous is { } was + ? new System.Collections.Generic.Dictionary { ["tag"] = tagId!.Value.ToString(CultureInfo.InvariantCulture), ["key"] = wire, ["value"] = was } + : new System.Collections.Generic.Dictionary { ["tag"] = tagId!.Value.ToString(CultureInfo.InvariantCulture), ["key"] = wire, ["clear"] = true }), + set_by = by, + message = clear + ? "The tag override is removed; servers under the tag fall back to a shallower tag, or the global value." + : "The tag override is stored. The service applies it within its reload interval to every server under the tag that has no server or database override of its own." + }, McpHelpers.JsonOptions); + } + private sealed class Applied { internal double? Previous; @@ -264,8 +326,8 @@ internal static async Task HoldsGrantAsync(NpgsqlDataSource postgres, Canc /// /// The get_alert_settings view for one server (#5916): every tunable threshold with its global value, the value in - /// effect for this server and where that value comes from (global or server_override), and the database - /// overrides the server holds. A read of the overrides table, so it answers what the service will apply after its next reload. + /// effect for this server and where that value comes from (global, tag_override naming the tag, or server_override), + /// and the database overrides the server holds. A read of the overrides table, so it answers what the service will apply after its next reload. /// internal static async Task BuildServerViewAsync( NpgsqlDataSource postgres, DarlingAlertReader.AlertSettingsReadRow s, int serverId, string serverName, @@ -291,6 +353,56 @@ internal static async Task BuildServerViewAsync( note = PreconditionMessage; } + /* #6926: the tag overrides that reach this server, resolved by the notification routes' own tag rule (TagThresholdResolver), the same + one the service loads. A tag read that fails reaches no server: the view then shows what the engine would apply. */ + var tagWinners = new System.Collections.Generic.Dictionary(); + try + { + var candidates = new System.Collections.Generic.List(); + var meta = new System.Collections.Generic.Dictionary<(int Tag, string Key), (string SetBy, DateTime UpdatedAt)>(); + await using (var command = postgres.CreateCommand( + "SELECT id, scope_tag_id, threshold_key, value, set_by, updated_at FROM " + Table + " WHERE scope_tag_id IS NOT NULL")) + { + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var key = reader.GetString(2); + var raw = reader.GetDouble(3); + if (!DarlingThresholdKeys.TryGet(key, out _) || raw != Math.Floor(raw) || raw < int.MinValue || raw > int.MaxValue) + continue; + candidates.Add(new TagOverrideCandidate(reader.GetInt64(0), reader.GetInt32(1), key, raw)); + meta[(reader.GetInt32(1), key)] = (reader.GetString(4), reader.GetDateTime(5)); + } + } + + if (candidates.Count > 0) + { + await using var connection = await postgres.OpenConnectionAsync(cancellationToken); + var inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, null, cancellationToken); + var names = new System.Collections.Generic.Dictionary(); + await using (var command = new NpgsqlCommand("SELECT id, name FROM config.server_tags", connection) { CommandTimeout = McpCommandDeadlines.ReadSeconds }) + await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) + { + while (await reader.ReadAsync(cancellationToken)) + names[reader.GetInt32(0)] = reader.GetString(1); + } + + foreach (var ((server, key), winner) in TagThresholdResolver.Resolve( + candidates, inputs.KnownServerIds, inputs.TagParents, inputs.ServerTags, inputs.TagsReadable)) + { + if (server != serverId) + continue; + var m = meta[(winner.TagId, key)]; + tagWinners[key] = (winner.TagId, names.GetValueOrDefault(winner.TagId, string.Empty), winner.Value, m.SetBy, m.UpdatedAt); + } + } + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.InsufficientPrivilege) + { + note ??= PreconditionMessage; + } + /* One number per key, read the way the engine reads it (#5916): the global value and the value with this server's override applied each go through the same pair clamp the enclosing get_alert_settings payload applies, so an override that inverts a warning/critical pair shows the pair the engine evaluates, not the stored one. */ @@ -305,19 +417,25 @@ internal static async Task BuildServerViewAsync( return null; } + int? TagOverride(string wire) => tagWinners.TryGetValue(wire, out var t) ? (int)t.Value : null; + var thresholds = new System.Collections.Generic.List(); foreach (var entry in DarlingThresholdKeys.Entries) { var global = GlobalValueAsEngineReadsIt(s, entry.Wire); var server = ServerOverride(entry.Wire); - var effective = Evaluated(entry.Wire, w => ServerOverride(w) ?? GlobalValue(s, w)); + var effective = Evaluated(entry.Wire, w => ServerOverride(w) ?? TagOverride(w) ?? GlobalValue(s, w)); + var found = tagWinners.TryGetValue(entry.Wire, out var tw); + var tagHit = server is null && found; thresholds.Add(new { key = entry.Wire, global_value = global, effective_value = effective, - source = server is null ? "global" : "server_override", + source = server is not null ? "server_override" : tagHit ? "tag_override" : "global", + tag = tagHit ? tw.TagName : null, + tag_id = tagHit ? tw.TagId : (int?)null, database_scope = entry.Scope == PerformanceMonitor.Alerting.AlertThresholdScope.ServerAndDatabase }); } @@ -338,10 +456,15 @@ internal static async Task BuildServerViewAsync( { server = serverName, server_id = serverId, - note = note ?? "effective_value is the global value with this server's override applied; a database override (database_overrides) applies to that database only. Set one with set_alert_threshold_override.", + note = note ?? "effective_value is the global value with this server's override applied, else the deepest server tag's (source tag_override, naming the tag; tag_overrides lists the ones that reach this server), else global; a database override (database_overrides) applies to that database only. Set one with set_alert_threshold_override.", thresholds, server_overrides = serverOverrides, - database_overrides = databaseOverrides + database_overrides = databaseOverrides, + tag_overrides = tagWinners.Select(t => (object)new + { + key = t.Key, tag = t.Value.TagName, tag_id = t.Value.TagId, value = t.Value.Value, set_by = t.Value.SetBy, + updated_at = t.Value.UpdatedAt.ToString("o", CultureInfo.InvariantCulture) + }).ToList() }; } diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js index 0abc05db92..ee27ef9ebf 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js @@ -1807,12 +1807,12 @@ async function buildPending(body, generation) { /* ─────────────────────────── threshold overrides (#5916) ─────────────────────────── */ -/* A threshold held for one server (or one database on it) that differs from the global value. The list is GET +/* A threshold held for one server (or one database on it), or for a server tag (#6926), that differs from the global value. The list is GET /api/threshold-overrides; a change is PUT /api/threshold-overrides, accepted only under a store login with the admin role. A managed or compose store connects this page as the viewer role, so there the grid is a list: the form and the Remove buttons are not drawn, and the note says where to change one. The service checks the key and the bounds again. */ export const OVERRIDE_COLUMNS = [ - { key: "server_name", label: "Server" }, + { key: "scope_text", label: "Scope" }, { key: "database_text", label: "Database" }, { key: "key", label: "Key" }, { key: "value", label: "Value" }, @@ -1823,10 +1823,10 @@ export const OVERRIDE_COLUMNS = [ ]; const OVERRIDES_NOTE_EDITING = - "A threshold held for one server, or for one database on it, that differs from the global value. The alert engine uses it for that server only " + - "and applies a change within its reload interval. To silence an alert, mute it."; + "A threshold held for one server, for one database on it, or for a server tag (every server under the tag and the tags below it), that differs from the global value. " + + "A database override wins over a server override, then the deepest tag the server is under, then the global value. A change applies within the alert engine's reload interval. To silence an alert, mute it."; const overridesNoteReadOnly = (managed, adminWritesOff) => - "A threshold held for one server, or for one database on it, that differs from the global value. " + + "A threshold held for one server, for one database on it, or for a server tag, that differs from the global value. " + listOnlyNote(managed, "this page lists them only", "To change an override", adminWritesOff); const OVERRIDE_DENIED = "Changing a threshold override needs"; @@ -1834,8 +1834,12 @@ const OVERRIDE_DENIED = "Changing a threshold override needs"; export function overrideRows(data) { const list = data && Array.isArray(data.overrides) ? data.overrides : []; return list.map((o) => ({ - server_id: o.server_id, - server_name: o.server_name, + server_id: o.server_id == null ? null : o.server_id, + server_name: o.server_name == null ? null : o.server_name, + tag_id: o.tag_id == null ? null : o.tag_id, + tag_name: o.tag_name == null ? null : o.tag_name, + orphaned: o.orphaned === true, + scope_text: o.tag_id == null ? o.server_name : (o.orphaned === true ? "Tag (deleted): orphaned, matches nothing" : "Tag: " + (o.tag_name || "")), database: o.database || null, database_text: o.database || "Whole server", key: o.key, @@ -1851,7 +1855,10 @@ export function overrideRows(data) { inside its bounds, and a database only on a key that takes one. The service applies the same rules again. */ export function validateOverride(keys, form) { const list = Array.isArray(keys) ? keys : []; - if (!form || !String(form.server || "").trim()) return "Choose a server."; + const hasServer = !!(form && String(form.server || "").trim()); + const hasTag = !!(form && String(form.tag || "").trim()); + if (hasServer && hasTag) return "Choose a server or a tag, not both."; + if (!hasServer && !hasTag) return "Choose a server or a tag."; const entry = list.find((k) => k.key === String(form.key || "").trim()); if (!entry) return "Choose a key from the list: only a tunable threshold can be overridden."; const text = String(form.value == null ? "" : form.value).trim(); @@ -1860,6 +1867,9 @@ export function validateOverride(keys, form) { if (!Number.isSafeInteger(n) || n < entry.min || n > entry.max) { return entry.key + " must be a whole number between " + entry.min + " and " + entry.max + ", the same bounds the global setting has."; } + if (hasTag && String(form.database || "").trim()) { + return "A tag override holds a server-level value, so it takes no database."; + } if (String(form.database || "").trim() && entry.database_scope !== true) { return entry.key + " is compared against a server-level figure, so it takes a server override only, not a database."; } @@ -1887,7 +1897,7 @@ export function overrideOutcome(verb, res) { } /* The form's entries and the last answer, in module scope so the 60 s repaint keeps what was typed. */ -const EMPTY_OVERRIDE_FORM = { server: "", key: "", database: "", value: "" }; +const EMPTY_OVERRIDE_FORM = { server: "", tag: "", key: "", database: "", value: "" }; let overrideEntries = { ...EMPTY_OVERRIDE_FORM }; let overrideStatus = null; let overrideBusy = false; @@ -1903,7 +1913,9 @@ async function changeOverride(verb, entry) { overrideBusy = true; let out; try { - const payload = { server_name: entry.server, key: entry.key, database_name: entry.database || null }; + const payload = { key: entry.key, database_name: entry.database || null }; + if (entry.tag) payload.tag = String(entry.tag); + else payload.server_name = entry.server; if (verb === "clear") payload.clear = true; else payload.value = Number(String(entry.value).trim()); out = overrideOutcome(verb, await apiWrite("PUT", "/api/threshold-overrides", payload)); @@ -1916,19 +1928,22 @@ async function changeOverride(verb, entry) { await redrawOverrides(); } -function overrideFormNode(servers, keys) { +function overrideFormNode(servers, keys, tags) { const field = (label, control) => el("label", { class: "tag-field" }, [el("span", { class: "mute-label", text: label }), control]); const choose = (items, textOf) => [el("option", { value: "", text: "(choose)" })].concat(items.map((i) => el("option", { value: textOf(i), text: textOf(i) }))); const server = el("select", { class: "tag-input", "data-field": "server" }, choose(servers, (s) => s.server_name)); + const tag = el("select", { class: "tag-input", "data-field": "tag" }, + [el("option", { value: "", text: "(none)" })].concat((Array.isArray(tags) ? tags : []).map((t) => el("option", { value: String(t.tag_id), text: t.tag_name })))); const key = el("select", { class: "tag-input", "data-field": "key" }, choose(keys, (k) => k.key)); const database = el("input", { type: "text", class: "tag-input", "data-field": "database", placeholder: "blank = whole server" }); const value = el("input", { type: "text", class: "tag-input", "data-field": "value", inputmode: "numeric" }); server.value = overrideEntries.server; + tag.value = overrideEntries.tag; key.value = overrideEntries.key; database.value = overrideEntries.database; value.value = overrideEntries.value; - const sync = () => { overrideEntries = { server: server.value, key: key.value, database: database.value, value: value.value }; }; - for (const c of [server, key, database, value]) c.addEventListener("input", sync); + const sync = () => { overrideEntries = { server: server.value, tag: tag.value, key: key.value, database: database.value, value: value.value }; }; + for (const c of [server, tag, key, database, value]) c.addEventListener("input", sync); const save = el("button", { class: "btn primary", type: "button", text: "Save override", "data-action": "save-override", onClick: async () => { @@ -1943,21 +1958,21 @@ function overrideFormNode(servers, keys) { }, }); return el("div", { class: "tag-form", "data-override-form": "1" }, [ - field("Server:", server), field("Key:", key), field("Database:", database), field("Value:", value), save, + field("Server:", server), field("or Tag:", tag), field("Key:", key), field("Database:", database), field("Value:", value), save, ]); } function overrideActions(row) { const edit = el("button", { - class: "btn", type: "button", text: "Edit", "aria-label": "Edit " + row.key + " for " + row.server_name, + class: "btn", type: "button", text: "Edit", "aria-label": "Edit " + row.key + " for " + row.scope_text, onClick: async () => { - overrideEntries = { server: row.server_name, key: row.key, database: row.database || "", value: String(row.value) }; + overrideEntries = { server: row.tag_id == null ? row.server_name : "", tag: row.tag_id == null ? "" : String(row.tag_id), key: row.key, database: row.database || "", value: String(row.value) }; await redrawOverrides(); }, }); const remove = el("button", { - class: "btn", type: "button", text: "Remove", "aria-label": "Remove " + row.key + " for " + row.server_name, "data-action": "remove-override", - onClick: () => changeOverride("clear", { server: row.server_name, key: row.key, database: row.database }), + class: "btn", type: "button", text: "Remove", "aria-label": "Remove " + row.key + " for " + row.scope_text, "data-action": "remove-override", + onClick: () => changeOverride("clear", { server: row.tag_id == null ? row.server_name : "", tag: row.tag_id == null ? "" : String(row.tag_id), key: row.key, database: row.database }), }); return el("span", { class: "row-actions" }, [edit, remove]); } @@ -1981,11 +1996,11 @@ async function buildOverrides(body, generation) { const keys = Array.isArray(res.data && res.data.keys) ? res.data.keys : []; nodes.push(noticeStrip(canEdit ? OVERRIDES_NOTE_EDITING : overridesNoteReadOnly(session.managed_store === true, session.admin_writes_off === true))); if (res.data && typeof res.data.note === "string" && res.data.note) nodes.push(noticeStrip(res.data.note)); - if (canEdit) nodes.push(overrideFormNode(Array.isArray(res.data.servers) ? res.data.servers : [], keys)); + if (canEdit) nodes.push(overrideFormNode(Array.isArray(res.data.servers) ? res.data.servers : [], keys, res.data.tags)); nodes.push(VIZ.table({ overrides: overrideRows(res.data) }, { rowsKey: "overrides", columns: overrideColumns(canEdit), - emptyText: "No server holds an override: every server uses the global values.", + emptyText: "No server or tag holds an override: every server uses the global values.", })); mount(body, nodes); } diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index c65424e31c..8ca7eb987e 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -2975,8 +2975,52 @@ holds the ordering. */ new Migration(224, "identity-capacity", V224Sql), new Migration(225, "server-aws-rds", V225Sql), new Migration(226, "server-aws-rds-web-or-express", V226Sql), + new Migration(227, "threshold-override-tag-scope", V227Sql), }; + /// + /// V227 (#6926): a threshold override can be held for a server tag. config.config_threshold_overrides gains the nullable + /// scope_tag_id (a config.server_tags id), server_id becomes nullable, and a CHECK says exactly one of the two is set, + /// so a stored row (server set, tag NULL) keeps its meaning and stays valid. A tag row holds no database (CHECK), because a tag value is + /// the server-level value for every server under the tag. A second unique index keeps one value per tag and key, since the V191 index is + /// on server_id and NULLs never collide. + /// + /// No foreign key on the tag, deliberately, as V181 did for a route: an override whose tag was deleted stays as written and + /// shows as orphaned, and the service resolves it to nothing. A cascade would delete the person's setting, and SET NULL would break the + /// exactly-one CHECK. The table's reload trigger (V191) and the tag tables' (V181) already bump the config version, so the service + /// re-reads when a tag moves. No GRANT: the table-level grants of V191 (admin and mcp write, viewer read) cover the new column. Every + /// statement is idempotent. No Lite twin: Lite has no overrides. + /// + private const string V227Sql = @" +ALTER TABLE config.config_threshold_overrides + ADD COLUMN IF NOT EXISTS scope_tag_id integer; + +ALTER TABLE config.config_threshold_overrides + ALTER COLUMN server_id DROP NOT NULL; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ck_config_threshold_overrides_one_scope' + AND conrelid = 'config.config_threshold_overrides'::regclass) THEN + ALTER TABLE config.config_threshold_overrides + ADD CONSTRAINT ck_config_threshold_overrides_one_scope + CHECK ((server_id IS NOT NULL) <> (scope_tag_id IS NOT NULL)); + END IF; + + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conname = 'ck_config_threshold_overrides_tag_no_database' + AND conrelid = 'config.config_threshold_overrides'::regclass) THEN + ALTER TABLE config.config_threshold_overrides + ADD CONSTRAINT ck_config_threshold_overrides_tag_no_database + CHECK (scope_tag_id IS NULL OR database_name IS NULL); + END IF; +END $$; + +CREATE UNIQUE INDEX IF NOT EXISTS ux_config_threshold_overrides_tag + ON config.config_threshold_overrides (scope_tag_id, threshold_key) + WHERE scope_tag_id IS NOT NULL;"; + /// /// V225 (#6894): collect.servers.is_aws_rds, whether the target is an Amazon RDS for SQL Server instance (the connect query's /// DB_ID('rdsadmin') IS NOT NULL), written on every connect. The collectors that RDS gates off (running jobs, Agent status, Agent alerts) diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs index bfebc69b62..e669ce62e7 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs @@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class StorageVersion { - public const int SchemaVersion = 226; + public const int SchemaVersion = 227; } diff --git a/Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs b/Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs new file mode 100644 index 0000000000..2a6f88ae5c --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/TagThresholdResolver.cs @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using PerformanceMonitor.Notifications; + +namespace PerformanceMonitor.Darling.Storage; + +/// One stored tag override: the table row's id, its tag, the wire key and the value. +public sealed record TagOverrideCandidate(long RowId, int TagId, string Key, double Value); + +/// The tag override that applies to a server for one key: the tag it comes from and its value. +public sealed record TagOverrideWinner(int TagId, double Value); + +/// +/// Which tag override applies to a server (#6926). It is the notification routes' rule, run by the routes' own code: each tag that +/// holds an override is expanded by exactly as a tag-scoped route is (a tag covers the servers +/// mapped to it or to any tag below it, a tag that no longer exists is orphaned and covers nothing, and tag tables that could not be read +/// cover nothing and are not called orphaned). When a server is covered by several tags that hold the same key, the deepest tag wins +/// (, the number of ancestors) and a tie goes to the lowest row id, the way the router's +/// ordering takes the lowest route id after depth. Pure: the service, the MCP view and the tests share this one resolution. +/// +public static class TagThresholdResolver +{ + /// The winning tag override per (server, key) for every server some tag covers. + public static IReadOnlyDictionary<(int ServerId, string Key), TagOverrideWinner> Resolve( + IReadOnlyCollection rows, + IReadOnlyCollection knownServerIds, + IReadOnlyDictionary tagParents, + IReadOnlyDictionary> serverTags, + bool tagsReadable = true) + { + ArgumentNullException.ThrowIfNull(rows); + if (rows.Count == 0) + return new Dictionary<(int ServerId, string Key), TagOverrideWinner>(); + + var tagIds = rows.Select(r => r.TagId).Distinct().ToList(); + var routes = tagIds + .Select(t => new NotificationRoute(t, NotificationRoute.WildcardMatch, "", "", "", "", "", true) { ScopeTagId = t }) + .ToList(); + var expanded = NotificationRouteScopeExpander.Expand(routes, knownServerIds, tagParents, serverTags, tagsReadable); + var byTag = new Dictionary(); + for (var i = 0; i < tagIds.Count; i++) + byTag[tagIds[i]] = expanded[i]; + + var winners = new Dictionary<(int ServerId, string Key), (int Depth, long RowId, TagOverrideWinner Winner)>(); + foreach (var row in rows) + { + var route = byTag[row.TagId]; + if (route.Orphaned || route.ExpandedServerIds is null) + continue; + foreach (var serverId in route.ExpandedServerIds) + { + var slot = (serverId, row.Key); + if (winners.TryGetValue(slot, out var current) + && (current.Depth > route.ScopeTagDepth || (current.Depth == route.ScopeTagDepth && current.RowId <= row.RowId))) + continue; + winners[slot] = (route.ScopeTagDepth, row.RowId, new TagOverrideWinner(row.TagId, row.Value)); + } + } + + return winners.ToDictionary(p => p.Key, p => p.Value.Winner); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs new file mode 100644 index 0000000000..b4d9a5ab65 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.TagScope.cs @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using NpgsqlTypes; + +namespace PerformanceMonitor.Darling.Storage; + +/// The tag-scoped statements of (#6926): a threshold held for a server tag, the deepest tag +/// a server is under winning as a notification route's does (). +public static partial class ThresholdOverrideStore +{ + private static NpgsqlCommand TagCommand(NpgsqlConnection connection, NpgsqlTransaction transaction, string sql, int tagId, string wire) + { + var command = new NpgsqlCommand(sql, connection) { Transaction = transaction, CommandTimeout = CommandTimeoutSeconds }; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = tagId }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = wire }); + return command; + } + + /// The tag's stored value for a key, or null. + public static async Task ReadTagValueAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, int tagId, string wire, CancellationToken ct) + { + await using var command = TagCommand(connection, transaction, + "SELECT value FROM " + Table + " WHERE scope_tag_id = $1 AND threshold_key = $2", tagId, wire); + return await command.ExecuteScalarAsync(ct) is double d ? d : null; + } + + public static async Task DeleteTagAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, int tagId, string wire, CancellationToken ct) + { + await using var command = TagCommand(connection, transaction, + "DELETE FROM " + Table + " WHERE scope_tag_id = $1 AND threshold_key = $2", tagId, wire); + await command.ExecuteNonQueryAsync(ct); + } + + public static async Task UpsertTagAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, int tagId, string wire, int value, + string setBy, Guid? requestId, CancellationToken ct) + { + await using var command = TagCommand(connection, transaction, + "INSERT INTO " + Table + " (scope_tag_id, threshold_key, value, set_by, request_id) VALUES ($1, $2, $3, $4, $5) " + + "ON CONFLICT (scope_tag_id, threshold_key) WHERE scope_tag_id IS NOT NULL DO UPDATE " + + "SET value = EXCLUDED.value, set_by = EXCLUDED.set_by, request_id = EXCLUDED.request_id, updated_at = now()", + tagId, wire); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Double, Value = (double)value }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = setBy }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Uuid, Value = (object?)requestId ?? DBNull.Value }); + await command.ExecuteNonQueryAsync(ct); + } + + /// Finds a tag from the text a person or an agent gave: a number is a tag id, anything else is a name matched without regard + /// to case. A name two tags share, or one that matches nothing, comes back as a refusal sentence. + public static async Task<(int? TagId, string? Name, string? Problem)> ResolveTagAsync(NpgsqlDataSource postgres, string text, CancellationToken ct) + { + var wanted = text.Trim(); + if (wanted.Length == 0) + return (null, null, "A tag name or id is required."); + var matches = new List<(int Id, string Name)>(); + await using var command = postgres.CreateCommand( + "SELECT id, name FROM config.server_tags WHERE id::text = $1 OR lower(name) = lower($1) ORDER BY id"); + command.CommandTimeout = CommandTimeoutSeconds; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = wanted }); + await using (var reader = await command.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + matches.Add((reader.GetInt32(0), reader.GetString(1))); + } + + var byId = matches.Where(m => m.Id.ToString(CultureInfo.InvariantCulture) == wanted).ToList(); + if (byId.Count == 1) + return (byId[0].Id, byId[0].Name, null); + if (matches.Count == 1) + return (matches[0].Id, matches[0].Name, null); + return matches.Count == 0 + ? (null, null, $"No server tag is named or numbered '{wanted}'.") + : (null, null, $"More than one server tag is named '{wanted}' (ids {string.Join(", ", matches.Select(m => m.Id))}): use the tag id."); + } + + /// The tag list statement: every tag override with its tag's name. A tag that was deleted has no name and is orphaned. + public const string TagListSql = + "SELECT o.scope_tag_id, t.name, o.threshold_key, o.value, o.set_by, o.request_id, o.updated_at " + + "FROM " + Table + " AS o LEFT JOIN config.server_tags AS t ON t.id = o.scope_tag_id " + + "WHERE o.scope_tag_id IS NOT NULL ORDER BY t.name NULLS LAST, o.scope_tag_id, o.threshold_key"; +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs index 3d11d4b0b6..9eb8a4584a 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/ThresholdOverrideStore.cs @@ -20,7 +20,12 @@ namespace PerformanceMonitor.Darling.Storage; /// override replaces (null when the store has no global row or no such column yet). public sealed record ThresholdOverrideRow( int ServerId, string ServerName, string? Database, string Key, double Value, double? GlobalValue, - string SetBy, Guid? RequestId, DateTime UpdatedAt); + string SetBy, Guid? RequestId, DateTime UpdatedAt, int? TagId = null, bool Orphaned = false) +{ + /// True for a row held for a tag (#6926): is set, is 0, + /// holds the tag's name (empty when the tag was deleted) and says the tag no longer exists, so the row matches nothing. + public bool IsTag => TagId.HasValue; +} /// /// The statements behind config.config_threshold_overrides (#5916), shared by the set_alert_threshold_override tool, @@ -28,7 +33,7 @@ public sealed record ThresholdOverrideRow( /// and transaction (the tool journals its write in the same transaction) or a data source (the list). Key and bound checks are /// not here: they are DarlingThresholdKeys' (PerformanceMonitor.Alerting), applied by every caller before any write. /// -public static class ThresholdOverrideStore +public static partial class ThresholdOverrideStore { /// The table, schema-qualified; the mcp role holds SELECT, INSERT, UPDATE and DELETE on it (#5920). public const string Table = "config.config_threshold_overrides"; @@ -141,6 +146,28 @@ public static async Task> ListAsync( reader.GetString(5), reader.IsDBNull(6) ? null : reader.GetGuid(6), reader.GetDateTime(7))); } + await reader.DisposeAsync(); + + await using var tagCommand = postgres.CreateCommand(TagListSql); + tagCommand.CommandTimeout = CommandTimeoutSeconds; + await using var tagReader = await tagCommand.ExecuteReaderAsync(ct); + while (await tagReader.ReadAsync(ct)) + { + var key = tagReader.GetString(2); + double? globalValue = null; + if (global is { } g && columnFor(key) is { } column + && g.TryGetProperty(column, out var element) && element.ValueKind == JsonValueKind.Number) + { + globalValue = element.GetDouble(); + } + + var name = tagReader.IsDBNull(1) ? null : tagReader.GetString(1); + rows.Add(new ThresholdOverrideRow( + 0, name ?? string.Empty, null, key, tagReader.GetDouble(3), globalValue, + tagReader.GetString(4), tagReader.IsDBNull(5) ? null : tagReader.GetGuid(5), tagReader.GetDateTime(6), + tagReader.GetInt32(0), name is null)); + } + return rows; } } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs index f5e4df60fc..a99e6db89c 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesViewModel.cs @@ -25,10 +25,18 @@ public interface IThresholdOverrideGateway Task SaveThresholdOverrideAsync(int serverId, string? database, string key, int value, string setBy, CancellationToken cancellationToken = default); Task RemoveThresholdOverrideAsync(int serverId, string? database, string key, CancellationToken cancellationToken = default); + + /// Sets a tag's override (#6926): the value for every server under the tag and the tags below it. + Task SaveThresholdOverrideForTagAsync(int tagId, string key, int value, string setBy, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + Task RemoveThresholdOverrideForTagAsync(int tagId, string key, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); } -/// One server the add form can pick. -public sealed record ThresholdOverrideServerChoice(int ServerId, string Name); +/// One scope the add form can pick: a server, or ( set, #6926) a server tag, whose +/// reads Tag: name and whose is 0. +public sealed record ThresholdOverrideServerChoice(int ServerId, string Name, int? TagId = null); /// The grid's row: the stored override as the window shows it. public sealed class ThresholdOverrideViewRow @@ -39,6 +47,11 @@ public sealed class ThresholdOverrideViewRow public int ServerId => Row.ServerId; public string ServerName => Row.ServerName; + public bool IsTag => Row.IsTag; + public int? TagId => Row.TagId; + /// The Scope column (#6926): the server's name, or Tag: name for a tag override. A tag override whose tag was deleted reads + /// Tag: (deleted) - orphaned, matches nothing, as an orphaned notification route is marked. + public string ScopeText => !Row.IsTag ? Row.ServerName : Row.Orphaned ? "Tag: (deleted) - orphaned, matches nothing" : "Tag: " + Row.ServerName; public string? Database => Row.Database; public string DatabaseText => Row.Database ?? "Whole server"; public string Key => Row.Key; @@ -91,6 +104,18 @@ public ThresholdOverridesViewModel(IThresholdOverrideGateway gateway, string log return DarlingThresholdKeys.ValidateValue(entry, value, database)?.Message; } + /// Why a tag entry is refused, or null: a tag must be chosen, the key must be a tunable threshold, the value a whole number + /// inside its bounds, and no database is given, because a tag value is the server-level value for every server under the tag. + public static string? ValidateForTag(int? tagId, string? key, string? valueText, string? databaseText, out int value) + { + value = 0; + if (tagId is null) + return "Choose a tag."; + if (!string.IsNullOrWhiteSpace(databaseText)) + return "A tag override holds a server-level value, so it takes no database. Clear the Database box."; + return Validate(0, key, valueText, null, out value); + } + public async Task> LoadAsync(CancellationToken cancellationToken = default) => (await _gateway.ListThresholdOverridesAsync(cancellationToken)).Select(r => new ThresholdOverrideViewRow(r)).ToList(); @@ -115,6 +140,26 @@ public async Task SaveAsync( return new ThresholdOverrideOutcome(true, $"Override saved: {entry.Wire} = {value.ToString(CultureInfo.CurrentCulture)}. The service applies it within its reload interval."); } + /// Checks a tag entry, then writes it (#6926). An entry that fails the check changes nothing. + public async Task SaveForTagAsync( + int? tagId, string? key, string? valueText, string? databaseText, CancellationToken cancellationToken = default) + { + if (ValidateForTag(tagId, key, valueText, databaseText, out var value) is { } refusal) + return new ThresholdOverrideOutcome(false, refusal); + + DarlingThresholdKeys.TryGet(key, out var entry); + try + { + await _gateway.SaveThresholdOverrideForTagAsync(tagId!.Value, entry.Wire, value, _setBy, cancellationToken); + } + catch (ViewerReadOnlyException ex) + { + return new ThresholdOverrideOutcome(false, ex.Message); + } + + return new ThresholdOverrideOutcome(true, $"Tag override saved: {entry.Wire} = {value.ToString(CultureInfo.CurrentCulture)}. The service applies it within its reload interval to every server under the tag that has no override of its own."); + } + /// Removes the chosen override so the global value applies again. public async Task RemoveAsync(ThresholdOverrideViewRow? row, CancellationToken cancellationToken = default) { @@ -122,13 +167,16 @@ public async Task RemoveAsync(ThresholdOverrideViewRow return new ThresholdOverrideOutcome(false, "Select an override to remove."); try { - await _gateway.RemoveThresholdOverrideAsync(row.ServerId, row.Database, row.Key, cancellationToken); + if (row.TagId is { } tagId) + await _gateway.RemoveThresholdOverrideForTagAsync(tagId, row.Key, cancellationToken); + else + await _gateway.RemoveThresholdOverrideAsync(row.ServerId, row.Database, row.Key, cancellationToken); } catch (ViewerReadOnlyException ex) { return new ThresholdOverrideOutcome(false, ex.Message); } - return new ThresholdOverrideOutcome(true, $"Override removed: {row.Key} on {row.ServerName} uses the global value again."); + return new ThresholdOverrideOutcome(true, $"Override removed: {row.Key} on {row.ScopeText} uses the global value again."); } } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml index d2e7718283..7984e4da4f 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml @@ -30,7 +30,7 @@ @@ -40,7 +40,7 @@ IsReadOnly="True" SelectionChanged="OverridesGrid_SelectionChanged"> - + @@ -51,7 +51,7 @@ - - + + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs index c45baa5ac9..28ea7dcb56 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ThresholdOverridesWindow.xaml.cs @@ -59,9 +59,13 @@ private async Task LoadAsync() try { var servers = await _dataService.GetMonitoredServersAsync(); + var tags = await _dataService.GetServerTagsAsync(); ServerCombo.ItemsSource = servers .OrderBy(s => s.Name, StringComparer.OrdinalIgnoreCase) .Select(s => new ThresholdOverrideServerChoice(s.ServerId, s.Name)) + .Concat(tags + .OrderBy(t => t.Name, StringComparer.OrdinalIgnoreCase) + .Select(t => new ThresholdOverrideServerChoice(0, "Tag: " + t.Name, t.Id))) .ToList(); _rows.Clear(); foreach (var row in await _model.LoadAsync()) @@ -85,7 +89,8 @@ private void OverridesGrid_SelectionChanged(object sender, SelectionChangedEvent } /* Selecting a row loads it into the form, so Save changes it and Remove removes it. */ - ServerCombo.SelectedItem = ServerCombo.Items.OfType().FirstOrDefault(s => s.ServerId == row.ServerId); + ServerCombo.SelectedItem = ServerCombo.Items.OfType() + .FirstOrDefault(s => row.IsTag ? s.TagId == row.TagId : s.TagId is null && s.ServerId == row.ServerId); KeyCombo.SelectedItem = row.Key; DatabaseBox.Text = row.Database ?? ""; ValueBox.Text = row.Row.Value.ToString("0.##", System.Globalization.CultureInfo.CurrentCulture); @@ -98,7 +103,9 @@ private async void Save_Click(object sender, RoutedEventArgs e) try { var server = ServerCombo.SelectedItem as ThresholdOverrideServerChoice; - var outcome = await _model.SaveAsync(server?.ServerId, KeyCombo.SelectedItem as string, ValueBox.Text, DatabaseBox.Text); + var outcome = server?.TagId is { } tagId + ? await _model.SaveForTagAsync(tagId, KeyCombo.SelectedItem as string, ValueBox.Text, DatabaseBox.Text) + : await _model.SaveAsync(server?.ServerId, KeyCombo.SelectedItem as string, ValueBox.Text, DatabaseBox.Text); SetStatus(outcome.Message, !outcome.Ok); if (outcome.Ok) { diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs index a9a422a345..20ae73c0de 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ThresholdOverrides.cs @@ -44,6 +44,37 @@ public async Task SaveThresholdOverrideAsync( } } + public async Task SaveThresholdOverrideForTagAsync( + int tagId, string key, int value, string setBy, CancellationToken cancellationToken = default) + { + try + { + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken); + await ThresholdOverrideStore.UpsertTagAsync(connection, transaction, tagId, key, value, setBy, null, cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + catch (PostgresException ex) when (ex.SqlState == InsufficientPrivilegeSqlState) + { + throw new ViewerReadOnlyException(ex); + } + } + + public async Task RemoveThresholdOverrideForTagAsync(int tagId, string key, CancellationToken cancellationToken = default) + { + try + { + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken); + await ThresholdOverrideStore.DeleteTagAsync(connection, transaction, tagId, key, cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + catch (PostgresException ex) when (ex.SqlState == InsufficientPrivilegeSqlState) + { + throw new ViewerReadOnlyException(ex); + } + } + public async Task RemoveThresholdOverrideAsync( int serverId, string? database, string key, CancellationToken cancellationToken = default) { diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs index 7b42e0ab90..3d9e58a532 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs @@ -1377,7 +1377,10 @@ per the V71 finding. */ EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds' AND NOT attisdropped), /* V226 (#6893) adds the AWS RDS Web or Express fact to the server registry. Found by catalog like the V225 column above. Named only in this probe line, never in prose, per the V71 finding. */ - EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped)"; + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped), + /* V227 (#6926) adds the tag scope to the threshold overrides. Found by catalog like the V225 column above. Named only in this probe line, never in + prose, per the V71 finding. */ + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.config_threshold_overrides') AND attname = 'scope_tag_id' AND NOT attisdropped)"; /// The store schema version this viewer build requires — the highest migration it knows /// (). The connect-time gate blocks a store below this. @@ -1399,7 +1402,7 @@ per the V71 finding. */ await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201)); + return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201), reader.GetBoolean(202)); } return null; @@ -1424,7 +1427,7 @@ per the V71 finding. */ /// is unit-tested without a live store; any schema bump past the newest arm trips the pinning test that keeps /// this in step with . /// - internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false) + internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false, bool hasThresholdOverrideTagScope = false) { /* V71 (the PostgreSQL blocking-edges rung): a table-existence sentinel and now the newest-first arm. A collector table would ordinarily get no arm at all — see the V63-V69 note below — but the TOP @@ -1633,6 +1636,11 @@ store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through t /* V226 (#6893): the AWS RDS Web or Express fact on the server registry. Column-existence sentinel, newest-first, and the TOP rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 225 and showing a spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + /* V227 (#6926): the tag scope on the threshold overrides. Catalog sentinel, the top rung, newest first. */ + if (hasThresholdOverrideTagScope) + { + return 227; + } if (hasServerAwsRdsWebOrExpress) { return 226; diff --git a/docs/darling/admin.md b/docs/darling/admin.md index df819a8722..1668dc998b 100644 --- a/docs/darling/admin.md +++ b/docs/darling/admin.md @@ -189,6 +189,7 @@ An older build ignores what those migrations added, so it can read some newer da - **V219**: an older build does not know the Query Store self-statement keys ([#6750](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6750)). Rolling back leaves `collect.query_store_self_keys`, `collect.query_store_self_keys_state` and the two triggers on `collect.query_store_text` in place. The older build ignores the tables, and the triggers keep the keys current whenever it writes statement text, so when you roll forward again the Query Store regressions read still hides the monitor's own statement. Nothing needs to be done before you roll back. - **V220**: an older build does not know the Job History statistics index ([#6798](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6798)). Rolling back leaves `idx_job_history_job_success` on `collect.job_history` in place. The older build's Job History read can still use it, and the collector keeps it current on every insert. Nothing needs doing before you roll back. Leave the index in place: the viewer recognizes a V220 store by it, so a store whose index was dropped answers 219 to the viewer while its schema version says 220, the viewer refuses it, and the migrator does not build the index again. - **V221**: an older build does not know the Query Store hash-hourly table ([#6800](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6800)). Rolling back leaves `collect.query_store_hash_hourly`, the day tables `collect.query_store_hash_daily`, `collect.query_store_hash_daily_built` and `collect.query_store_hash_daily_failures`, and the `hash_built_seq` column of `collect.query_store_compose_stamp_built` in place. The older build neither reads nor fills them, so the tables stop growing and their hours and days go stale. Nothing to do before rolling back. If you roll forward again, the next hourly passes rebuild the stale hours, newest first, and the next cleanup removes the hash-hourly rows of any hour the older build retired without deleting them (it deletes by the retention floor, not by the hours it just retired) +- **V227**: an older build does not know a threshold override held for a server tag ([#6926](https://github.com/erikdarlingdata/PerformanceMonitor/issues/6926)). Rolling back leaves the `scope_tag_id` column on `config.config_threshold_overrides` (and `server_id` nullable), and any tag override rows stay. An older build reads the whole overrides table when it loads, and a row with no server stops that read, so it applies **no** threshold override at all (server or database ones included) until the tag rows are gone. Before rolling back past V227, remove every tag override (the Viewer's Threshold Overrides window, the web Admin tab, or `set_alert_threshold_override` with `clear` and a tag), or run `DELETE FROM config.config_threshold_overrides WHERE server_id IS NULL;` as the owner. ### Upgrading from a build that predates the script From 735da3965ef91c348886fa88a5671d18b3c61cb3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:50:01 -0400 Subject: [PATCH 2/4] Put the cancellation token last on GetSweepPressureAsync (CA1068) and use Assert.Single in the tag scope test --- Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs | 2 +- Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs | 4 ++-- .../ViewerDataService.CollectionHealth.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs index 1c5c98400b..b97cef96a9 100644 --- a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -165,7 +165,7 @@ await RunAsync(async (ds, connection, store, ct) => Assert.Equal("Production", cpu["tag"]!.GetValue()); Assert.Equal(92, cpu["effective_value"]!.GetValue()); Assert.Equal("global", Row(eight, "blocking.count_threshold")!["source"]!.GetValue()); - Assert.Equal(1, eight["tag_overrides"]!.AsArray().Count); + Assert.Single(eight["tag_overrides"]!.AsArray()); var nine = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-9", ct))!["server_view"]!; Assert.Equal("global", Row(nine, "cpu.threshold_percent")!["source"]!.GetValue()); diff --git a/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs b/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs index 59fec5ed1f..bbdb559253 100644 --- a/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs +++ b/Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs @@ -77,7 +77,7 @@ public async Task TheViewersSweepPressureAndHeaviestCollectors_AreTheFiguresTheW var web = doc.RootElement.GetProperty("sweep_pressure"); await using var viewer = new ViewerDataService(scratch.ConnectionString); - var report = await viewer.GetSweepPressureAsync(ServerId, ct); + var report = await viewer.GetSweepPressureAsync(ServerId, cancellationToken: ct); Assert.NotNull(report); var pressure = report.Pressure; @@ -113,7 +113,7 @@ public async Task TheViewersSweepPressureAndHeaviestCollectors_AreTheFiguresTheW SweepPressurePanel.StatRows(pressure).Select(r => r.Label)); /* A server with no collection log rows has no report, as the tool answers it "unavailable". */ - Assert.Null(await viewer.GetSweepPressureAsync(EmptyServerId, ct)); + Assert.Null(await viewer.GetSweepPressureAsync(EmptyServerId, cancellationToken: ct)); bodySucceeded = true; } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs index 97cb7b2228..f0752874d1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.CollectionHealth.cs @@ -518,7 +518,7 @@ public async Task> GetCollectionHealthAsync(int serverI /// pre-gate runs are left out of both panels, as get_collection_health computes its figures from the running rows only, so the /// panel and the tool name the same peak collector for the same server. Null leaves every row in. public async Task GetSweepPressureAsync( - int serverId, CancellationToken cancellationToken = default, IReadOnlySet? engineGated = null) + int serverId, IReadOnlySet? engineGated = null, CancellationToken cancellationToken = default) { var scheduleOverrides = await ReadScheduleOverridesForHealthAsync(cancellationToken); var costs = new List<(CollectorHealthRow Row, double Avg, double P95, double Max)>(); From 3dee3305c98d4ad2b50ada268ae3f1173a0a85cf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:53:43 -0400 Subject: [PATCH 3/4] Keep tag overrides on a failed tag read and match the loader's accept rule in the server view (#6926) LoadAsync now throws when the tag tables could not be read, so the worker keeps the whole snapshot in force instead of installing one with no tag winners until the next config change. get_alert_settings filtered tag rows through its own known-key and integer checks. It now calls the loader's Accept, so a deeper tag whose value is outside the key's bounds is skipped there as it is in the engine, and the view names the tag the engine applies. --- .../ThresholdOverrideTagScopeTests.cs | 53 +++++++++++++++++++ .../DarlingThresholdOverrides.cs | 9 +++- .../Mcp/DarlingMcpThresholdOverrideTools.cs | 3 +- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs index b97cef96a9..7aeb652800 100644 --- a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -218,6 +218,59 @@ await RunAsync(async (ds, connection, store, ct) => }); } + [Fact] + public async Task AFailedTagTableRead_KeepsTheSnapshotInForce_NotAGlobalOne() + { + await RunAsync(async (ds, connection, store, ct) => + { + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, null, "cpu.threshold_percent", 92, tag: "Production"))); + Assert.Equal("ok", Status(await Set(ds, store, WritesArm, "srv-9", "cpu.threshold_percent", 70))); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); + Assert.Equal(3, loader.Count); /* srv-9's own row, and the Production override for srv-7 and srv-8 */ + + /* The server-to-tag map cannot be read (a reset or a timeout stands in as a missing table), and a server override changes. */ + await ExecAsync(connection, "ALTER TABLE config.server_tag_map RENAME TO server_tag_map_away", ct); + await ExecAsync(connection, "UPDATE config.config_threshold_overrides SET value = 71 WHERE server_id = 9", ct); + await Assert.ThrowsAnyAsync(() => loader.LoadAsync(ds, ct)); + + /* Nothing was replaced: the tag override still applies and the changed server row is not half-loaded. */ + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); + Assert.Equal(70, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + Assert.Equal(3, loader.Count); /* srv-9's own row, and the Production override for srv-7 and srv-8 */ + + await ExecAsync(connection, "ALTER TABLE config.server_tag_map_away RENAME TO server_tag_map", ct); + await loader.LoadAsync(ds, ct); + Assert.Equal(71, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "9", null, 80)); + Assert.Equal(92, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "8", null, 80)); + }); + } + + [Fact] + public async Task TheServerView_SkipsATagRowOutsideTheBounds_AsTheLoaderDoes() + { + await RunAsync(async (ds, connection, store, ct) => + { + /* srv-7 is under east (2) under Production (1). The deeper tag holds a value outside a percent's bounds (written by direct SQL). */ + await ExecAsync(connection, "INSERT INTO config.config_threshold_overrides (scope_tag_id, threshold_key, value, set_by) VALUES " + + "(1, 'cpu.threshold_percent', 85, 'erik'), (2, 'cpu.threshold_percent', 5000, 'erik')", ct); + await ExecAsync(connection, "INSERT INTO config.config_alert_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + await ExecAsync(connection, "INSERT INTO config.config_notification (id) VALUES (1) ON CONFLICT (id) DO NOTHING", ct); + + var loader = new DarlingThresholdOverrides(); + await loader.LoadAsync(ds, ct); + Assert.Equal(85, loader.Resolve(AlertThresholdKey.CpuThresholdPercent, "7", null, 80)); + + var view = JsonNode.Parse(await DarlingMcpAlertTools.GetAlertSettings(ds, "srv-7", ct))!["server_view"]!; + var cpu = view["thresholds"]!.AsArray().First(n => n!["key"]!.GetValue() == "cpu.threshold_percent")!; + Assert.Equal("tag_override", cpu["source"]!.GetValue()); + Assert.Equal("Production", cpu["tag"]!.GetValue()); + Assert.Equal(85, cpu["effective_value"]!.GetValue()); + }); + } + [Fact] public async Task ListShowsTagRowsWithTheirTagName() { diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs index 1ea10e46c2..cb4906e908 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs @@ -46,7 +46,7 @@ private sealed record Snapshot(Dictionary<(int Server, string Database, string K /// read that throws leaves the overrides already in force. A row whose key is not a tunable threshold, whose value is /// not an integer inside the key's bounds, or whose database name sits on a server-only key is skipped and logged, /// never applied: the engine uses the value as-is. Tag overrides (#6926) are resolved with the tag tree and the server-to-tag - /// map, read only when the table holds a tag row; a tag-table read that fails leaves them matching nothing, as a tag-scoped route does. + /// map, read only when the table holds a tag row; a tag-table read that fails throws, so the whole snapshot already in force stays (it is not replaced by one with no tag overrides). /// internal async Task LoadAsync(NpgsqlDataSource postgres, CancellationToken cancellationToken = default) { @@ -72,6 +72,10 @@ internal async Task LoadAsync(NpgsqlDataSource postgres, CancellationToken cance { await using var connection = await postgres.OpenConnectionAsync(cancellationToken); inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, _logger, cancellationToken); + /* ReadRouteScopeInputsAsync swallows a failed tag read, and installing a snapshot with no tag winners would drop every + tag override to the global value until the next config change. Throw so the worker keeps the snapshot in force. */ + if (!inputs.TagsReadable) + throw new InvalidOperationException("The server tag tables could not be read, so the tag overrides cannot be resolved."); } Replace(direct, tag, inputs); @@ -125,7 +129,8 @@ internal void Replace( _snapshot = new Snapshot(direct, tagWinners); } - private static string? Accept(string key, double raw, string database, out int value) + /// Why a stored value cannot be applied (null: it can). The MCP server view filters tag rows through this too, so it never names a tag the engine skips. + internal static string? Accept(string key, double raw, string database, out int value) { value = 0; if (!DarlingThresholdKeys.TryGet(key, out var entry)) diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs index edab71bae2..f27e14ca86 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs @@ -369,7 +369,8 @@ internal static async Task BuildServerViewAsync( { var key = reader.GetString(2); var raw = reader.GetDouble(3); - if (!DarlingThresholdKeys.TryGet(key, out _) || raw != Math.Floor(raw) || raw < int.MinValue || raw > int.MaxValue) + /* The loader's own accept rule (known key, integer, inside the key's bounds), so a row the engine skips never names a tag here. */ + if (DarlingThresholdOverrides.Accept(key, raw, string.Empty, out _) is not null) continue; candidates.Add(new TagOverrideCandidate(reader.GetInt64(0), reader.GetInt32(1), key, raw)); meta[(reader.GetInt32(1), key)] = (reader.GetString(4), reader.GetDateTime(5)); From 9dd1a4677d154d9feebce6e0e3b572243ab7c5bf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:30:15 -0400 Subject: [PATCH 4/4] Declare the V227 rung in the data-moving census and build the tag-name read from the store source (#6926) V227 adds a nullable column, two CHECKs and a partial unique index to the operator-written config.config_threshold_overrides table, a handful of rows, so it is declared SetsTheFloor false and the lock-wait multiple stays 5. The tag-name read in the override tool now uses postgres.CreateCommand like its sibling reads, which is the store receiver the read-deadline census recognises. --- .../Darling.Tests/MigrationDataMovingRungCensusPins.cs | 10 ++++++++++ .../Mcp/DarlingMcpThresholdOverrideTools.cs | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs b/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs index 2c222b64ce..5bd785e1c8 100644 --- a/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs +++ b/Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs @@ -359,6 +359,16 @@ public sealed class MigrationDataMovingRungCensusPins + "singleton (id = 1): turns the Client Timeouts alert on by default (#5736). V147's shape on the same kind of table. The " + "UPDATE's WHERE client_timeouts_enabled = FALSE touches at most one row, the SET DEFAULT is catalog-only, and neither reads " + "any collected data, so the rung spends none of the lock-wait budget and the multiple stays 5."), + new( + 227, + SetsTheFloor: false, + "one CREATE UNIQUE INDEX and two ADD CONSTRAINT ... CHECK on config.config_threshold_overrides (#6926, created V191), " + + "which hold the new tag scope. The table is an operator-written config table: one row per threshold somebody chose to " + + "override for a server, so a store holds a handful of rows and a very large one a few hundred. The CHECKs are validated " + + "against every stored row and the partial index (WHERE scope_tag_id IS NOT NULL) indexes nothing, because the column the " + + "same rung adds is null on every stored row, so each statement reads the table once and takes milliseconds. The ADD " + + "COLUMN is nullable with no default and the DROP NOT NULL on server_id is catalog-only, so neither rewrites a row. It does " + + "not move the lock-wait floor, so the multiple stays 5."), ]; /// diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs index f27e14ca86..cf8748de8d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpThresholdOverrideTools.cs @@ -382,9 +382,10 @@ internal static async Task BuildServerViewAsync( await using var connection = await postgres.OpenConnectionAsync(cancellationToken); var inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, null, cancellationToken); var names = new System.Collections.Generic.Dictionary(); - await using (var command = new NpgsqlCommand("SELECT id, name FROM config.server_tags", connection) { CommandTimeout = McpCommandDeadlines.ReadSeconds }) - await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) + await using (var command = postgres.CreateCommand("SELECT id, name FROM config.server_tags")) { + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) names[reader.GetInt32(0)] = reader.GetString(1); }