Hold alert threshold overrides for a server tag (#6926)

Darling/Darling.Tests/MigrationDataMovingRungCensusPins.cs

@@ -359,6 +359,16 @@ public sealed class MigrationDataMovingRungCensusPins
             + "singleton (id = 1): turns the Client Timeouts alert on by default (#5736). V147's shape on the same kind of table. The "
             + "UPDATE's WHERE client_timeouts_enabled = FALSE touches at most one row, the SET DEFAULT is catalog-only, and neither reads "
             + "any collected data, so the rung spends none of the lock-wait budget and the multiple stays 5."),
+        new(
+            227,
+            SetsTheFloor: false,
+            "one CREATE UNIQUE INDEX and two ADD CONSTRAINT ... CHECK on config.config_threshold_overrides (#6926, created V191), "
+            + "which hold the new tag scope. The table is an operator-written config table: one row per threshold somebody chose to "
+            + "override for a server, so a store holds a handful of rows and a very large one a few hundred. The CHECKs are validated "
+            + "against every stored row and the partial index (WHERE scope_tag_id IS NOT NULL) indexes nothing, because the column the "
+            + "same rung adds is null on every stored row, so each statement reads the table once and takes milliseconds. The ADD "
+            + "COLUMN is nullable with no default and the DROP NOT NULL on server_id is catalog-only, so neither rewrites a row. It does "
+            + "not move the lock-wait floor, so the multiple stays 5."),
     ];
 
     /// <summary>

Darling/Darling.Tests/RungRewindArtifacts.cs

@@ -577,6 +577,16 @@ ALTER TABLE config.config_alert_settings
         // so a re-run of V226 does everything again. Safe on a store that never had it.
         new(226, "hasServerAwsRdsWebOrExpress",
             @"ALTER TABLE collect.servers DROP COLUMN IF EXISTS is_aws_rds_web_or_express;"),
+        // V227 (#6926): the tag scope on the threshold overrides. The column is the probe's newest sentinel. The rewind removes the tag rows,
+        // the two constraints and the tag index, puts server_id back to NOT NULL and drops the column, so a re-run of V227 does everything
+        // again. Safe on a store that never had it.
+        new(227, "hasThresholdOverrideTagScope",
+            @"DELETE FROM config.config_threshold_overrides WHERE server_id IS NULL;
+ALTER TABLE config.config_threshold_overrides DROP CONSTRAINT IF EXISTS ck_config_threshold_overrides_one_scope;
+ALTER TABLE config.config_threshold_overrides DROP CONSTRAINT IF EXISTS ck_config_threshold_overrides_tag_no_database;
+DROP INDEX IF EXISTS config.ux_config_threshold_overrides_tag;
+ALTER TABLE config.config_threshold_overrides ALTER COLUMN server_id SET NOT NULL;
+ALTER TABLE config.config_threshold_overrides DROP COLUMN IF EXISTS scope_tag_id;"),
     ];
 
     /// <summary>

Darling/Darling.Tests/ViewerSweepPressureLiveTests.cs

@@ -77,7 +77,7 @@ public async Task TheViewersSweepPressureAndHeaviestCollectors_AreTheFiguresTheW
             var web = doc.RootElement.GetProperty("sweep_pressure");
 
             await using var viewer = new ViewerDataService(scratch.ConnectionString);
-            var report = await viewer.GetSweepPressureAsync(ServerId, ct);
+            var report = await viewer.GetSweepPressureAsync(ServerId, cancellationToken: ct);
             Assert.NotNull(report);
 
             var pressure = report.Pressure;
@@ -113,7 +113,7 @@ public async Task TheViewersSweepPressureAndHeaviestCollectors_AreTheFiguresTheW
                 SweepPressurePanel.StatRows(pressure).Select(r => r.Label));
 
             /* A server with no collection log rows has no report, as the tool answers it "unavailable". */
-            Assert.Null(await viewer.GetSweepPressureAsync(EmptyServerId, ct));
+            Assert.Null(await viewer.GetSweepPressureAsync(EmptyServerId, cancellationToken: ct));
 
             bodySucceeded = true;
         }

Darling/PerformanceMonitor.Darling.Service/DarlingThresholdOverrides.cs

@@ -14,11 +14,12 @@
 using Microsoft.Extensions.Logging;
 using Npgsql;
 using PerformanceMonitor.Alerting;
+using PerformanceMonitor.Darling.Storage;
 
 namespace PerformanceMonitor.Darling.Service;
 
 /// <summary>
-/// The per-server and per-database threshold overrides the alert engine and the PostgreSQL count checks resolve through
+/// The per-server, per-database and per-tag threshold overrides the alert engine and the PostgreSQL count checks resolve through
 /// (#5916): the Darling loader behind <see cref="IAlertThresholdOverrides"/>. It holds one immutable snapshot of
 /// <c>config.config_threshold_overrides</c> and answers from it, so the sweep path does no I/O. The service reloads it at
 /// start and on every control-plane reload (the table is on the <c>config_version</c> beacon), and a failed load keeps the
@@ -27,69 +28,109 @@ namespace PerformanceMonitor.Darling.Service;
 internal sealed class DarlingThresholdOverrides : IAlertThresholdOverrides
 {
     internal const string SelectSql =
-        "SELECT server_id, database_name, threshold_key, value FROM config.config_threshold_overrides";
+        "SELECT id, server_id, database_name, scope_tag_id, threshold_key, value FROM config.config_threshold_overrides";
+
+    /// <summary>The direct (server and database) overrides, and the tag override that wins per server and key.</summary>
+    private sealed record Snapshot(Dictionary<(int Server, string Database, string Key), int> Direct, Dictionary<(int Server, string Key), int> Tag);
 
     private readonly ILogger? _logger;
-    private volatile Dictionary<(int Server, string Database, string Key), int> _snapshot = new();
+    private volatile Snapshot _snapshot = new(new(), new());
 
     internal DarlingThresholdOverrides(ILogger? logger = null) => _logger = logger;
 
-    /// <summary>How many overrides are in force.</summary>
-    internal int Count => _snapshot.Count;
+    /// <summary>How many overrides are in force: the server and database ones, and the tag ones that some server resolves to.</summary>
+    internal int Count => _snapshot.Direct.Count + _snapshot.Tag.Count;
 
     /// <summary>
     /// Replaces the snapshot with the table's rows. The read is awaited before the swap and nothing here catches, so a
     /// read that throws leaves the overrides already in force. A row whose key is not a tunable threshold, whose value is
     /// not an integer inside the key's bounds, or whose database name sits on a server-only key is skipped and logged,
-    /// never applied: the engine uses the value as-is.
+    /// never applied: the engine uses the value as-is. Tag overrides (#6926) are resolved with the tag tree and the server-to-tag
+    /// map, read only when the table holds a tag row; a tag-table read that fails throws, so the whole snapshot already in force stays (it is not replaced by one with no tag overrides).
     /// </summary>
     internal async Task LoadAsync(NpgsqlDataSource postgres, CancellationToken cancellationToken = default)
     {
-        var next = new Dictionary<(int, string, string), int>();
-        await using var command = postgres.CreateCommand(SelectSql);
-        command.CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds;
-        await using var reader = await command.ExecuteReaderAsync(cancellationToken);
-        while (await reader.ReadAsync(cancellationToken))
+        var direct = new List<(int ServerId, string? Database, string Key, double Value)>();
+        var tag = new List<(long RowId, int TagId, string Key, double Value)>();
+        await using (var command = postgres.CreateCommand(SelectSql))
         {
-            var serverId = reader.GetInt32(0);
-            var database = reader.IsDBNull(1) ? string.Empty : reader.GetString(1);
-            var key = reader.GetString(2);
-            var raw = reader.GetDouble(3);
-            if (Accept(key, raw, database, out var value) is { } problem)
+            command.CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds;
+            await using var reader = await command.ExecuteReaderAsync(cancellationToken);
+            while (await reader.ReadAsync(cancellationToken))
             {
-                LogSkipped(serverId, key, database, problem);
-                continue;
+                var key = reader.GetString(4);
+                var raw = reader.GetDouble(5);
+                if (!reader.IsDBNull(3))
+                    tag.Add((reader.GetInt64(0), reader.GetInt32(3), key, raw));
+                else if (!reader.IsDBNull(1))
+                    direct.Add((reader.GetInt32(1), reader.IsDBNull(2) ? null : reader.GetString(2), key, raw));
             }
+        }
 
-            next[(serverId, database, key)] = value;
+        StoreConfigProvider.RouteScopeInputs? inputs = null;
+        if (tag.Count > 0)
+        {
+            await using var connection = await postgres.OpenConnectionAsync(cancellationToken);
+            inputs = await StoreConfigProvider.ReadRouteScopeInputsAsync(connection, _logger, cancellationToken);
+            /* ReadRouteScopeInputsAsync swallows a failed tag read, and installing a snapshot with no tag winners would drop every
+               tag override to the global value until the next config change. Throw so the worker keeps the snapshot in force. */
+            if (!inputs.TagsReadable)
+                throw new InvalidOperationException("The server tag tables could not be read, so the tag overrides cannot be resolved.");
         }
 
-        _snapshot = next;
+        Replace(direct, tag, inputs);
     }
 
-    private void LogSkipped(int serverId, string key, string database, string problem) =>
+    private void LogSkipped(string scope, string key, string database, string problem) =>
         _logger?.LogWarning(
-            "Threshold override for server {ServerId} key '{Key}' database '{Database}' was skipped: {Problem}",
-            serverId, key, database, problem);
+            "Threshold override for {Scope} key '{Key}' database '{Database}' was skipped: {Problem}",
+            scope, key, database, problem);
 
     /// <summary>Replaces the snapshot from rows already in hand (the loader's own tests, and any caller that read the table). A
     /// skipped row is logged the way <see cref="LoadAsync"/> logs it.</summary>
-    internal void Replace(IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows)
+    internal void Replace(IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows) =>
+        Replace(rows, [], null);
+
+    /// <summary>The same with tag overrides (#6926). <paramref name="tagRows"/> carry the table row id, which breaks a tie between two
+    /// tags of the same depth, and <paramref name="inputs"/> the registry, tag tree and server-to-tag map the routes expand against
+    /// (null: the tag tables were not read, and a tag override matches nothing).</summary>
+    internal void Replace(
+        IEnumerable<(int ServerId, string? Database, string Key, double Value)> rows,
+        IEnumerable<(long RowId, int TagId, string Key, double Value)> tagRows,
+        StoreConfigProvider.RouteScopeInputs? inputs)
     {
-        var next = new Dictionary<(int, string, string), int>();
+        var direct = new Dictionary<(int, string, string), int>();
         foreach (var (serverId, database, key, raw) in rows)
         {
             var db = database ?? string.Empty;
             if (Accept(key, raw, db, out var value) is { } problem)
-                LogSkipped(serverId, key, db, problem);
+                LogSkipped("server " + serverId.ToString(CultureInfo.InvariantCulture), key, db, problem);
             else
-                next[(serverId, db, key)] = value;
+                direct[(serverId, db, key)] = value;
+        }
+
+        var accepted = new List<TagOverrideCandidate>();
+        foreach (var (rowId, tagId, key, raw) in tagRows)
+        {
+            if (Accept(key, raw, string.Empty, out var value) is { } problem)
+                LogSkipped("tag " + tagId.ToString(CultureInfo.InvariantCulture), key, string.Empty, problem);
+            else
+                accepted.Add(new TagOverrideCandidate(rowId, tagId, key, value));
+        }
+
+        var tagWinners = new Dictionary<(int, string), int>();
+        if (accepted.Count > 0 && inputs is not null)
+        {
+            foreach (var (slot, winner) in TagThresholdResolver.Resolve(
+                         accepted, inputs.KnownServerIds, inputs.TagParents, inputs.ServerTags, inputs.TagsReadable))
+                tagWinners[slot] = (int)winner.Value;
         }
 
-        _snapshot = next;
+        _snapshot = new Snapshot(direct, tagWinners);
     }
 
-    private static string? Accept(string key, double raw, string database, out int value)
+    /// <summary>Why a stored value cannot be applied (null: it can). The MCP server view filters tag rows through this too, so it never names a tag the engine skips.</summary>
+    internal static string? Accept(string key, double raw, string database, out int value)
     {
         value = 0;
         if (!DarlingThresholdKeys.TryGet(key, out var entry))
@@ -106,18 +147,21 @@ public int Resolve(AlertThresholdKey key, string serverKey, string? database, in
 
     /// <summary>
     /// The value of the wire-named threshold for the server (and database): a database override first, then the
-    /// server's, else <paramref name="globalValue"/>. <paramref name="serverKey"/> is the store server id as text, the
-    /// engine's server key. This is the one resolution the PostgreSQL count thresholds use too.
+    /// server's, then the tag's (#6926: the deepest tag the server is under), else <paramref name="globalValue"/>.
+    /// <paramref name="serverKey"/> is the store server id as text, the engine's server key. This is the one resolution the
+    /// PostgreSQL count thresholds use too.
     /// </summary>
     internal int ResolveWire(string wire, string serverKey, string? database, int globalValue)
     {
         if (!int.TryParse(serverKey, NumberStyles.Integer, CultureInfo.InvariantCulture, out var serverId))
             return globalValue;
         var snapshot = _snapshot;
-        if (snapshot.Count == 0)
+        if (snapshot.Direct.Count == 0 && snapshot.Tag.Count == 0)
             return globalValue;
-        if (!string.IsNullOrEmpty(database) && snapshot.TryGetValue((serverId, database, wire), out var perDatabase))
+        if (!string.IsNullOrEmpty(database) && snapshot.Direct.TryGetValue((serverId, database, wire), out var perDatabase))
             return perDatabase;
-        return snapshot.TryGetValue((serverId, string.Empty, wire), out var perServer) ? perServer : globalValue;
+        if (snapshot.Direct.TryGetValue((serverId, string.Empty, wire), out var perServer))
+            return perServer;
+        return snapshot.Tag.TryGetValue((serverId, wire), out var perTag) ? perTag : globalValue;
     }
 }

Darling/PerformanceMonitor.Darling.Service/DarlingWebThresholdOverrides.cs

@@ -42,8 +42,9 @@ internal static string RefusedMessage(bool managedStore, bool adminWritesOff = f
             + "Darling/tools/provision-roles.sql re-run since the upgrade that added threshold overrides: run it once as a superuser (it is idempotent).",
             adminWritesOff);
 
-    /// <summary>The list body: <c>{overrides: [{server_id, server_name, database, key, value, global_value, set_by, request_id,
-    /// updated_at}], keys: [{key, min, max, database_scope}], servers: [{server_id, server_name}], can_edit, note}</c>.
+    /// <summary>The list body: <c>{overrides: [{server_id, server_name, tag_id, tag_name, orphaned, database, key, value, global_value, set_by,
+    /// request_id, updated_at}], keys: [{key, min, max, database_scope}], servers: [{server_id, server_name}], tags: [{tag_id, tag_name}],
+    /// can_edit, note}</c>. A tag override (#6926) has a tag_id, a null server_id and server_name, and orphaned true when its tag was deleted.
     /// <c>can_edit</c> is whether the store login holds the write grant, so a page can say list-only before anyone tries.</summary>
     internal static async Task<JsonObject> ListBodyAsync(NpgsqlDataSource postgres, CancellationToken ct, NpgsqlDataSource? accessSource = null)
     {
@@ -56,8 +57,11 @@ internal static async Task<JsonObject> ListBodyAsync(NpgsqlDataSource postgres,
             {
                 overrides.Add(new JsonObject
                 {
-                    ["server_id"] = r.ServerId,
-                    ["server_name"] = r.ServerName,
+                    ["server_id"] = r.IsTag ? null : r.ServerId,
+                    ["server_name"] = r.IsTag ? null : r.ServerName,
+                    ["tag_id"] = r.TagId,
+                    ["tag_name"] = r.IsTag && !r.Orphaned ? r.ServerName : null,
+                    ["orphaned"] = r.Orphaned,
                     ["database"] = r.Database,
                     ["key"] = r.Key,
                     ["value"] = r.Value,
@@ -95,11 +99,21 @@ internal static async Task<JsonObject> ListBodyAsync(NpgsqlDataSource postgres,
                 servers.Add(new JsonObject { ["server_id"] = reader.GetInt32(0), ["server_name"] = reader.GetString(1) });
         }
 
+        var tags = new JsonArray();
+        await using (var command = postgres.CreateCommand("SELECT id, name FROM config.server_tags ORDER BY name, id"))
+        {
+            command.CommandTimeout = McpCommandDeadlines.ReadSeconds;
+            await using var reader = await command.ExecuteReaderAsync(ct);
+            while (await reader.ReadAsync(ct))
+                tags.Add(new JsonObject { ["tag_id"] = reader.GetInt32(0), ["tag_name"] = reader.GetString(1) });
+        }
+
         return new JsonObject
         {
             ["overrides"] = overrides,
             ["keys"] = keys,
             ["servers"] = servers,
+            ["tags"] = tags,
             ["can_edit"] = await DarlingMcpThresholdOverrideTools.HoldsGrantAsync(accessSource ?? postgres, ct),
             ["note"] = note,
         };
@@ -116,6 +130,7 @@ internal static async Task<JsonObject> ListBodyAsync(NpgsqlDataSource postgres,
         var server = body["server_name"]?.GetValueKind() == JsonValueKind.String ? body["server_name"]!.GetValue<string>() : null;
         var key = body["key"]?.GetValueKind() == JsonValueKind.String ? body["key"]!.GetValue<string>() : null;
         var database = body["database_name"]?.GetValueKind() == JsonValueKind.String ? body["database_name"]!.GetValue<string>() : null;
+        var tag = body["tag"]?.GetValueKind() == JsonValueKind.String ? body["tag"]!.GetValue<string>() : null;
         var clear = body["clear"]?.GetValueKind() == JsonValueKind.True;
 
         int? value = null;
@@ -126,13 +141,13 @@ internal static async Task<JsonObject> ListBodyAsync(NpgsqlDataSource postgres,
             value = parsed;
         }
 
-        if (string.IsNullOrWhiteSpace(server))
-            return (400, new JsonObject { ["error"] = "server_name is required." });
+        if (string.IsNullOrWhiteSpace(server) && string.IsNullOrWhiteSpace(tag))
+            return (400, new JsonObject { ["error"] = "server_name or tag is required." });
 
         var json = await DarlingMcpThresholdOverrideTools.SetAsync(
             postgres, new DarlingMcpWritePolicy.Effective(true, false),
             new InvestigationRequestStore(postgres, McpCommandDeadlines.ReadSeconds), logger,
-            server, key ?? string.Empty, value, database, clear, ThresholdOverrideStore.SetByFor(editor), null, ct);
+            server, key ?? string.Empty, value, database, clear, ThresholdOverrideStore.SetByFor(editor), null, ct, tag: tag);
 
         var answer = JsonNode.Parse(json)!.AsObject();
         var status = answer["status"]?.GetValue<string>();