diff --git a/Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs b/Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs new file mode 100644 index 0000000000..6f835bcf51 --- /dev/null +++ b/Darling/Darling.Tests/BlockingBandSettingsLiveTests.cs @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// The blocking band tiers (#6927, V228) on a real PostgreSQL store: an upgraded store whose settings row already exists keeps +/// today's defaults and its other values, and the tiers round trip through update_alert_settings and get_alert_settings +/// with a critical bar below its warn bar refused and nothing written. Each fact mints its own scratch database. +/// +/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Each fact mints its own scratch database through + ScratchPostgres and never touches the shared store's tables. */ +public sealed class BlockingBandSettingsLiveTests +{ + private const string TiersSql = "SELECT blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1"; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task AnUpgradedStoreWithASettingsRow_KeepsTodaysDefaults_AndItsOtherValues() + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the V228 upgrade test."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + + await PgMigrations.MigrateAsync(connection, ct); + await SeedAsync(scratch.ConnectionString, ct); + + /* Roll the store back to look like V227: the settings row exists and carries an operator's own deadlock tier, the four + columns do not. */ + await ExecAsync(connection, ct, "UPDATE config.config_alert_settings SET deadlock_warn_per_hour = 7.0, deadlock_critical_per_hour = 30.0 WHERE id = 1"); + foreach (var column in new[] { "blocking_warn_per_hour", "blocking_critical_per_hour", "blocking_warn_wait_seconds", "blocking_critical_wait_seconds" }) + { + await ExecAsync(connection, ct, "ALTER TABLE config.config_alert_settings DROP COLUMN " + column); + } + + await ExecAsync(connection, ct, "DELETE FROM darling_schema_version WHERE version >= 228"); + + var applied = await PgMigrations.MigrateAsync(connection, ct); + Assert.Equal(PgMigrations.Scripts.Count(m => m.Version >= 228), applied); + + Assert.Equal((5.0, 20.0, 10.0, 60.0), await ReadTiersAsync(connection, ct)); + + /* The band reads today's constants from the upgraded row, and the operator's deadlock tier did not move. */ + await using var dataSource = NpgsqlDataSource.Create(scratch.ConnectionString); + Assert.Equal(BlockingBandThresholds.Default, await BlockingBandStore.ReadAsync(dataSource, 10, ct)); + using var deadlock = new NpgsqlCommand("SELECT deadlock_warn_per_hour, deadlock_critical_per_hour FROM config_alert_settings WHERE id = 1", connection); + using var reader = await deadlock.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct)); + Assert.Equal(7.0, reader.GetDouble(0)); + Assert.Equal(30.0, reader.GetDouble(1)); + } + + [Fact] + public async Task UpdateAlertSettings_SavesTheFourTiers_GetAlertSettingsReadsThemBack_AndACriticalBelowWarnIsRefused() + { + var baseConnectionString = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the blocking band settings test."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await SeedAsync(scratch.ConnectionString, ct); + + await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString); + + /* The seeded row reads today's constants. */ + AssertBands(await DarlingMcpAlertTools.GetAlertSettings(postgres), 5, 20, 10, 60); + + /* A valid body saves, and a fresh read carries it. */ + var saved = await DarlingMcpAlertTools.UpdateAlertSettings(postgres, + "{\"health_bands\":{\"blocking_warn_per_hour\":50,\"blocking_critical_per_hour\":100,\"blocking_warn_wait_seconds\":60,\"blocking_critical_wait_seconds\":180}}"); + Assert.Equal("updated", DarlingMcpTestData.StatusOf(saved)); + Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct)); + AssertBands(await DarlingMcpAlertTools.GetAlertSettings(postgres), 50, 100, 60, 180); + Assert.Equal(new BlockingBandThresholds(50, 100, 60, 180), await BlockingBandStore.ReadAsync(postgres, 10, ct)); + + /* A critical bar below its warn bar is refused with nothing written: in one body, and as a lone value against the stored other one. */ + foreach (var body in new[] + { + "{\"health_bands\":{\"blocking_warn_per_hour\":80,\"blocking_critical_per_hour\":30}}", + "{\"health_bands\":{\"blocking_critical_per_hour\":40}}", + "{\"health_bands\":{\"blocking_warn_wait_seconds\":200}}", + "{\"health_bands\":{\"blocking_warn_wait_seconds\":90,\"blocking_critical_wait_seconds\":30}}", + }) + { + var refused = await DarlingMcpAlertTools.UpdateAlertSettings(postgres, body); + Assert.Equal("invalid", DarlingMcpTestData.StatusOf(refused)); + Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct)); + } + + /* Out of range is refused too. */ + Assert.Equal("invalid", DarlingMcpTestData.StatusOf(await DarlingMcpAlertTools.UpdateAlertSettings(postgres, + "{\"health_bands\":{\"blocking_warn_per_hour\":0}}"))); + Assert.Equal((50.0, 100.0, 60.0, 180.0), await ReadTiersAsync(connection, ct)); + } + + private static void AssertBands(string settings, double warnPerHour, double criticalPerHour, double warnWait, double criticalWait) + { + Assert.False(McpHelpers.IsErrorEnvelope(settings), settings); + using var doc = JsonDocument.Parse(settings); + var bands = doc.RootElement.GetProperty("health_bands"); + Assert.Equal(warnPerHour, bands.GetProperty("blocking_warn_per_hour").GetDouble()); + Assert.Equal(criticalPerHour, bands.GetProperty("blocking_critical_per_hour").GetDouble()); + Assert.Equal(warnWait, bands.GetProperty("blocking_warn_wait_seconds").GetDouble()); + Assert.Equal(criticalWait, bands.GetProperty("blocking_critical_wait_seconds").GetDouble()); + } + + private static async Task<(double WarnPerHour, double CriticalPerHour, double WarnWait, double CriticalWait)> ReadTiersAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var command = new NpgsqlCommand(TiersSql, connection); + using var reader = await command.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct), "config_alert_settings has no singleton row"); + return (reader.GetDouble(0), reader.GetDouble(1), reader.GetDouble(2), reader.GetDouble(3)); + } + + private static async Task ExecAsync(NpgsqlConnection connection, CancellationToken ct, string sql) + { + using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(ct); + } + + /// Seeds the singleton rows through the product's own path, so this test need not track their other NOT NULL columns. + private static async Task SeedAsync(string connectionString, CancellationToken ct) + { + await using var dataSource = NpgsqlDataSource.Create(connectionString); + await new StoreConfigProvider(dataSource).SeedIfEmptyAsync(new DarlingConfig(), ct); + } +} diff --git a/Darling/Darling.Tests/BlockingBandTiersTests.cs b/Darling/Darling.Tests/BlockingBandTiersTests.cs new file mode 100644 index 0000000000..7f0e52cfe1 --- /dev/null +++ b/Darling/Darling.Tests/BlockingBandTiersTests.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace PerformanceMonitor.Darling.Tests; + +/// #6927: the blocking band reads settable tiers whose defaults are the constants every earlier build used. +public sealed class BlockingBandTiersTests +{ + private static readonly TimeSpan Day = TimeSpan.FromHours(24); + + [Fact] + public void Defaults_EqualTheCompileTimeConstants() + { + var d = BlockingBandThresholds.Default; + Assert.Equal(ServerHealthThresholds.BlockingWarnPerHour, d.WarnPerHour); + Assert.Equal(ServerHealthThresholds.BlockingCriticalPerHour, d.CriticalPerHour); + Assert.Equal(ServerHealthThresholds.BlockingWarnWaitSeconds, d.WarnWaitSeconds); + Assert.Equal(ServerHealthThresholds.BlockingCriticalWaitSeconds, d.CriticalWaitSeconds); + Assert.Equal( + ServerHealthClassifier.BlockingSeverity(1434, 162.0, Day), + ServerHealthClassifier.BlockingSeverity(1434, 162.0, Day, d)); + } + + [Fact] + public void ADayTunedTo100EventsAnd180Seconds_IsNotCritical_ButIsUnderTheDefaults() + { + /* 2026-10-23 in the issue: 1,434 reports over a day is 59.75 an hour here (73.5 on the day's own window), peak block 2.7 minutes. */ + const long reports = 1764; // 73.5 an hour over 24 hours + const double peakSeconds = 162.0; // 2.7 minutes + + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(reports, peakSeconds, Day)); + + var tuned = new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(reports, peakSeconds, Day, tuned)); + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(2400, 100.0, Day, tuned)); // 100 an hour + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(1, 180.0, Day, tuned)); + } + + [Fact] + public void TheDailyClassifier_UsesTheTunedTiers() + { + var signals = new DailyHealthSignals + { + HasData = true, + Window = Day, + BlockingReports = 1764, + BlockingEvents = 1764, + PeakBlockWaitMs = 162_000, + }; + + Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(signals)); + var tuned = new DailyHealthThresholds { BlockingBands = new BlockingBandThresholds(50.0, 100.0, 60.0, 180.0) }; + Assert.NotEqual(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(signals, tuned)); + } + + [Fact] + public void RawValuesClampToTheFloorsAndCeilings_AndADefaultStructClampsToTheFloorNotZero() + { + var wild = new BlockingBandThresholds(-5, double.NaN, 0, 1e12); + Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, wild.WarnPerHour); + Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, wild.CriticalPerHour); + Assert.Equal(ServerHealthThresholds.BlockingWaitSecondsFloor, wild.WarnWaitSeconds); + Assert.Equal(ServerHealthThresholds.BlockingWaitSecondsCeiling, wild.CriticalWaitSeconds); + Assert.Equal(ServerHealthThresholds.BlockingRatePerHourFloor, default(BlockingBandThresholds).WarnPerHour); + } + + private static FleetServerCard CardAt(int id, HealthSeverity blocking, HealthSeverity deadlock) => new() + { + ServerId = id, + DisplayName = "s" + id, + ServerName = "s" + id, + BlockingCount = 3, + BlockingSeverity = blocking, + DeadlockSeverity = deadlock, + }; + + [Fact] + public void TheFleetTotalsTiles_TakeTheWorstCardBand_SoTenServersAtThreeAnHourAreNotCritical() + { + var now = new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc); + var empty = DarlingFleetReader.BuildRollup(Array.Empty(), now, now.AddHours(-1), now); + Assert.Equal(HealthSeverity.Healthy, empty.TotalBlockingSeverity); + Assert.Equal(HealthSeverity.Healthy, empty.TotalDeadlockSeverity); + + /* Ten servers each at 3 reports an hour: no card is even Warning at the default 5 an hour, and the fleet sum of 30 an hour + must not band as one server's rate. */ + var window = TimeSpan.FromHours(1); + var cards = new List(); + for (var i = 1; i <= 10; i++) + { + var sev = ServerHealthClassifier.BlockingSeverity(3, 0.0, window, BlockingBandThresholds.Default); + Assert.Equal(HealthSeverity.Healthy, sev); + cards.Add(CardAt(i, sev, HealthSeverity.Healthy)); + } + Assert.Equal(30, DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now).TotalBlockingEvents); + Assert.Equal(HealthSeverity.Healthy, DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now).TotalBlockingSeverity); + + cards[4] = CardAt(5, HealthSeverity.Critical, HealthSeverity.Warning); + var one = DarlingFleetReader.BuildRollup(cards, now, now.AddHours(-1), now); + Assert.Equal(HealthSeverity.Critical, one.TotalBlockingSeverity); + Assert.Equal(HealthSeverity.Warning, one.TotalDeadlockSeverity); + } +} diff --git a/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs b/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs index cfd8abc4ef..325bb6f96b 100644 --- a/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs +++ b/Darling/Darling.Tests/ClientTimeoutsSettingsSurfaceTests.cs @@ -23,7 +23,7 @@ namespace Darling.Tests; /// #5640 (Lane E1): the Darling Viewer's Client Timeouts settings, the V179 twin of /// RunnableQueueSettingsSurfaceTests. The row carries the engine's defaults (ON, 10, 60), compares and saves /// all three columns through the engine's clamp, and the Settings window names the alert. V179 is the newest append, so -/// this class owns the settings row's total width (86). The viewer needs no per-version gate for the columns: it refuses a +/// this class owns the settings row's total width (90). The viewer needs no per-version gate for the columns: it refuses a /// store below StorageVersion.SchemaVersion at connect, as it did for V177's. /// public sealed class ClientTimeoutsSettingsSurfaceTests @@ -72,7 +72,7 @@ public void TheUpsertAndTheSelect_CarryTheThreeColumns_AtTheEnd() /// /// V179's three columns sit at the FIXED ordinals 73, 74 and 75 (parameters $74 to $76) on every positional - /// settings-row surface, and the row is 86 columns wide (V200 appended the excluded volumes at 76, V211 the two multiples at 77 and 78, V222 the Agent alert gaps switch at 79, V223 the three database protection switches at 80 to 82, V224 the three identity capacity columns at 83 to 85): the viewer's select and upsert, the service's own read and + /// settings-row surface, and the row is 90 columns wide (V228 appended the four blocking band tiers at 86 to 89, V200 appended the excluded volumes at 76, V211 the two multiples at 77 and 78, V222 the Agent alert gaps switch at 79, V223 the three database protection switches at 80 to 82, V224 the three identity capacity columns at 83 to 85): the viewer's select and upsert, the service's own read and /// its seed, and the MCP read with its positional record. This is the test that was red while the seed had 76 columns /// and the viewer 73. /// @@ -82,7 +82,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co var viewerColumns = ViewerDataService.AlertSettingsSelectSql .Replace("SELECT ", "", StringComparison.Ordinal).Split(" FROM ")[0] .Split(',').Select(c => c.Trim()).ToList(); - Assert.Equal(86, viewerColumns.Count); + Assert.Equal(90, viewerColumns.Count); Assert.Equal(Expected, viewerColumns.Skip(73).Take(3).ToArray()); var service = File.ReadAllText(Path.Combine(RepoRoot(), "Darling", "PerformanceMonitor.Darling.Service", "StoreConfigProvider.cs")); @@ -91,7 +91,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co var select = service[start..]; select = select[(select.IndexOf("SELECT", StringComparison.Ordinal) + 6)..select.IndexOf("FROM config_alert_settings WHERE id = 1", StringComparison.Ordinal)]; var serviceColumns = select.Split(',').Select(c => c.Trim()).Where(c => c.Length > 0).ToList(); - Assert.Equal(86, serviceColumns.Count); + Assert.Equal(90, serviceColumns.Count); Assert.Equal(Expected, serviceColumns.Skip(73).Take(3).ToArray()); Assert.Contains("ClientTimeoutsEnabled = reader.GetBoolean(73),", service, StringComparison.Ordinal); Assert.Contains("ClientTimeoutsWarningPerMinute = reader.GetInt32(74),", service, StringComparison.Ordinal); @@ -107,10 +107,10 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co var mcpColumns = DarlingAlertReader.AlertSettingsSelectSql .Split("FROM config_alert_settings")[0].Replace("SELECT", "", StringComparison.Ordinal) .Split(',').Select(c => c.Trim()).Where(c => c.Length > 0).ToList(); - Assert.Equal(86, mcpColumns.Count); + Assert.Equal(90, mcpColumns.Count); Assert.Equal(Expected, mcpColumns.Skip(73).Take(3).ToArray()); var parameters = typeof(DarlingAlertReader.AlertSettingsReadRow).GetConstructors().Single().GetParameters(); - Assert.Equal(86, parameters.Length); + Assert.Equal(90, parameters.Length); Assert.Equal(new[] { "ClientTimeoutsEnabled", "ClientTimeoutsWarningPerMinute", "ClientTimeoutsCriticalPerMinute" }, parameters.Skip(73).Take(3).Select(p => p.Name!).ToArray()); Assert.Equal(typeof(bool), parameters[73].ParameterType); @@ -120,7 +120,7 @@ public void TheThreeColumns_SitAtOrdinals73To75_OnEveryPositionalSurface_OfA86Co /* The viewer's bind supplies $74 to $76 at indexes 73 to 75, in column order (V200's excluded_volumes follows at 76). */ var row = new AlertSettingsRow { ClientTimeoutsEnabled = true, ClientTimeoutsWarningPerMinute = 5, ClientTimeoutsCriticalPerMinute = 11 }; var command = Bind(row); - Assert.Equal(86, command.Parameters.Count); + Assert.Equal(90, command.Parameters.Count); Assert.Equal(true, command.Parameters[73].Value); Assert.Equal(5, command.Parameters[74].Value); Assert.Equal(11, command.Parameters[75].Value); diff --git a/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs b/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs index 2b0ee84ee0..92ff8853e8 100644 --- a/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs @@ -1333,7 +1333,8 @@ dropped the columns and fell back to the engine defaults still match. Warning is ClientTimeoutsWarningMultiple: 2, ClientTimeoutsCriticalMultiple: 7, AgentAlertGapsEnabled: true, BackupOverdueEnabled: true, LogBackupOverdueEnabled: true, CheckDbOverdueEnabled: true, - IdentityCapacityEnabled: true, IdentityCapacityWarningPercent: 72, IdentityCapacityCriticalPercent: 92); + IdentityCapacityEnabled: true, IdentityCapacityWarningPercent: 72, IdentityCapacityCriticalPercent: 92, + BlockingWarnPerHour: 6.5, BlockingCriticalPerHour: 22.5, BlockingWarnWaitSeconds: 11.5, BlockingCriticalWaitSeconds: 62.5); [Fact] public void AlertSettingsSql_ReadsSingleGlobalRow() diff --git a/Darling/Darling.Tests/RungRewindArtifacts.cs b/Darling/Darling.Tests/RungRewindArtifacts.cs index a4273f778b..123cae6d21 100644 --- a/Darling/Darling.Tests/RungRewindArtifacts.cs +++ b/Darling/Darling.Tests/RungRewindArtifacts.cs @@ -587,6 +587,14 @@ ALTER TABLE config.config_alert_settings DROP INDEX IF EXISTS config.ux_config_threshold_overrides_tag; ALTER TABLE config.config_threshold_overrides ALTER COLUMN server_id SET NOT NULL; ALTER TABLE config.config_threshold_overrides DROP COLUMN IF EXISTS scope_tag_id;"), + // V228 (#6927): the four blocking band columns on the alert settings row. The warn per hour column is the probe's newest sentinel. The + // rewind drops all four, so a re-run of V228 adds them again. Safe on a store that never had them. + new(228, "hasBlockingBands", + @"ALTER TABLE config.config_alert_settings + DROP COLUMN IF EXISTS blocking_warn_per_hour, + DROP COLUMN IF EXISTS blocking_critical_per_hour, + DROP COLUMN IF EXISTS blocking_warn_wait_seconds, + DROP COLUMN IF EXISTS blocking_critical_wait_seconds;"), ]; /// diff --git a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs index 7aeb652800..553cdd8a60 100644 --- a/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs +++ b/Darling/Darling.Tests/ThresholdOverrideTagScopeTests.cs @@ -313,7 +313,7 @@ await ExecAsync(store, "INSERT INTO config.config_threshold_overrides (server_id "SELECT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'config.config_threshold_overrides'::regclass AND attname = 'scope_tag_id' AND NOT attisdropped)", ct)); await PgMigrations.MigrateAsync(store, ct); - Assert.Equal("227", await ScalarAsync(store, "SELECT max(version) FROM darling_schema_version", ct)); + Assert.Equal(StorageVersion.SchemaVersion.ToString(CultureInfo.InvariantCulture), await ScalarAsync(store, "SELECT max(version) FROM darling_schema_version", ct)); Assert.Equal("2", await ScalarAsync(store, "SELECT count(*) FROM config.config_threshold_overrides WHERE server_id = 7 AND scope_tag_id IS NULL", ct)); Assert.Equal("91", await ScalarAsync(store, "SELECT value FROM config.config_threshold_overrides WHERE database_name IS NULL", ct)); diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs index 02690d7096..1d765cde4f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs @@ -1105,6 +1105,27 @@ public sealed class AlertsConfig [JsonPropertyName("deadlockCriticalPerHour")] public double DeadlockCriticalPerHour { get; set; } = ServerHealthThresholds.DeadlockCriticalPerHourDefault; + /// #6927 (V228): the blocking health band's WARNING tier, blocked-process reports per hour over the window a card + /// counted them in. Seeded from , named rather than restated, so a store + /// that never sets it bands as every earlier build did. BlockingBandThresholds clamps it on read. + [JsonPropertyName("blockingWarnPerHour")] + public double BlockingWarnPerHour { get; set; } = ServerHealthThresholds.BlockingWarnPerHour; + + /// #6927 (V228): the blocking health band's CRITICAL tier, reports per hour, seeded from + /// . + [JsonPropertyName("blockingCriticalPerHour")] + public double BlockingCriticalPerHour { get; set; } = ServerHealthThresholds.BlockingCriticalPerHour; + + /// #6927 (V228): the blocking band's WARNING bar on the longest single block, seconds, seeded from + /// . + [JsonPropertyName("blockingWarnWaitSeconds")] + public double BlockingWarnWaitSeconds { get; set; } = ServerHealthThresholds.BlockingWarnWaitSeconds; + + /// #6927 (V228): the blocking band's CRITICAL bar on the longest single block, seconds, seeded from + /// . + [JsonPropertyName("blockingCriticalWaitSeconds")] + public double BlockingCriticalWaitSeconds { get; set; } = ServerHealthThresholds.BlockingCriticalWaitSeconds; + [JsonPropertyName("longRunningJobEnabled")] public bool LongRunningJobEnabled { get; set; } = true; diff --git a/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs b/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs index 54d5b52a91..3e219964e0 100644 --- a/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs +++ b/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs @@ -197,7 +197,8 @@ public static FleetSweepComposition Compose( IReadOnlyList previousVerdicts, IReadOnlyList activeWatchItems, FleetSweepInstrumentCounters instruments, - DeadlockRateThresholds deadlockRateTiers) + DeadlockRateThresholds deadlockRateTiers, + BlockingBandThresholds? blockingBandTiers = null) { ArgumentNullException.ThrowIfNull(readings); ArgumentNullException.ThrowIfNull(previousVerdicts); @@ -209,7 +210,12 @@ public static FleetSweepComposition Compose( a caller that kept the old call would compile and silently band on the shipped pair while the Overview card used the store's. Built once, because the banding thresholds must be one configuration for the whole sweep. */ - var banding = new DailyHealthThresholds { DeadlockRates = deadlockRateTiers }; + /* #6927: the blocking band's tiers travel the same way; null is today's constants. */ + var banding = new DailyHealthThresholds + { + DeadlockRates = deadlockRateTiers, + BlockingBands = blockingBandTiers ?? BlockingBandThresholds.Default, + }; var sweepId = nowUtc.Ticks; var inSettleWindow = nowUtc - instruments.ServiceStartedUtc < PostRestartSettleWindow; @@ -785,26 +791,26 @@ The production read fills both from one MAX. */ unit its rate tiers were measured in; BlockingEvents counts WAITS and is what the evidence shows as the count. Classify reads the same pair, so the ledger still cannot page on a count the verdict did not band. */ var bandedBlocking = signals.BlockingReports ?? signals.BlockingEvents; - if (ServerHealthClassifier.BlockingSeverity(bandedBlocking, peakBlockSeconds, signals.Window) + if (ServerHealthClassifier.BlockingSeverity(bandedBlocking, peakBlockSeconds, signals.Window, thresholds.BlockingBands) == HealthSeverity.Critical) { - yield return (FamilyBlocking, BlockingEvidence(signals.BlockingEvents, bandedBlocking, signals.BlockingReports.HasValue, signals.Window, peakBlockSeconds)); + yield return (FamilyBlocking, BlockingEvidence(signals.BlockingEvents, bandedBlocking, signals.BlockingReports.HasValue, signals.Window, peakBlockSeconds, thresholds.BlockingBands)); } } /// The blocking family's evidence (#3539 A3): the arm that banded is the one named. A 60 s /// block is the wait arm's Critical whatever the rate; otherwise the RATE is the value, with the raw /// count as its own member for the operator reconciling against the blocking grid. - private static string BlockingEvidence(long count, long bandedCount, bool bandedOnReports, TimeSpan window, double peakBlockSeconds) + private static string BlockingEvidence(long count, long bandedCount, bool bandedOnReports, TimeSpan window, double peakBlockSeconds, BlockingBandThresholds tiers) { - if (peakBlockSeconds >= ServerHealthThresholds.BlockingCriticalWaitSeconds) + if (peakBlockSeconds >= tiers.CriticalWaitSeconds) { return JsonSerializer.Serialize(new { derivation = "summary-scoring critical trigger under alerts_enabled: false — not an alert-engine replay", trigger = "longest single block in the sweep span, seconds", value = peakBlockSeconds, - threshold = ServerHealthThresholds.BlockingCriticalWaitSeconds, + threshold = tiers.CriticalWaitSeconds, blocking_count = count, }); } @@ -819,7 +825,7 @@ rate is present by construction. */ ? "blocked-process reports per hour over the sweep span" : "blocking events per hour over the sweep span", value = ratePerHour, - threshold = ServerHealthThresholds.BlockingCriticalPerHour, + threshold = tiers.CriticalPerHour, blocking_count = count, }); } @@ -924,6 +930,7 @@ public static async Task RunAsync( — an engine-seam read, so a fault here loudly costs this sweep slot rather than quietly banding the fleet on the shipped pair. */ var deadlockRateTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken).ConfigureAwait(false); + var blockingBandTiers = await BlockingBandStore.ReadAsync(postgres, McpCommandDeadlines.ReadSeconds, cancellationToken).ConfigureAwait(false); var nowUtc = DateTime.UtcNow; var spanStartUtc = ComputeSpanStart(nowUtc, interval, previousRun); @@ -942,7 +949,7 @@ at a time and a herd all at once — and the sweep is a background errand racing var composition = Compose( nowUtc, spanStartUtc, alertsEnabled, servers.Count, readings, - previousRun, previousVerdicts, activeItems, ReadInstrumentCounters(), deadlockRateTiers); + previousRun, previousVerdicts, activeItems, ReadInstrumentCounters(), deadlockRateTiers, blockingBandTiers); await FleetSweepStore.RecordSweepAsync( postgres, composition.Run, composition.Verdicts, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs index 846a2a2dcc..b33da5ef24 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs @@ -542,7 +542,12 @@ homes decided (DarlingAlertSettings.ResolveUncorroboratedRoute takes this beside /* #6875 (V224): the Identity Capacity switch (boolean NOT NULL, default off) and its warning and critical bars (integer NOT NULL, 70 and 90). APPENDED, same reason. */ bool IdentityCapacityEnabled, int IdentityCapacityWarningPercent, - int IdentityCapacityCriticalPercent); + int IdentityCapacityCriticalPercent, + /* #6927 (V228): the four blocking band tiers (double precision NOT NULL, 5, 20 reports per hour and 10, 60 seconds). */ + double BlockingWarnPerHour, + double BlockingCriticalPerHour, + double BlockingWarnWaitSeconds, + double BlockingCriticalWaitSeconds); /// The single global alert-settings row (id=1) — the viewer's AlertSettingsSelectSql. The /// columns are read in the SAME order the service reads them (StoreConfigProvider), and @@ -582,7 +587,8 @@ homes decided (DarlingAlertSettings.ResolveUncorroboratedRoute takes this beside excluded_volumes, client_timeouts_warning_multiple, client_timeouts_critical_multiple, agent_alert_gaps_enabled, backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, - identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent + identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, + blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1"; @@ -653,7 +659,9 @@ FROM config_alert_settings /* #6874 (V222): the Agent Alert Gaps switch at 79. #6873 (V223): the three database protection switches at 80-82. */ reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), /* #6875 (V224): the Identity Capacity switch and bars at 83-85. */ - reader.GetBoolean(83), reader.GetInt32(84), reader.GetInt32(85)); + reader.GetBoolean(83), reader.GetInt32(84), reader.GetInt32(85), + /* #6927 (V228): the four blocking band tiers at 86-89. */ + reader.GetDouble(86), reader.GetDouble(87), reader.GetDouble(88), reader.GetDouble(89)); } /* ─────────────────────── delivery cooldown (a SECOND config table) ─────────────────────── */ diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index 44262175a5..38c6c669c4 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -930,6 +930,8 @@ the production photo this answers. */ servers on the old pair and the rest on the new one — a mixed reading no configuration ever held, and the band counts and the worst-first ranking would be derived from it. */ var deadlockTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken); + /* #6927 (V228): the blocking band's tiers, read once per roll-up for the same reason. */ + var blockingTiers = await BlockingBandStore.ReadAsync(postgres, McpCommandDeadlines.ReadSeconds, cancellationToken); var identityCapacity = await ReadIdentityCapacityAsync(postgres, now, cancellationToken); var cards = new List(servers.Count); @@ -967,7 +969,7 @@ server the read reported with an empty window reads Offline or never-collected b cards.Add(BuildCard( server, c, pg, m, mp, t, b, deadlock, pgDeadlock, collected.LastCollection, collectors, serverTags, now, - windowEndUtc - windowStartUtc, deadlockTiers, collected.RegisteredAt, collected.OlderCollection, serverAgentAlertGaps, serverDatabaseProtection, serverIdentityCapacity)); + windowEndUtc - windowStartUtc, deadlockTiers, collected.RegisteredAt, collected.OlderCollection, serverAgentAlertGaps, serverDatabaseProtection, serverIdentityCapacity, blockingTiers)); } return BuildRollup(cards, now, windowStartUtc, windowEndUtc, worstCount, tagForest, collectionHealthAgeSeconds, @@ -1093,7 +1095,8 @@ internal static FleetServerCard BuildCard( DateTime? olderCollection = null, IReadOnlyList? agentAlertGaps = null, IReadOnlyList? databaseProtectionOverdue = null, - FleetIdentityCapacity? identityCapacity = null) + FleetIdentityCapacity? identityCapacity = null, + BlockingBandThresholds? blockingTiers = null) { /* Lite's XE-preferred / DMV-fallback, per server: XE when it has any row this window, else the DMV @@ -1188,6 +1191,8 @@ because cpuForAlert there is percent of an allocation that moves. The source is arm, one metric over. */ DeadlockWindow = deadlockWindow, DeadlockRateThresholds = deadlockTiers, + /* #6927: the blocking tiers travel with the count and the window, for the reason the deadlock ones do. */ + BlockingBandThresholds = blockingTiers, TotalThreads = threads.TotalThreads, AvailableThreads = availableThreads, ThreadsWaitingForCpu = threads.RunnableTasks, @@ -1268,7 +1273,7 @@ deliberately not surfaced. */ ? ServerHealthClassifier.BlockingRatePerHour(blockingForBand.Value, deadlockWindow) : null, BlockingWindow = deadlockWindow, - BlockingSeverity = ServerHealthClassifier.BlockingSeverity(blockingForBand, maxBlockedSeconds, deadlockWindow), + BlockingSeverity = ServerHealthClassifier.BlockingSeverity(blockingForBand, maxBlockedSeconds, deadlockWindow, blockingTiers), DeadlockCount = deadlockCount, DeadlockLastSeen = deadlockLastSeen, DeadlockMeasured = deadlockMeasured, @@ -1281,6 +1286,7 @@ over. Null exactly when the PostgreSQL arm took no difference (#3539). */ : null, DeadlockWindow = deadlockWindow, DeadlockRateThresholds = deadlockTiers, + BlockingBandThresholds = blockingTiers, /* #6875: set only when a column is at the warning bar or past it. The Band below is not read from it. */ IdentityCapacity = identityCapacity, DeadlockSeverity = ServerHealthClassifier.DeadlockSeverity( @@ -1333,6 +1339,11 @@ public static FleetOverviewResult BuildRollup( var failures = 0; long totalBlocking = 0; long totalDeadlocks = 0; + /* #6927: each totals tile takes the worst band among the cards for that signal, the classification the cards already show. + A fleet total is not a per-server rate: ten servers at 3 an hour each would otherwise band the sum of 30 against the + per-server tiers and turn Critical with no card even Warning. */ + var blockingTileSeverity = HealthSeverity.Healthy; + var deadlockTileSeverity = HealthSeverity.Healthy; var deadlockSourcesRead = 0; var deadlockPostgresTargets = 0; var deadlockCollectorsSilent = 0; @@ -1376,6 +1387,8 @@ attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on totalBlocking += card.BlockingCount; totalDeadlocks += card.DeadlockCount; + if (card.BlockingSeverity > blockingTileSeverity) blockingTileSeverity = card.BlockingSeverity; + if (card.DeadlockSeverity > deadlockTileSeverity) deadlockTileSeverity = card.DeadlockSeverity; } var problems = cards @@ -1411,6 +1424,8 @@ attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on ServersWithCollectionFailures = failures, TotalBlockingEvents = totalBlocking, TotalDeadlocks = totalDeadlocks, + TotalBlockingSeverity = blockingTileSeverity, + TotalDeadlockSeverity = deadlockTileSeverity, DeadlockCoverage = new FleetDeadlockCoverage { ServersRead = deadlockSourcesRead, @@ -2952,6 +2967,11 @@ public sealed class FleetServerCard [JsonIgnore] public DeadlockRateThresholds DeadlockRateThresholds { get; init; } + /// The blocking tiers this card banded on (#6927), or null for today's constants. Carried, not serialized, for + /// , so the worst-first score re-bands on the card's own tiers. + [JsonIgnore] + public BlockingBandThresholds? BlockingBandThresholds { get; init; } + /// This server's deadlock-source collector band over the trailing seven days of collection /// health (#3017) — the fact that explains a of zero. The collector is /// deadlocks on SQL Server and pg_database_stats on PostgreSQL (#3539). Null when that @@ -3087,6 +3107,7 @@ was taken - see DeadlockMeasured. */ dot used the store's. */ DeadlockWindow = DeadlockWindow, DeadlockRateThresholds = DeadlockRateThresholds, + BlockingBandThresholds = BlockingBandThresholds, TotalThreads = TotalThreads, AvailableThreads = AvailableThreads, ThreadsWaitingForCpu = ThreadsWaitingForCpu, @@ -3152,6 +3173,13 @@ public sealed class FleetOverviewResult [JsonPropertyName("total_blocking_events")] public long TotalBlockingEvents { get; init; } [JsonPropertyName("total_deadlocks")] public long TotalDeadlocks { get; init; } + /// The Blocking totals tile's band (#6927): the fleet's blocked-process reports per hour and longest block against the + /// store's blocking tiers, through the same classifier the cards use. Unknown when no card has a blocking source. + [JsonPropertyName("total_blocking_severity")] public HealthSeverity TotalBlockingSeverity { get; init; } + + /// The Deadlocks totals tile's band (#6927): the fleet's deadlocks per hour against the store's deadlock tiers. + [JsonPropertyName("total_deadlock_severity")] public HealthSeverity TotalDeadlockSeverity { get; init; } + /// How much of the fleet actually read a deadlock source for, with /// the causes named (#3017). Never null — a total with no denominator beside it is the defect. [JsonPropertyName("deadlock_coverage")] public FleetDeadlockCoverage DeadlockCoverage { get; init; } = new(); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs index ef80e8d186..8026178ac3 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs @@ -275,6 +275,10 @@ public sealed record DailySummaryReadRow( /// defaults holds anyway. public DeadlockRateThresholds RateTiers { get; init; } = DeadlockRateThresholds.Default; + /// The blocking band's tiers the day is banded on (#6927), the store's values for the same reason + /// is. + public BlockingBandThresholds BlockingTiers { get; init; } = BlockingBandThresholds.Default; + /// The clock the still-forming day's window clamps against (#3525 review): anchored /// reads hand their resolved window end so a backdated as_of clamps against its own "now"; /// unanchored reads (the explicit-date tool, the viewer path) band against the wall clock. @@ -337,7 +341,7 @@ have not happened yet (review finding on #3525). The fleet sweep does NOT read t }; public DailyHealthBand HealthBand => - DailyHealthBandCalculator.Classify(ToSignals(), new DailyHealthThresholds { DeadlockRates = RateTiers }); + DailyHealthBandCalculator.Classify(ToSignals(), new DailyHealthThresholds { DeadlockRates = RateTiers, BlockingBands = BlockingTiers }); /// Human label for the band ("Healthy" / "Warning" / "Critical" / "No Data"). public string OverallHealth => DailyHealthBandCalculator.Label(HealthBand); @@ -496,6 +500,7 @@ a floor moves back only on a backfill (stale, it under-claims, so a window stays DarlingFleetReader's own hoist argument: a settings write mid-read must not band some days on the old pair and the rest on the new one. */ var rateTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken); + var blockingTiers = await BlockingBandStore.ReadAsync(postgres, McpCommandDeadlines.ReadSeconds, cancellationToken); /* #3541 A9: the retention horizon, from the store's effective retention and the READER's wall clock. The clock is deliberately NOT the caller's anchor — a purge is a wall-clock event and a backdated @@ -592,6 +597,7 @@ twice McpCommandDeadlines.ReadSeconds. */ var results = rawResults.Select(row => row with { RateTiers = rateTiers, + BlockingTiers = blockingTiers, ReferenceUtc = referenceUtc ?? DateTime.UtcNow, DataState = DailySummaryRetention.StateFor(row.SummaryDate, row.CollectionRuns, row.SignalSourcesPresent, horizon), RetentionHorizon = horizon, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs index b2686932c9..ff5a1824ea 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs @@ -477,7 +477,12 @@ invite tuning one and expecting the other to move. */ health_bands = new { deadlock_warn_per_hour = s.DeadlockWarnPerHour, - deadlock_critical_per_hour = s.DeadlockCriticalPerHour + deadlock_critical_per_hour = s.DeadlockCriticalPerHour, + /* #6927 (V228): the blocking band, beside the deadlock one: reports per hour and the longest single block in seconds. */ + blocking_warn_per_hour = s.BlockingWarnPerHour, + blocking_critical_per_hour = s.BlockingCriticalPerHour, + blocking_warn_wait_seconds = s.BlockingWarnWaitSeconds, + blocking_critical_wait_seconds = s.BlockingCriticalWaitSeconds }, poison_wait = new { @@ -1205,6 +1210,11 @@ internal static JsonObject BuildMuteRuleWithState(MuteRule r, DateTime nowUtc) "per hour is the tightest setting that is still a rate, so no value here can restore the 'any deadlock " + "is Critical' reading these tiers replaced. Setting critical BELOW warn is accepted and means every " + "banded rate is Critical. " + + "health_bands.blocking_warn_per_hour and health_bands.blocking_critical_per_hour are the blocking band's tiers in blocked-process reports per HOUR " + + "(accepting 1 to 10000), and health_bands.blocking_warn_wait_seconds and health_bands.blocking_critical_wait_seconds are its bars on the longest " + + "single block in seconds (accepting 1 to 86400). They default to 5 and 20 reports per hour and 10 and 60 seconds, and they decide the colour " + + "of the fleet cards, the Overview tab, the Daily Health Calendar and get_fleet_overview, so set them to match the blocking alert's own " + + "threshold. Unlike the deadlock tiers, each blocking pair must keep critical at or above warn, and a write that would leave it below is refused. " + "file_growth.rise_mb is megabytes per HOUR averaged over file_growth.lookback_minutes (a rate — the same 10240 is 10 GB/hr on any lookback; the engine scales it to the window), so shortening the lookback does not tighten the rise gate and lengthening it does not loosen it; only the rate does. " + "Two keys govern the PostgreSQL versions of the two count alerts and are NOT the same numbers as their SQL Server neighbours: deadlocks.pg_count_threshold and blocking.pg_count_threshold, both accepting 1 upward. They sit inside those groups rather than a section of their own so both engines' figures are visible together, but tuning deadlocks.count_threshold does NOT move the PostgreSQL gate and tuning deadlocks.pg_count_threshold does NOT move the SQL Server one. The enabled switch in each group DOES govern both engines. They are separate because the two engines count with different instruments - SQL Server's figure is captured deadlock graphs, the PostgreSQL one is deadlocks parsed from the server log (get_pg_deadlocks) - and an operator tuning one should not silently move the other. Both engines' fleet cards now band deadlocks through the SAME health_bands.deadlock_warn_per_hour tiers (the PostgreSQL card differences the server's own pg_stat_database.deadlocks counter over the window), so the move - raising a fire gate to meet the band's Warning bar so a page and an amber dot describe the same server - is available on either knob. On the blocking side the SQL Server count is engine-recorded blocked-process reports and the PostgreSQL one is distinct root blockers in a periodic SAMPLE of pg_stat_activity. Both PostgreSQL keys are ignored on a store with no PostgreSQL targets. " + "The fleet_sweep group is NOT an alert family and the alerts_enabled master switch does not govern it: " + @@ -1384,6 +1394,29 @@ Widening that allowlist to admit a transaction would have let a real target comm } } + /* #6927: the same lone-bar rule for the two blocking band pairs. */ + if (updates.Any(u => u.Column is BlockingWarnPerHourColumn or BlockingCriticalPerHourColumn or BlockingWarnWaitColumn or BlockingCriticalWaitColumn)) + { + const string blockingBandSql = "SELECT blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1"; + await using var blockingBandCommand = new NpgsqlCommand(blockingBandSql, connection) { Transaction = transaction }; + blockingBandCommand.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var blockingBandReader = await blockingBandCommand.ExecuteReaderAsync(); + if (await blockingBandReader.ReadAsync()) + { + if (blockingBandReader.GetDouble(1) < blockingBandReader.GetDouble(0)) + { + return Outcome("invalid", BlockingRateBarsOrderMessage + + $" (stored after this write: warn {blockingBandReader.GetDouble(0).ToString("0.###", CultureInfo.InvariantCulture)}, critical {blockingBandReader.GetDouble(1).ToString("0.###", CultureInfo.InvariantCulture)}; nothing was written.)"); + } + + if (blockingBandReader.GetDouble(3) < blockingBandReader.GetDouble(2)) + { + return Outcome("invalid", BlockingWaitBarsOrderMessage + + $" (stored after this write: warn {blockingBandReader.GetDouble(2).ToString("0.###", CultureInfo.InvariantCulture)}, critical {blockingBandReader.GetDouble(3).ToString("0.###", CultureInfo.InvariantCulture)}; nothing was written.)"); + } + } + } + /* #5640: the same lone-bar rule for the Client Timeouts pair. */ if (updates.Any(u => u.Column == ClientTimeoutsWarningColumn || u.Column == ClientTimeoutsCriticalColumn)) { @@ -2586,6 +2619,14 @@ private static string AlreadyExists(MuteRule existing) => "'client_timeouts.critical_multiple' must not be lower than 'client_timeouts.warning_multiple' (the Critical bar can never sit below the Warning bar)."; private const string ClientTimeoutsBarsOrderMessage = "'client_timeouts.critical_per_minute' must not be lower than 'client_timeouts.warning_per_minute' (the Critical bar can never sit below the Warning bar)."; + private const string BlockingWarnPerHourColumn = "blocking_warn_per_hour"; + private const string BlockingCriticalPerHourColumn = "blocking_critical_per_hour"; + private const string BlockingWarnWaitColumn = "blocking_warn_wait_seconds"; + private const string BlockingCriticalWaitColumn = "blocking_critical_wait_seconds"; + private const string BlockingRateBarsOrderMessage = + "'health_bands.blocking_critical_per_hour' must not be lower than 'health_bands.blocking_warn_per_hour' (the Critical bar can never sit below the Warning bar)."; + private const string BlockingWaitBarsOrderMessage = + "'health_bands.blocking_critical_wait_seconds' must not be lower than 'health_bands.blocking_warn_wait_seconds' (the Critical bar can never sit below the Warning bar)."; private const string RunnableQueueBarsOrderMessage = "'runnable_queue.critical_per_scheduler' must not be lower than 'runnable_queue.warning_per_scheduler' (the Critical bar can never sit below the Warning bar)."; @@ -2963,6 +3004,27 @@ threshold that cannot reach a bare-count band. */ ServerHealthThresholds.DeadlockRatePerHourFloor, ServerHealthThresholds.DeadlockRatePerHourCeiling); break; + /* #6927 (V228): the blocking band, same named bounds BlockingBandThresholds clamps to. */ + case "blocking_warn_per_hour": + AddDouble("blocking_warn_per_hour", n, "health_bands.blocking_warn_per_hour", + ServerHealthThresholds.BlockingRatePerHourFloor, + ServerHealthThresholds.BlockingRatePerHourCeiling); + break; + case "blocking_critical_per_hour": + AddDouble("blocking_critical_per_hour", n, "health_bands.blocking_critical_per_hour", + ServerHealthThresholds.BlockingRatePerHourFloor, + ServerHealthThresholds.BlockingRatePerHourCeiling); + break; + case "blocking_warn_wait_seconds": + AddDouble("blocking_warn_wait_seconds", n, "health_bands.blocking_warn_wait_seconds", + ServerHealthThresholds.BlockingWaitSecondsFloor, + ServerHealthThresholds.BlockingWaitSecondsCeiling); + break; + case "blocking_critical_wait_seconds": + AddDouble("blocking_critical_wait_seconds", n, "health_bands.blocking_critical_wait_seconds", + ServerHealthThresholds.BlockingWaitSecondsFloor, + ServerHealthThresholds.BlockingWaitSecondsCeiling); + break; default: error = $"Unknown field 'health_bands.{k}'."; break; } }); @@ -3375,6 +3437,24 @@ is a Postgres error (multiple assignments to the same column) -- so refusing it error = RunnableQueueBarsOrderMessage; } + /* #6927: both bars of a blocking band pair in one body must already be in order; a lone bar is compared with the stored other one + inside UpdateAlertSettings' transaction (the builder has no store). */ + if (error == null + && updates.FirstOrDefault(u => u.Column == BlockingWarnPerHourColumn) is { Param: NpgsqlParameter blockWarnRate } + && updates.FirstOrDefault(u => u.Column == BlockingCriticalPerHourColumn) is { Param: NpgsqlParameter blockCriticalRate } + && blockCriticalRate.TypedValue < blockWarnRate.TypedValue) + { + error = BlockingRateBarsOrderMessage; + } + + if (error == null + && updates.FirstOrDefault(u => u.Column == BlockingWarnWaitColumn) is { Param: NpgsqlParameter blockWarnWait } + && updates.FirstOrDefault(u => u.Column == BlockingCriticalWaitColumn) is { Param: NpgsqlParameter blockCriticalWait } + && blockCriticalWait.TypedValue < blockWarnWait.TypedValue) + { + error = BlockingWaitBarsOrderMessage; + } + /* #5640: both Client Timeouts bars in one body must already be in order; a lone bar is compared with the stored other one. */ if (error == null && updates.FirstOrDefault(u => u.Column == ClientTimeoutsWarningColumn) is { Param: NpgsqlParameter timeoutWarningParam } diff --git a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs index 965aa93a3a..0c6883ad1f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs +++ b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs @@ -1431,11 +1431,12 @@ INSERT INTO config_alert_settings ( excluded_volumes, client_timeouts_warning_multiple, client_timeouts_critical_multiple, agent_alert_gaps_enabled, backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, - identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent) + identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, + blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds) VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46, $47, $48, $49, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, - $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86) + $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86, $87, $88, $89, $90) ON CONFLICT (id) DO NOTHING", connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; command.Parameters.AddWithValue(a.Enabled); command.Parameters.AddWithValue(a.CpuEnabled); @@ -1565,6 +1566,11 @@ apply and get_alert_settings reports back. */ command.Parameters.AddWithValue(a.IdentityCapacityEnabled); command.Parameters.AddWithValue(a.IdentityCapacityWarningPercent); command.Parameters.AddWithValue(a.IdentityCapacityCriticalPercent); + /* #6927, bound last in the order the V228 columns were appended. Seeded RAW like every sibling: the clamp lives on BlockingBandThresholds. */ + command.Parameters.AddWithValue(a.BlockingWarnPerHour); + command.Parameters.AddWithValue(a.BlockingCriticalPerHour); + command.Parameters.AddWithValue(a.BlockingWarnWaitSeconds); + command.Parameters.AddWithValue(a.BlockingCriticalWaitSeconds); await command.ExecuteNonQueryAsync(ct); } @@ -2204,7 +2210,8 @@ internal static string NormalizePlanXmlCompression(string? value) => excluded_volumes, client_timeouts_warning_multiple, client_timeouts_critical_multiple, agent_alert_gaps_enabled, backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, - identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent + identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, + blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds FROM config_alert_settings WHERE id = 1", connection) { CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds }; using var reader = await command.ExecuteReaderAsync(ct); if (!await reader.ReadAsync(ct)) @@ -2375,6 +2382,11 @@ selected but not read here -- or read but not selected -- would reset the list o IdentityCapacityEnabled = reader.GetBoolean(83), IdentityCapacityWarningPercent = reader.GetInt32(84), IdentityCapacityCriticalPercent = reader.GetInt32(85), + /* #6927 (V228): the four blocking band tiers at 86-89 (same reachability rule as every appended knob). */ + BlockingWarnPerHour = reader.GetDouble(86), + BlockingCriticalPerHour = reader.GetDouble(87), + BlockingWarnWaitSeconds = reader.GetDouble(88), + BlockingCriticalWaitSeconds = reader.GetDouble(89), }; var analysis = new AnalysisConfig { diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js index ee27ef9ebf..caf8e871af 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/admin.js @@ -196,7 +196,8 @@ export const GROUP_FIELDS = { cpu: ["enabled", "threshold_percent", "mode"], blocking: ["enabled", "count_threshold", "wait_threshold_seconds", "pg_count_threshold"], deadlocks: ["enabled", "count_threshold", "pg_count_threshold"], - health_bands: ["deadlock_warn_per_hour", "deadlock_critical_per_hour"], + health_bands: ["deadlock_warn_per_hour", "deadlock_critical_per_hour", "blocking_warn_per_hour", "blocking_critical_per_hour", + "blocking_warn_wait_seconds", "blocking_critical_wait_seconds"], poison_wait: ["enabled", "threshold_ms"], runnable_queue: ["enabled", "warning_per_scheduler", "critical_per_scheduler"], client_timeouts: ["enabled", "warning_per_minute", "critical_per_minute", "warning_multiple", "critical_multiple"], diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js index 643e07fde6..69590932a7 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js @@ -705,7 +705,9 @@ export function rollup(d) { not a confirmed incident either — warning is the "interrogate this number" class without claiming a deadlock happened. A real deadlock keeps critical whatever the coverage: a measured incident outranks an incomplete denominator, and the sub-line under it already says the count may be short. */ - const deadlockClass = d.total_deadlocks > 0 ? "critical" : deadlockSub && deadlockSub.partial ? "warning" : ""; + /* #6927: banded by the service through the same classifier and the store's tiers as the cards, not on any non-zero count. */ + const totalClass = (sev) => (sev === "Critical" ? "critical" : sev === "Warning" ? "warning" : ""); + const deadlockClass = totalClass(d.total_deadlock_severity) || (deadlockSub && deadlockSub.partial ? "warning" : ""); /* Two fixed groups (server-band counts | event counts) split by a divider; a non-zero blocking / deadlock total takes a severity color. */ @@ -719,7 +721,7 @@ export function rollup(d) { ]), el("div", { class: "rollup-divider" }), el("div", { class: "rollup-group" }, [ - tile(d.total_blocking_events, "Blocking (recent)", d.total_blocking_events > 0 ? "warning" : ""), + tile(d.total_blocking_events, "Blocking (recent)", totalClass(d.total_blocking_severity)), /* The API's own note rides along as the tile's title — the causes and the two windows, in the words the service already renders for get_fleet_overview, so the hover is DETAIL on a signal that is already visible rather than the only place the signal exists. */ diff --git a/Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs b/Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs new file mode 100644 index 0000000000..c854472ffc --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/BlockingBandStore.cs @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Storage; + +/// +/// The one read of the blocking health band's four tiers (#6927, V228) from the store's singleton settings row. It lives in Storage because +/// every Darling reader of the band needs the same values: the fleet cards and get_fleet_overview, the fleet sweep, the Daily Health +/// Calendar and get_daily_health in the service, and the Overview tab, the calendar and the fleet grid in the viewer. Values come back +/// RAW and clamps on read. A store with no row yet bands on today's constants, which is what such a +/// store would seed. +/// +public static class BlockingBandStore +{ + /// The singleton row's four tiers. No parameters: this is the current configuration, not history. + public const string ReadSql = @" +SELECT blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds +FROM config_alert_settings +WHERE id = 1"; + + /// Reads the tiers, or when the row is absent. + public static async Task ReadAsync( + NpgsqlDataSource postgres, int commandTimeoutSeconds, CancellationToken cancellationToken) + { + await using var command = postgres.CreateCommand(ReadSql); + command.CommandTimeout = commandTimeoutSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return new BlockingBandThresholds( + reader.GetDouble(0), reader.GetDouble(1), reader.GetDouble(2), reader.GetDouble(3)); + } + + return BlockingBandThresholds.Default; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index 8ca7eb987e..5b7a30ad75 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -2976,6 +2976,7 @@ holds the ordering. */ new Migration(225, "server-aws-rds", V225Sql), new Migration(226, "server-aws-rds-web-or-express", V226Sql), new Migration(227, "threshold-override-tag-scope", V227Sql), + new Migration(228, "blocking-band-tiers", V228Sql), }; /// @@ -3046,6 +3047,24 @@ ALTER TABLE collect.servers ALTER TABLE collect.servers ADD COLUMN IF NOT EXISTS is_aws_rds_web_or_express boolean;"; + /// + /// V228 (#6927): the blocking health band's four tiers on the alert settings row, beside the V120 deadlock ones. They carry the + /// figures ServerHealthThresholds.BlockingWarnPerHour, BlockingCriticalPerHour, BlockingWarnWaitSeconds and + /// BlockingCriticalWaitSeconds have always been, so a store that never sets them bands exactly as before. They are literals here + /// because SQL text cannot name a C# constant. No CHECK ordering the two tiers of a pair, for the reason V120 gives: the writers refuse + /// a pair with critical below warn, and a hand-edited row reads as written. Idempotent: every column is added with IF NOT EXISTS, so a + /// store that already has them is left as it is. Table-level grants already cover new columns on this table, so no GRANT is added. + /// + private const string V228Sql = @" +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_warn_per_hour double precision NOT NULL DEFAULT 5.0; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_critical_per_hour double precision NOT NULL DEFAULT 20.0; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_warn_wait_seconds double precision NOT NULL DEFAULT 10.0; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS blocking_critical_wait_seconds double precision NOT NULL DEFAULT 60.0;"; + /// /// V189 (#5867): collect.query_store_compose_stamp_failures, the failure record of the Query Store compose builder: one row /// per hour whose last build threw, with its consecutive failures, last error and next attempt, so a build that cannot finish backs diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs index e669ce62e7..0db43132aa 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs @@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class StorageVersion { - public const int SchemaVersion = 227; + public const int SchemaVersion = 228; } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs index 98886ade29..7dd680f253 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/MainWindow.xaml.cs @@ -1669,6 +1669,17 @@ is exactly why DarlingFleetReader.GetFleetOverviewAsync hoists its own copy of t deadlockTiers = DeadlockRateThresholds.Default; } + /* #6927: the blocking band's tiers, hoisted like the deadlock ones above and for the same reasons. */ + BlockingBandThresholds blockingTiers; + try + { + blockingTiers = await service.GetBlockingBandThresholdsAsync(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + blockingTiers = BlockingBandThresholds.Default; + } + var perLane = await Task.WhenAll(lanes.Select(async lane => { var found = new List(lane.Count); @@ -1679,6 +1690,8 @@ is exactly why DarlingFleetReader.GetFleetOverviewAsync hoists its own copy of t { var summary = await service.GetServerSummaryAsync( server.ServerId, server.DisplayName, deadlockTiers); + /* #6927: the store's blocking tiers; the card's blocking band and its re-band read them from the item. */ + summary.BlockingBandThresholds = blockingTiers; summary.ServerName = server.ServerName; /* #3029: the engine discriminator comes from the REGISTRY row, which already carries it (servers.engine_kind, via ManagedServersSql / ServersSql) — the per-server summary diff --git a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml index 0ca93eac7f..e05447a6b1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml @@ -445,6 +445,26 @@ + + + + + + + + + + + + + + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs index 025787b455..7b4fe4dd7d 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs @@ -923,6 +923,10 @@ and the alert text agree about what the number looks like. */ card agree about what the number looks like. */ BandDeadlockWarnPerHourBox.Text = r.DeadlockWarnPerHour.ToString("0.0", CultureInfo.InvariantCulture); BandDeadlockCriticalPerHourBox.Text = r.DeadlockCriticalPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnPerHourBox.Text = r.BlockingWarnPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalPerHourBox.Text = r.BlockingCriticalPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnWaitSecondsBox.Text = r.BlockingWarnWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalWaitSecondsBox.Text = r.BlockingCriticalWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); AlertPvsCheckBox.IsChecked = r.PvsEnabled; AlertPvsThresholdPercentBox.Text = r.PvsThresholdPercent.ToString(CultureInfo.InvariantCulture); AlertPvsFloorGbBox.Text = r.PvsFloorGb.ToString(CultureInfo.InvariantCulture); @@ -967,6 +971,23 @@ seeded from the row like every other alert-engine control. */ UpdateAlertControlStates(); } + /// Parses one blocking band box (#6927): InvariantCulture, within the named bounds the classifier clamps to and the MCP writer + /// accepts. A bad value adds an error naming the box and the range, and leaves null. + private static void BlockingBandBox( + System.Windows.Controls.TextBox box, double min, double max, string label, SettingsValidationErrors errors, out double? value) + { + if (double.TryParse(box.Text, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) && parsed >= min && parsed <= max) + { + value = parsed; + return; + } + + value = null; + errors.Add( + $"{label} must be a number between {min.ToString("0.0", CultureInfo.InvariantCulture)} and {max.ToString("0.0", CultureInfo.InvariantCulture)}.", + box); + } + /// Builds the store row from the alert + analysis controls, validating the numeric fields. Adds /// range errors to ; a bad field keeps its default rather than throwing. internal AlertSettingsRow BuildAlertRowFromControls(SettingsValidationErrors errors) @@ -1162,6 +1183,37 @@ not a pair to reject. */ && dlCritical >= ServerHealthThresholds.DeadlockRatePerHourFloor && dlCritical <= ServerHealthThresholds.DeadlockRatePerHourCeiling) row.DeadlockCriticalPerHour = dlCritical; + /* #6927: the blocking band, validated against the SAME named bounds BlockingBandThresholds clamps to and the MCP writer accepts, + InvariantCulture on the parse. Unlike the deadlock pair, a pair with critical below warn is refused, the rule update_alert_settings + applies, so the window cannot save a pair the tool would refuse. */ + BlockingBandBox(BandBlockingWarnPerHourBox, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling, + "Blocking warn per hour", errors, out var blWarnRate); + BlockingBandBox(BandBlockingCriticalPerHourBox, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling, + "Blocking critical per hour", errors, out var blCriticalRate); + BlockingBandBox(BandBlockingWarnWaitSecondsBox, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling, + "Blocking warn seconds", errors, out var blWarnWait); + BlockingBandBox(BandBlockingCriticalWaitSecondsBox, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling, + "Blocking critical seconds", errors, out var blCriticalWait); + if (blWarnRate.HasValue && blCriticalRate.HasValue) + { + if (blCriticalRate.Value < blWarnRate.Value) + errors.Add("Blocking critical per hour must not be lower than blocking warn per hour.", BandBlockingCriticalPerHourBox); + else + { + row.BlockingWarnPerHour = blWarnRate.Value; + row.BlockingCriticalPerHour = blCriticalRate.Value; + } + } + if (blWarnWait.HasValue && blCriticalWait.HasValue) + { + if (blCriticalWait.Value < blWarnWait.Value) + errors.Add("Blocking critical seconds must not be lower than blocking warn seconds.", BandBlockingCriticalWaitSecondsBox); + else + { + row.BlockingWarnWaitSeconds = blWarnWait.Value; + row.BlockingCriticalWaitSeconds = blCriticalWait.Value; + } + } if (int.TryParse(AlertPvsThresholdPercentBox.Text, out var pvsPct) && pvsPct is >= 0 and <= 100) row.PvsThresholdPercent = pvsPct; if (int.TryParse(AlertPvsFloorGbBox.Text, out var pvsFloor) && pvsFloor >= 0) @@ -1292,6 +1344,14 @@ the grid moved would arm the alert past the point it exists to precede. */ ServerHealthThresholds.DeadlockWarnPerHourDefault.ToString("0.0", CultureInfo.InvariantCulture); BandDeadlockCriticalPerHourBox.Text = ServerHealthThresholds.DeadlockCriticalPerHourDefault.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnPerHourBox.Text = + ServerHealthThresholds.BlockingWarnPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalPerHourBox.Text = + ServerHealthThresholds.BlockingCriticalPerHour.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingWarnWaitSecondsBox.Text = + ServerHealthThresholds.BlockingWarnWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); + BandBlockingCriticalWaitSecondsBox.Text = + ServerHealthThresholds.BlockingCriticalWaitSeconds.ToString("0.0", CultureInfo.InvariantCulture); AnalysisNotifyCooldownBox.Text = "360"; AlertPvsThresholdPercentBox.Text = "40"; AlertPvsFloorGbBox.Text = "1"; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs index f92c77fb31..f97ae27e73 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs @@ -99,7 +99,9 @@ the same reason and under the same connect-time store gate. */ /* #6873 (V223): the three database protection alert switches (backup, log backup, CHECKDB overdue), APPENDED, same gate. */ "backup_overdue_enabled, log_backup_overdue_enabled, checkdb_overdue_enabled, " + /* #6875 (V224): the Identity Capacity alert's switch (off by default) and its two bars, APPENDED for the same reason and under the same gate. */ - "identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent"; + "identity_capacity_enabled, identity_capacity_warning_percent, identity_capacity_critical_percent, " + + /* #6927 (V228): the blocking health band's four tiers, APPENDED for the same reason and under the same connect-time store gate. */ + "blocking_warn_per_hour, blocking_critical_per_hour, blocking_warn_wait_seconds, blocking_critical_wait_seconds"; /// The single global alert-settings row (id=1), for the Settings window prefill + the migrate-in /// defaults check. Column order matches . @@ -116,7 +118,7 @@ INSERT INTO config_alert_settings (id, " + AlertSettingsColumns + @", modified_a VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46, $47, $48, $49, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, - $63, $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86, + $63, $64, $65, $66, $67, $68, $69, $70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85, $86, $87, $88, $89, $90, (now() AT TIME ZONE 'UTC')) ON CONFLICT (id) DO UPDATE SET enabled = EXCLUDED.enabled, @@ -205,6 +207,10 @@ ON CONFLICT (id) DO UPDATE SET identity_capacity_enabled = EXCLUDED.identity_capacity_enabled, identity_capacity_warning_percent = EXCLUDED.identity_capacity_warning_percent, identity_capacity_critical_percent = EXCLUDED.identity_capacity_critical_percent, + blocking_warn_per_hour = EXCLUDED.blocking_warn_per_hour, + blocking_critical_per_hour = EXCLUDED.blocking_critical_per_hour, + blocking_warn_wait_seconds = EXCLUDED.blocking_warn_wait_seconds, + blocking_critical_wait_seconds = EXCLUDED.blocking_critical_wait_seconds, modified_at = (now() AT TIME ZONE 'UTC')"; /// The two cpu_mode values the service honors (it compares case-insensitively against @@ -344,6 +350,11 @@ parameter keeps its type on the INSERT arm. */ command.Parameters.Add(new NpgsqlParameter { TypedValue = r.IdentityCapacityEnabled }); // $84 (#6875, V224) command.Parameters.Add(new NpgsqlParameter { TypedValue = identityWarning }); // $85 (#6875, V224) command.Parameters.Add(new NpgsqlParameter { TypedValue = Math.Clamp(r.IdentityCapacityCriticalPercent, identityWarning, 100) }); // $86 (#6875, V224) + /* #6927 (V228): the blocking band tiers, stored raw like the deadlock pair above them. */ + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingWarnPerHour }); // $87 (#6927, V228) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingCriticalPerHour }); // $88 (#6927, V228) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingWarnWaitSeconds }); // $89 (#6927, V228) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.BlockingCriticalWaitSeconds }); // $90 (#6927, V228) } private static AlertSettingsRow ReadAlertSettingsRow(NpgsqlDataReader reader) => new() @@ -457,6 +468,11 @@ parameter keeps its type on the INSERT arm. */ IdentityCapacityEnabled = reader.GetBoolean(83), IdentityCapacityWarningPercent = reader.GetInt32(84), IdentityCapacityCriticalPercent = reader.GetInt32(85), + /* #6927 (V228) the four blocking band tiers appended at ordinals 86-89. */ + BlockingWarnPerHour = reader.GetDouble(86), + BlockingCriticalPerHour = reader.GetDouble(87), + BlockingWarnWaitSeconds = reader.GetDouble(88), + BlockingCriticalWaitSeconds = reader.GetDouble(89), }; /// Maps the Settings window's CPU-mode combo tag ("Total"/"SqlOnly") to the store value. @@ -614,6 +630,11 @@ is MB per HOUR averaged over the lookback (#3539 A8c) -- the Settings row's labe public int IdentityCapacityWarningPercent { get; set; } = 70; /// #6875 (V224): percent used at which an identity column is critical. public int IdentityCapacityCriticalPercent { get; set; } = 90; + /// #6927 (V228): the blocking health band's tiers, today's constants by default. + public double BlockingWarnPerHour { get; set; } = ServerHealthThresholds.BlockingWarnPerHour; + public double BlockingCriticalPerHour { get; set; } = ServerHealthThresholds.BlockingCriticalPerHour; + public double BlockingWarnWaitSeconds { get; set; } = ServerHealthThresholds.BlockingWarnWaitSeconds; + public double BlockingCriticalWaitSeconds { get; set; } = ServerHealthThresholds.BlockingCriticalWaitSeconds; public bool LongRunningQueryEnabled { get; set; } = true; public int LongRunningQueryThresholdMinutes { get; set; } = 30; public bool TempDbSpaceEnabled { get; set; } = true; @@ -749,6 +770,10 @@ public bool ValueEquals(AlertSettingsRow other) && IdentityCapacityEnabled == other.IdentityCapacityEnabled && IdentityCapacityWarningPercent == other.IdentityCapacityWarningPercent && IdentityCapacityCriticalPercent == other.IdentityCapacityCriticalPercent + && Math.Abs(BlockingWarnPerHour - other.BlockingWarnPerHour) < 0.0001 + && Math.Abs(BlockingCriticalPerHour - other.BlockingCriticalPerHour) < 0.0001 + && Math.Abs(BlockingWarnWaitSeconds - other.BlockingWarnWaitSeconds) < 0.0001 + && Math.Abs(BlockingCriticalWaitSeconds - other.BlockingCriticalWaitSeconds) < 0.0001 && LongRunningQueryEnabled == other.LongRunningQueryEnabled && LongRunningQueryThresholdMinutes == other.LongRunningQueryThresholdMinutes && TempDbSpaceEnabled == other.TempDbSpaceEnabled diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs index ccbebc3da0..e2381642ac 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs @@ -124,6 +124,7 @@ public async Task> GetDailySummaryRangeAsync( var banding = new DailyHealthThresholds { DeadlockRates = await GetDeadlockRateThresholdsAsync(cancellationToken), + BlockingBands = await GetBlockingBandThresholdsAsync(cancellationToken), }; /* #3653: the horizon, read once per range like the banding tiers above — a schedule save mid-read must diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs index dae50e1780..b033997450 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs @@ -623,6 +623,11 @@ FROM config_alert_settings /// Values come back RAW; clamps on read, so a hand-edited row /// cannot drive a nonsense threshold. /// + /// The store's blocking-band tiers (#6927), or today's constants when the singleton row is absent. The same read + /// the service uses (). + public Task GetBlockingBandThresholdsAsync(CancellationToken cancellationToken = default) => + BlockingBandStore.ReadAsync(_dataSource, ViewerCommandDeadlines.CurrentInteractiveReadSeconds, cancellationToken); + public async Task GetDeadlockRateThresholdsAsync(CancellationToken cancellationToken = default) { await using var command = _dataSource.CreateCommand(DeadlockRateThresholdSql); @@ -1333,10 +1338,10 @@ decision the service's fleet card uses so the two cannot disagree about whether : null; /// Blocking band (#3539 A3) — a 60 s block Critical whatever the rate; else blocking events per - /// hour over the window against the shared tiers (20/hr Critical, 5/hr Warning); a 10 s block Warning; - /// no source Unknown. + /// hour over the window against the store's tiers (#6927, 20/hr Critical, 5/hr Warning by default); a 10 s block + /// Warning by default; no source Unknown. public HealthSeverity BlockingSeverity => - ServerHealthClassifier.BlockingSeverity(BlockingCountForBand, MaxBlockedSeconds, BlockingWindow); + ServerHealthClassifier.BlockingSeverity(BlockingCountForBand, MaxBlockedSeconds, BlockingWindow, BlockingBandThresholds); /// /// The window and cover (#3368) — the @@ -1354,6 +1359,10 @@ decision the service's fleet card uses so the two cannot disagree about whether /// about which numbers are in force. public DeadlockRateThresholds? DeadlockRateThresholds { get; set; } + /// The store's blocking-band tiers (#6927), or null to band on today's constants, set by the read from + /// config_alert_settings so this card and the service's fleet card agree. + public BlockingBandThresholds? BlockingBandThresholds { get; set; } + /// Deadlocks per HOUR over — what the band evaluates (#3368), or /// null when the window is too short to normalise. Rendered beside the count so the dot's reason is /// legible. @@ -1429,6 +1438,7 @@ no difference (#3539). */ contradicting itself, which is the shape #3281 fixed on the CPU arm. */ DeadlockWindow = DeadlockWindow, DeadlockRateThresholds = DeadlockRateThresholds, + BlockingBandThresholds = BlockingBandThresholds, TotalThreads = TotalThreads, AvailableThreads = AvailableThreads, ThreadsWaitingForCpu = ThreadsWaitingForCpu, diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs index 3d9e58a532..804d58102c 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs @@ -1380,7 +1380,10 @@ per the V71 finding. */ EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('collect.servers') AND attname = 'is_aws_rds_web_or_express' AND NOT attisdropped), /* V227 (#6926) adds the tag scope to the threshold overrides. Found by catalog like the V225 column above. Named only in this probe line, never in prose, per the V71 finding. */ - EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.config_threshold_overrides') AND attname = 'scope_tag_id' AND NOT attisdropped)"; + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.config_threshold_overrides') AND attname = 'scope_tag_id' AND NOT attisdropped), + /* V228 (#6927) adds the four settable blocking band columns to the alert settings row. Found by catalog like the V226 column above. The + warn per hour column is the sentinel, the first of the four. Named only in this probe line, never in prose, per the V71 finding. */ + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.config_alert_settings') AND attname = 'blocking_warn_per_hour' AND NOT attisdropped)"; /// The store schema version this viewer build requires — the highest migration it knows /// (). The connect-time gate blocks a store below this. @@ -1402,7 +1405,7 @@ per the V71 finding. */ await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201), reader.GetBoolean(202)); + return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164), reader.GetBoolean(165), reader.GetBoolean(166), reader.GetBoolean(167), reader.GetBoolean(168), reader.GetBoolean(169), reader.GetBoolean(170), reader.GetBoolean(171), reader.GetBoolean(172), reader.GetBoolean(173), reader.GetBoolean(174), reader.GetBoolean(175), reader.GetBoolean(176), reader.GetBoolean(177), reader.GetBoolean(178), reader.GetBoolean(179), reader.GetBoolean(180), reader.GetBoolean(181), reader.GetBoolean(182), reader.GetBoolean(183), reader.GetBoolean(184), reader.GetBoolean(185), reader.GetBoolean(186), reader.GetBoolean(187), reader.GetBoolean(188), reader.GetBoolean(189), reader.GetBoolean(190), reader.GetBoolean(191), reader.GetBoolean(192), reader.GetBoolean(193), reader.GetBoolean(194), reader.GetBoolean(195), reader.GetBoolean(196), reader.GetBoolean(197), reader.GetBoolean(198), reader.GetBoolean(199), reader.GetBoolean(200), reader.GetBoolean(201), reader.GetBoolean(202), reader.GetBoolean(203)); } return null; @@ -1427,7 +1430,7 @@ per the V71 finding. */ /// is unit-tested without a live store; any schema bump past the newest arm trips the pinning test that keeps /// this in step with . /// - internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false, bool hasThresholdOverrideTagScope = false) + internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false, bool hasStoreMetricsSizeEstimated = false, bool hasAgentLevel2Store = false, bool hasQueryTagsStore = false, bool hasPhaseCStore = false, bool hasQueryTagSightingsDatabaseKey = false, bool hasQueryStoreForcedPlans = false, bool hasServerActionSwitches = false, bool hasForceChecks = false, bool hasPseudonymKey = false, bool hasBlockingHeadEvidence = false, bool hasExcludedVolumes = false, bool hasForcedPlanVariantsAndGuides = false, bool hasAgPrimaryReplica = false, bool hasHypotheticalIndexLogin = false, bool hasCustomViewFolders = false, bool hasCustomAlertRuleExpiry = false, bool hasInheritableCollectorEnabled = false, bool hasChangeWatches = false, bool hasMuteRuleStartsAt = false, bool hasQueryStoreFetchTopN = false, bool hasAgentRouteTagGuard = false, bool hasClientTimeoutsMultiples = false, bool hasServerMachineIdentity = false, bool hasQueryStoreReplica = false, bool hasKnownPatterns = false, bool hasAgentTableIndexes = false, bool hasClientTimeoutsOnByDefault = false, bool hasQueryStoreHints = false, bool hasServerActionRecompile = false, bool hasQueryStoreSelfKeys = false, bool hasJobHistorySuccessIndex = false, bool hasQueryStoreHashHourly = false, bool hasAgentAlerts = false, bool hasDatabaseProtection = false, bool hasIdentityCapacity = false, bool hasServerAwsRds = false, bool hasServerAwsRdsWebOrExpress = false, bool hasThresholdOverrideTagScope = false, bool hasBlockingBands = false) { /* V71 (the PostgreSQL blocking-edges rung): a table-existence sentinel and now the newest-first arm. A collector table would ordinarily get no arm at all — see the V63-V69 note below — but the TOP @@ -1636,7 +1639,14 @@ store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through t /* V226 (#6893): the AWS RDS Web or Express fact on the server registry. Column-existence sentinel, newest-first, and the TOP rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 225 and showing a spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ - /* V227 (#6926): the tag scope on the threshold overrides. Catalog sentinel, the top rung, newest first. */ + /* V228 (#6927): the settable blocking band columns on the alert settings row. Column-existence sentinel, newest-first, and the TOP + rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 227 and showing a + spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + if (hasBlockingBands) + { + return 228; + } + /* V227 (#6926): the tag scope on the threshold overrides. Catalog sentinel, one rung under V228. */ if (hasThresholdOverrideTagScope) { return 227; diff --git a/PerformanceMonitor.Common/DailyHealthBand.cs b/PerformanceMonitor.Common/DailyHealthBand.cs index adb9176412..47a7dfbee0 100644 --- a/PerformanceMonitor.Common/DailyHealthBand.cs +++ b/PerformanceMonitor.Common/DailyHealthBand.cs @@ -211,6 +211,13 @@ public sealed record DailyHealthThresholds /// public DeadlockRateThresholds DeadlockRates { get; init; } = DeadlockRateThresholds.Default; + /// + /// The blocking band's tiers the day bands with (#6927) — the SAME store-backed values the Overview + /// card's blocking dot reads, so a day cell and the card cannot disagree about what a blocking rate or + /// a peak block means. Defaults to today's constants; Lite has no such knobs and bands on them. + /// + public BlockingBandThresholds BlockingBands { get; init; } = BlockingBandThresholds.Default; + /// The high-CPU sample count at or above which is Critical: the /// greater of and /// × the window's hours. An undeclared window @@ -275,7 +282,7 @@ arm are documented on ServerHealthThresholds. Delegating rather than restating i undeclared span) is the fallback this classifier wants: a non-zero count reads Warning, a zero count (Unknown) stays out of the blocking trigger. */ var blockingSeverity = ServerHealthClassifier.BlockingSeverity( - signals.BlockingReports ?? signals.BlockingEvents, signals.PeakBlockWaitMs / 1000.0, signals.Window); + signals.BlockingReports ?? signals.BlockingEvents, signals.PeakBlockWaitMs / 1000.0, signals.Window, t.BlockingBands); var cpuSeverity = HighCpuSeverity(signals.HighCpuEvents, signals.Window, t); var collectionSeverity = CollectionErrorSeverity(signals.CollectionErrors, signals.CollectionRuns); diff --git a/PerformanceMonitor.Common/ServerHealthBands.cs b/PerformanceMonitor.Common/ServerHealthBands.cs index c72b60d8c2..9450b9fc4d 100644 --- a/PerformanceMonitor.Common/ServerHealthBands.cs +++ b/PerformanceMonitor.Common/ServerHealthBands.cs @@ -448,6 +448,69 @@ public static class ServerHealthThresholds /// reach it first regardless. /// public const double DeadlockRatePerHourCeiling = 1000.0; + + /* ── the settable blocking band (#6927) ── */ + + /// + /// The FLOOR the settable blocking rate tiers clamp to (#6927): one event per hour, for the reason + /// gives. It keeps a tier a rate, and it keeps zero and + /// negative values, which would band every window Critical, out of the classifier. + /// + public const double BlockingRatePerHourFloor = 1.0; + + /// The CEILING the settable blocking rate tiers clamp to (#6927), so a tier stays a threshold and + /// not an undisclosed off switch. The worst measured server-hour held 232 reports, so 10,000 is far + /// above anything real. + public const double BlockingRatePerHourCeiling = 10000.0; + + /// The FLOOR the settable blocking wait tiers clamp to, seconds (#6927). A tier of zero would + /// band every block Critical. + public const double BlockingWaitSecondsFloor = 1.0; + + /// The CEILING the settable blocking wait tiers clamp to, seconds (#6927): one day. + public const double BlockingWaitSecondsCeiling = 86400.0; + } + + /// + /// The four settable tiers of the blocking band (#6927): events per hour and the longest single block in + /// seconds, each with a Warning and a Critical bar. The same shape as : + /// the store holds what the operator set, these accessors clamp on read, and a default struct + /// (every field zero) clamps to the floors rather than to zero. + /// + /// Defaults are today's constants ( and + /// its three siblings), so a store that never set them bands exactly as every build before this one + /// did. Lite has no store and keeps the constants. + /// + /// Like the deadlock pair, the critical bar is not floored at the warning bar here, so a hand-edited + /// pair reads as written. The writers refuse a pair with critical below warn. + /// + public readonly record struct BlockingBandThresholds( + double WarnPerHourRaw, + double CriticalPerHourRaw, + double WarnWaitSecondsRaw, + double CriticalWaitSecondsRaw) + { + /// Warning bar, events per hour, clamped. + public double WarnPerHour => Clamp(WarnPerHourRaw, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling); + + /// Critical bar, events per hour, clamped independently of the warning bar. + public double CriticalPerHour => Clamp(CriticalPerHourRaw, ServerHealthThresholds.BlockingRatePerHourFloor, ServerHealthThresholds.BlockingRatePerHourCeiling); + + /// Warning bar, longest block in seconds, clamped. + public double WarnWaitSeconds => Clamp(WarnWaitSecondsRaw, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling); + + /// Critical bar, longest block in seconds, clamped independently of the warning bar. + public double CriticalWaitSeconds => Clamp(CriticalWaitSecondsRaw, ServerHealthThresholds.BlockingWaitSecondsFloor, ServerHealthThresholds.BlockingWaitSecondsCeiling); + + /// Today's constants: what a store at its V228 column defaults hands the band. + public static BlockingBandThresholds Default => new( + ServerHealthThresholds.BlockingWarnPerHour, + ServerHealthThresholds.BlockingCriticalPerHour, + ServerHealthThresholds.BlockingWarnWaitSeconds, + ServerHealthThresholds.BlockingCriticalWaitSeconds); + + private static double Clamp(double value, double floor, double ceiling) => + double.IsNaN(value) ? floor : Math.Clamp(value, floor, ceiling); } /// @@ -582,6 +645,12 @@ public readonly record struct ServerHealthMetrics /// public DeadlockRateThresholds? DeadlockRateThresholds { get; init; } + /// The blocking band's four tiers (#6927), or null for today's constants + /// (). A Darling path with a store hands the + /// config_alert_settings values in so the card, the Overview tab, the calendar and the fleet + /// sweep band on what the operator set. Lite has no such knobs and leaves it null. + public BlockingBandThresholds? BlockingBandThresholds { get; init; } + /// Worker-thread ceiling (max_workers_count), or null with no scheduler snapshot (e.g. Azure SQL DB). public int? TotalThreads { get; init; } @@ -887,7 +956,8 @@ public static HealthSeverity MemorySeverity(bool? hasMemoryPressure) public static HealthSeverity BlockingSeverity( long? blockingCountOrNullWhenUnmeasured, double maxBlockedSeconds, - TimeSpan window) + TimeSpan window, + BlockingBandThresholds? thresholds = null) { if (!blockingCountOrNullWhenUnmeasured.HasValue) { @@ -895,19 +965,20 @@ public static HealthSeverity BlockingSeverity( } var blockingCount = blockingCountOrNullWhenUnmeasured.Value; + var tiers = thresholds ?? BlockingBandThresholds.Default; - if (maxBlockedSeconds >= ServerHealthThresholds.BlockingCriticalWaitSeconds) + if (maxBlockedSeconds >= tiers.CriticalWaitSeconds) { return HealthSeverity.Critical; } var ratePerHour = BlockingRatePerHour(blockingCount, window); - if (ratePerHour.HasValue && ratePerHour.Value >= ServerHealthThresholds.BlockingCriticalPerHour) + if (ratePerHour.HasValue && ratePerHour.Value >= tiers.CriticalPerHour) { return HealthSeverity.Critical; } - if (maxBlockedSeconds >= ServerHealthThresholds.BlockingWarnWaitSeconds) + if (maxBlockedSeconds >= tiers.WarnWaitSeconds) { return HealthSeverity.Warning; } @@ -917,7 +988,7 @@ public static HealthSeverity BlockingSeverity( return blockingCount > 0 ? HealthSeverity.Warning : HealthSeverity.Unknown; } - return ratePerHour.Value >= ServerHealthThresholds.BlockingWarnPerHour + return ratePerHour.Value >= tiers.WarnPerHour ? HealthSeverity.Warning : HealthSeverity.Healthy; } @@ -1140,7 +1211,7 @@ public static IEnumerable MetricSeverities(ServerHealthMetrics m yield return CpuSeverity(m.CpuPercentForAlert, m.CapacityUtilizationPercent, m.CpuSource); yield return ThreadsSeverity(m.TotalThreads, m.AvailableThreads, m.ThreadsWaitingForCpu, m.RequestsWaitingForThreads); yield return MemorySeverity(m.HasMemoryPressure); - yield return BlockingSeverity(m.BlockingCount, m.MaxBlockedSeconds, m.BlockingWindow); + yield return BlockingSeverity(m.BlockingCount, m.MaxBlockedSeconds, m.BlockingWindow, m.BlockingBandThresholds); yield return DeadlockSeverity( m.DeadlockCount, m.DeadlockWindow,