From f24d8a22690a1aca1c44531ae929e795ca488191 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:47:54 -0400 Subject: [PATCH 01/42] Web time-series charts break their line at collection gaps (#5644 W7) --- .../WebChartGapBehaviourTests.cs | 166 ++++++++++++++++++ .../Darling.Tests/web-chart-gap-harness.mjs | 160 +++++++++++++++++ .../wwwroot/js/charts.js | 151 +++++++++++----- 3 files changed, 429 insertions(+), 48 deletions(-) create mode 100644 Darling/Darling.Tests/WebChartGapBehaviourTests.cs create mode 100644 Darling/Darling.Tests/web-chart-gap-harness.mjs diff --git a/Darling/Darling.Tests/WebChartGapBehaviourTests.cs b/Darling/Darling.Tests/WebChartGapBehaviourTests.cs new file mode 100644 index 0000000000..67c216e9af --- /dev/null +++ b/Darling/Darling.Tests/WebChartGapBehaviourTests.cs @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Linq; +using System.Text.Json; +using Xunit; +using static Darling.Tests.RepoFile; + +namespace Darling.Tests; + +/// +/// A web time-series chart breaks its line, area and stacked bands where no samples were collected (#5644 W7): the service host +/// is off overnight, and a line drawn point to point across the silence read as data. These run the shipped +/// charts.js under Node (web-chart-gap-harness.mjs). Skipped when Node is not installed. +/// +public sealed class WebChartGapBehaviourTests +{ + private static JsonElement Run() + { + var psi = new ProcessStartInfo("node") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false }; + psi.ArgumentList.Add(PathTo("Darling", "Darling.Tests", "web-chart-gap-harness.mjs")); + psi.ArgumentList.Add(PathTo("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js")); + + Process proc; + try + { + proc = Process.Start(psi)!; + } + catch (Win32Exception) + { + Assert.Skip("Node is not installed, so the shipped chart script cannot be run."); + return default; + } + + using (proc) + { + var error = proc.StandardError.ReadToEndAsync(); + var output = proc.StandardOutput.ReadToEnd().Trim(); + if (!proc.WaitForExit(20000)) + { + proc.Kill(entireProcessTree: true); + Assert.Fail("the chart gap harness did not finish in 20 s"); + } + + Assert.True(proc.ExitCode == 0, "the chart gap harness failed: " + error.Result); + using var doc = JsonDocument.Parse(output.Split('\n')[0]); + return doc.RootElement.Clone(); + } + } + + private static int N(JsonElement r, string scenario, string mark) => r.GetProperty(scenario).GetProperty(mark).GetInt32(); + + [Fact] + public void ALineChart_WithNoGap_DrawsOneLinePerSeries() + { + var r = Run(); + Assert.Equal(2, N(r, "noGapLine", "lines")); + Assert.Equal(0, N(r, "noGapLine", "dots")); + } + + [Fact] + public void ALineChart_BreaksEachSeriesAtACollectionGap() + { + var r = Run(); + Assert.Equal(4, N(r, "gapLine", "lines")); + Assert.Equal(0, N(r, "gapLine", "dots")); + } + + [Fact] + public void NoLineSegment_SpansTheGap() + { + var r = Run(); + var runs = r.GetProperty("gapLineXs").EnumerateArray().Select(l => l.EnumerateArray().Select(x => x.GetDouble()).ToArray()).ToArray(); + Assert.Equal(2, runs.Length); + // The widest step inside either run is one 1-minute bucket; the gap is 7 hours, so no step comes near it. + var steps = runs.SelectMany(run => run.Zip(run.Skip(1), (a, b) => b - a)).ToArray(); + Assert.True(steps.Max() < runs[1][0] - runs[0][^1], "a step inside a run is shorter than the gap between runs"); + } + + [Fact] + public void AnAreaChart_BreaksItsFillAndLineAtTheGap() + { + var r = Run(); + Assert.Equal(4, N(r, "gapArea", "lines")); + Assert.Equal(4, N(r, "gapArea", "areas")); + } + + [Fact] + public void AStackedChart_BreaksEachBandAtTheGap() + { + var r = Run(); + Assert.Equal(2, N(r, "noGapStacked", "areas")); + Assert.Equal(4, N(r, "gapStacked", "areas")); + } + + [Fact] + public void AStackedBarChart_IsLeftAlone() + { + var r = Run(); + Assert.Equal(N(r, "noGapStackedBar", "bars"), N(r, "gapStackedBar", "bars")); + Assert.Equal(0, N(r, "gapStackedBar", "lines")); + Assert.Equal(0, N(r, "gapStackedBar", "areas")); + } + + [Fact] + public void TheDualAxisOverlay_BreaksAtTheGapToo() + { + var r = Run(); + Assert.Equal(1, N(r, "noGapOverlay", "overlays")); + Assert.Equal(2, N(r, "gapOverlay", "overlays")); + } + + [Fact] + public void ALoneBucketAfterAGap_IsADot_AndTheStretchBeforeItStaysALine() + { + var r = Run(); + Assert.Equal(1, N(r, "loneAfterGap", "lines")); + Assert.Equal(1, N(r, "loneAfterGap", "dots")); + } + + [Fact] + public void OneOrTwoBuckets_NeverBreak() + { + var r = Run(); + Assert.Equal(2, N(r, "single", "dots")); + Assert.Equal(0, N(r, "single", "lines")); + Assert.Equal(2, N(r, "two", "lines")); + Assert.Equal(0, N(r, "two", "dots")); + } + + [Fact] + public void AMissedCycle_BreaksOnlyPastTheFloorAndThreeTimesTheMedianInterval() + { + var r = Run(); + Assert.Equal(1, N(r, "smallMiss", "lines")); + Assert.Equal(2, N(r, "bigMiss", "lines")); + Assert.Equal(1, N(r, "slowCollector", "lines")); + Assert.Equal(2, N(r, "slowCollectorBreak", "lines")); + } + + [Fact] + public void TheThreshold_HasNothingToCompareUnderThreeBuckets_AndNeverFallsUnderTheFloor() + { + var r = Run().GetProperty("threshold"); + Assert.Equal("Infinity", r.GetProperty("empty").GetString()); + Assert.Equal("Infinity", r.GetProperty("one").GetString()); + Assert.Equal("Infinity", r.GetProperty("two").GetString()); + Assert.Equal(5 * 60000, r.GetProperty("oneMinute").GetDouble()); + Assert.Equal(45 * 60000, r.GetProperty("fifteenMinute").GetDouble()); + } + + [Fact] + public void SplitAtGaps_StartsANewRunWhereTheJumpExceedsTheThreshold() + { + var r = Run(); + Assert.Equal(new[] { "0,1,2", "3,4" }, r.GetProperty("split").EnumerateArray().Select(e => e.GetString()!).ToArray()); + } +} diff --git a/Darling/Darling.Tests/web-chart-gap-harness.mjs b/Darling/Darling.Tests/web-chart-gap-harness.mjs new file mode 100644 index 0000000000..432b8a9f42 --- /dev/null +++ b/Darling/Darling.Tests/web-chart-gap-harness.mjs @@ -0,0 +1,160 @@ +/* Runs the web viewer's shipped charts.js (renderLineChart, with util.js) against a stand-in DOM and counts the line, area and dot + marks drawn across a collection gap (#5644 W7), printing the counts as one line of JSON. + WebChartGapBehaviourTests starts it as + node web-chart-gap-harness.mjs + The DOM is a node tree of plain objects; a chart's marks are the svg children it appends. */ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const jsDir = process.argv[process.argv.length - 1]; + +class FakeNode { + constructor(tag, text) { + this.tag = tag; + this.children = []; + this.attrs = {}; + this.dataset = {}; + this.style = {}; + this.className = ""; + this.listeners = {}; + this.text = text == null ? null : String(text); + this.classList = { add() {}, remove() {}, toggle() {}, contains: () => false }; + } + get firstChild() { + return this.children[0] || null; + } + appendChild(child) { + child.parent = this; + this.children.push(child); + return child; + } + removeChild(child) { + const i = this.children.indexOf(child); + if (i >= 0) this.children.splice(i, 1); + return child; + } + setAttribute(name, value) { + this.attrs[name] = String(value); + } + getAttribute(name) { + return name in this.attrs ? this.attrs[name] : null; + } + addEventListener(type, listener) { + (this.listeners[type] = this.listeners[type] || []).push(listener); + } + setPointerCapture() {} + focus() {} + get offsetWidth() { return 0; } + get offsetHeight() { return 0; } + getBoundingClientRect() { + return { left: 0, top: 0, width: 1000, height: 320 }; + } + set textContent(value) { + this.children = []; + this.text = String(value); + } + get textContent() { + return (this.text || "") + this.children.map((c) => c.textContent).join(""); + } +} + +globalThis.Node = FakeNode; +globalThis.document = { + createElement: (tag) => new FakeNode(tag), + createElementNS: (ns, tag) => new FakeNode(tag), + createTextNode: (text) => new FakeNode("#text", text), + body: new FakeNode("body"), +}; +globalThis.location = { hash: "#/server/A/cpu" }; + +const scratch = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "chart-gap-"))); +let charts; +try { + fs.cpSync(jsDir, scratch, { recursive: true }); + fs.writeFileSync(path.join(scratch, "package.json"), '{ "type": "module" }'); + charts = await import(pathToFileURL(path.join(scratch, "charts.js")).href); +} catch (e) { + fs.rmSync(scratch, { recursive: true, force: true }); + throw e; +} + +const find = (node, pred, out = []) => { + if (pred(node)) out.push(node); + node.children.forEach((c) => find(c, pred, out)); + return out; +}; +const cls = (n) => String(n.className || n.attrs.class || ""); +const marks = (host) => ({ + lines: find(host, (n) => n.tag === "polyline" && cls(n).includes("series-line") && !cls(n).includes("overlay")).length, + overlays: find(host, (n) => n.tag === "polyline" && cls(n).includes("series-line-overlay")).length, + areas: find(host, (n) => n.tag === "polygon" && cls(n).includes("series-area")).length, + dots: find(host, (n) => n.tag === "circle" && cls(n).includes("series-dot")).length, + bars: find(host, (n) => n.tag === "rect" && cls(n).includes("series-bar")).length, +}); +/* The x of every vertex of every line, so a test can show no segment spans the gap. */ +const lineXs = (host) => + find(host, (n) => n.tag === "polyline" && cls(n).includes("series-line") && !cls(n).includes("overlay")).map((n) => + n.attrs.points.split(" ").map((p) => Number(p.split(",")[0])) + ); + +const T0 = Date.UTC(2026, 0, 1, 0, 0, 0); +const MIN = 60000; +const iso = (ms) => new Date(ms).toISOString().slice(0, 19); +/* minutes: offsets from T0; each row carries a, b, and c (c only where `withC` says so). */ +const rowsAt = (minutes) => minutes.map((m, i) => ({ t: iso(T0 + m * MIN), a: 10 + (i % 7), b: 20 + (i % 5), c: 5 + (i % 3) })); +const seq = (from, to, step = 1) => { + const o = []; + for (let m = from; m <= to; m += step) o.push(m); + return o; +}; +const spec = (minutes, extra) => ({ + points: rowsAt(minutes), + xKey: "t", + series: [ + { key: "a", label: "A", color: "#f00" }, + { key: "b", label: "B", color: "#0f0" }, + ], + ...extra, +}); +const draw = (minutes, extra) => charts.renderLineChart(spec(minutes, extra)); + +/* 60 one-minute buckets, then a 7 hour silence, then 60 more. */ +const withGap = seq(0, 59).concat(seq(60 * 7 + 59, 60 * 7 + 118)); +const noGap = seq(0, 119); + +const out = {}; +out.noGapLine = marks(draw(noGap)); +out.gapLine = marks(draw(withGap)); +out.gapLineXs = lineXs(draw(withGap, { series: [{ key: "a", label: "A", color: "#f00" }] })); +out.gapArea = marks(draw(withGap, { mode: "area" })); +out.gapStacked = marks(draw(withGap, { mode: "stacked" })); +out.noGapStacked = marks(draw(noGap, { mode: "stacked" })); +out.gapStackedBar = marks(draw(withGap, { mode: "stacked-bar" })); +out.noGapStackedBar = marks(draw(noGap, { mode: "stacked-bar" })); +out.gapOverlay = marks(draw(withGap, { series2: { key: "c", label: "C", color: "#00f" } })); +out.noGapOverlay = marks(draw(noGap, { series2: { key: "c", label: "C", color: "#00f" } })); +/* a gap that leaves one lone bucket at the end draws that bucket as a dot, and the stretch before it as a line */ +out.loneAfterGap = marks(draw(seq(0, 59).concat([60 * 7]), { series: [{ key: "a", label: "A", color: "#f00" }] })); +/* a single bucket and two buckets never break */ +out.single = marks(draw([0])); +out.two = marks(draw([0, 600])); +/* a 4 minute miss in one-minute buckets is under the 5 minute floor and stays connected; a 6 minute miss breaks */ +out.smallMiss = marks(draw(seq(0, 30).concat(seq(34, 60)), { series: [{ key: "a", label: "A", color: "#f00" }] })); +out.bigMiss = marks(draw(seq(0, 30).concat(seq(37, 60)), { series: [{ key: "a", label: "A", color: "#f00" }] })); +/* a 15 minute collector with one missed cycle (30 minutes) stays connected: 3x the median is 45 minutes */ +out.slowCollector = marks(draw(seq(0, 300, 15).concat(seq(330, 600, 15)), { series: [{ key: "a", label: "A", color: "#f00" }] })); +out.slowCollectorBreak = marks(draw(seq(0, 300, 15).concat(seq(420, 600, 15)), { series: [{ key: "a", label: "A", color: "#f00" }] })); + +out.threshold = { + empty: String(charts.gapThresholdMs([])), + one: String(charts.gapThresholdMs([0])), + two: String(charts.gapThresholdMs([0, 600 * MIN])), + oneMinute: charts.gapThresholdMs(noGap.map((m) => m * MIN)), + fifteenMinute: charts.gapThresholdMs(seq(0, 300, 15).map((m) => m * MIN)), +}; +out.split = charts.splitAtGaps([0, 1, 2, 3, 4], [0, 1, 2, 100, 101], 10).map((r) => r.join(",")); + +fs.rmSync(scratch, { recursive: true, force: true }); +console.log(JSON.stringify(out)); diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js index d265f2157a..75ff7393f2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js @@ -185,6 +185,44 @@ function watchPlotWidth(host, draw, key, drawnW) { observer.observe(host); } +/* Collection gaps (#5644 W7). The service host is off overnight, so a series has a stretch with no samples; a line drawn + point to point would run a straight diagonal across it and read as data. A gap is a jump between two neighbouring + timestamps of more than GAP_FACTOR times the series' own median interval, never less than GAP_FLOOR_MS (a collector can + miss a cycle without that being worth a break). Fewer than three points have no normal interval to compare against, so + they never break. Every time-series line, area and stacked chart goes through this, so no caller inserts null points. */ +const GAP_FACTOR = 3; +const GAP_FLOOR_MS = 5 * 60 * 1000; + +/* The break threshold, in ms, for ascending timestamps (epoch ms); Infinity when there is nothing to compare against. */ +export function gapThresholdMs(times) { + const deltas = []; + for (let i = 1; i < times.length; i++) { + const d = times[i] - times[i - 1]; + if (d > 0) deltas.push(d); + } + if (deltas.length < 2) return Infinity; + deltas.sort((a, b) => a - b); + const mid = deltas.length >> 1; + const median = deltas.length % 2 ? deltas[mid] : (deltas[mid - 1] + deltas[mid]) / 2; + return Math.max(GAP_FACTOR * median, GAP_FLOOR_MS); +} + +/* Splits ascending positions (indices into times) into runs of neighbours no further apart than thresholdMs. */ +export function splitAtGaps(indices, times, thresholdMs) { + const runs = []; + let run = null; + let prev = -1; + for (const i of indices) { + if (run === null || times[i] - times[prev] > thresholdMs) { + run = []; + runs.push(run); + } + run.push(i); + prev = i; + } + return runs; +} + /** * Render a multi-series time chart into a returned `.chart` node. * spec: { points, xKey, series:[{key,label,color}], formatValue?, clampMax?, unit?, mode?, thresholds? } @@ -281,6 +319,8 @@ export function renderLineChart(spec) { const tMin = hasWindow ? windowStart : dataTMin; const tMax = hasWindow ? windowEnd : dataTMax; const spanMs = tMax - tMin; + const rowTimes = rows.map((p) => p.t.getTime()); + const gapMs = gapThresholdMs(rowTimes); /* A numeric reader: null/NaN reads as null for line/area (a gap), or 0 for a stacked mode (a continuous baseline). */ const readVal = (r, key) => { @@ -485,76 +525,91 @@ export function renderLineChart(spec) { } else { /* Filled bands drawn top series first so lower bands paint over the seams; each band is bounded above by its own cumulative top and below by the previous series' cumulative top (the x-axis for series 0). */ + /* A collection gap (#5644 W7) ends one run of bands and starts the next, so the bands do not ramp across it; + a run of one bucket has no extent, so it is dots at the stack tops, as the single-bucket case above. */ + const runs = splitAtGaps(rows.map((_, i) => i), rowTimes, gapMs); for (let k = series.length - 1; k >= 0; k--) { - const top = []; - const bottom = []; - for (let i = 0; i < rows.length; i++) { - top.push(xs[i] + "," + plotY(stackTops[i][k])); - bottom.push(xs[i] + "," + plotY(k === 0 ? 0 : stackTops[i][k - 1])); + for (const run of runs) { + if (run.length === 1) { + root.appendChild(svg("circle", { class: "series-dot", cx: xs[run[0]], cy: plotY(stackTops[run[0]][k]), r: 4, fill: normalizeColor(series[k].color) })); + continue; + } + const top = []; + const bottom = []; + for (const i of run) { + top.push(xs[i] + "," + plotY(stackTops[i][k])); + bottom.push(xs[i] + "," + plotY(k === 0 ? 0 : stackTops[i][k - 1])); + } + bottom.reverse(); + root.appendChild( + svg("polygon", { + class: "series-area", + "clip-path": clipAttr, + points: top.concat(bottom).join(" "), + fill: normalizeColor(series[k].color), + "fill-opacity": "0.72", + }) + ); } - bottom.reverse(); - root.appendChild( - svg("polygon", { - class: "series-area", - "clip-path": clipAttr, - points: top.concat(bottom).join(" "), - fill: normalizeColor(series[k].color), - "fill-opacity": "0.72", - }) - ); } } } else { /* Area fill (each series to the baseline) then the line on top; or just the line. Nulls drop the gap. */ for (const s of series) { - const linePts = []; + const plottable = []; for (let i = 0; i < rows.length; i++) { const v = readVal(rows[i].r, s.key); if (v == null) continue; - linePts.push(xs[i] + "," + plotY(v)); + plottable.push(i); } - /* A lone plottable point has no segment to stroke, so draw it as a dot — one bucket still shows its - reading. (Its resting marker matches the hover dot; static class so it does not vanish on mouseout.) */ - if (linePts.length === 1) { - const [cx, cy] = linePts[0].split(","); - root.appendChild(svg("circle", { class: "series-dot", cx, cy, r: 4, fill: normalizeColor(s.color) })); - continue; - } - if (linePts.length < 2) continue; - if (filled) { - const first = linePts[0].split(",")[0]; - const last = linePts[linePts.length - 1].split(",")[0]; - root.appendChild( - svg("polygon", { - class: "series-area", - "clip-path": clipAttr, - points: `${first},${baseY} ${linePts.join(" ")} ${last},${baseY}`, - fill: normalizeColor(s.color), - "fill-opacity": "0.15", - }) - ); + /* One run per stretch with samples (#5644 W7): a collection gap ends a run, so no line or fill crosses it. */ + for (const run of splitAtGaps(plottable, rowTimes, gapMs)) { + const linePts = run.map((i) => xs[i] + "," + plotY(readVal(rows[i].r, s.key))); + /* A lone plottable point has no segment to stroke, so draw it as a dot — one bucket still shows its + reading. (Its resting marker matches the hover dot; static class so it does not vanish on mouseout.) */ + if (linePts.length === 1) { + const [cx, cy] = linePts[0].split(","); + root.appendChild(svg("circle", { class: "series-dot", cx, cy, r: 4, fill: normalizeColor(s.color) })); + continue; + } + if (filled) { + const first = linePts[0].split(",")[0]; + const last = linePts[linePts.length - 1].split(",")[0]; + root.appendChild( + svg("polygon", { + class: "series-area", + "clip-path": clipAttr, + points: `${first},${baseY} ${linePts.join(" ")} ${last},${baseY}`, + fill: normalizeColor(s.color), + "fill-opacity": "0.15", + }) + ); + } + root.appendChild(svg("polyline", { class: "series-line", "clip-path": clipAttr, points: linePts.join(" "), stroke: normalizeColor(s.color) })); } - root.appendChild(svg("polyline", { class: "series-line", "clip-path": clipAttr, points: linePts.join(" "), stroke: normalizeColor(s.color) })); } } /* The dual-axis overlay line (#1606): plotted against ITS axis (scaleY2), clamped into the plot box, nulls dropped as gaps — same discipline as a primary line. Always a plain line (never filled/stacked). */ if (series2 && scaleY2) { - const pts2 = []; + const plottable2 = []; for (let i = 0; i < rows.length; i++) { const v = rows[i].r[series2.key]; if (v == null || isNaN(v)) continue; - const y2 = Math.max(M.t, Math.min(M.t + PLOT_H, scaleY2(Number(v)))); - pts2.push(xs[i] + "," + y2); + plottable2.push(i); } - if (pts2.length === 1) { - /* Same single-bucket rule as the primary series: a lone overlay reading draws as a dot, not a dropped - series, so "a dot per series" holds for the right axis too. */ - const [cx, cy] = pts2[0].split(","); - root.appendChild(svg("circle", { class: "series-dot", cx, cy, r: 4, fill: normalizeColor(series2.color) })); - } else if (pts2.length >= 2) { - root.appendChild(svg("polyline", { class: "series-line series-line-overlay", "clip-path": clipAttr, points: pts2.join(" "), stroke: normalizeColor(series2.color) })); + /* Broken at collection gaps like a primary line (#5644 W7). */ + for (const run of splitAtGaps(plottable2, rowTimes, gapMs)) { + const pts2 = run.map((i) => xs[i] + "," + Math.max(M.t, Math.min(M.t + PLOT_H, scaleY2(Number(rows[i].r[series2.key]))))); + if (pts2.length === 1) { + /* Same single-bucket rule as the primary series: a lone overlay reading draws as a dot, not a dropped + series, so "a dot per series" holds for the right axis too. */ + const [cx, cy] = pts2[0].split(","); + root.appendChild(svg("circle", { class: "series-dot", cx, cy, r: 4, fill: normalizeColor(series2.color) })); + } else { + root.appendChild(svg("polyline", { class: "series-line series-line-overlay", "clip-path": clipAttr, points: pts2.join(" "), stroke: normalizeColor(series2.color) })); + } } } From efc0b7b255c97381d03f1b490777725f3e857e02 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:07:11 -0400 Subject: [PATCH 02/42] FinOps top-by-average ranks on the unrounded CPU average; Version Store reads tempdb's live size (#5644 W4, W5) --- .../FinOpsDatabaseResourcesViewTests.cs | 2 +- .../Darling.Tests/FinOpsTopAvgRankingTests.cs | 67 +++++++++++++++++++ ...DarlingMcpFinOpsTools.DatabaseResources.cs | 6 +- .../wwwroot/js/pages/finops/utilization.js | 2 +- .../DarlingFinOpsDatabaseResourcesReader.cs | 15 +++-- .../FinOpsTab.xaml | 2 +- .../ViewerDataService.FinOps.cs | 4 +- .../PvsStatsCollectorDefinitionTests.cs | 17 ++++- Lite/Controls/FinOpsTab.xaml | 2 +- .../LocalDataService.FinOps.Workload.cs | 6 +- Lite/Services/LocalDataService.FinOps.cs | 2 + .../PvsStatsCollector.cs | 30 +++++++-- 12 files changed, 133 insertions(+), 22 deletions(-) create mode 100644 Darling/Darling.Tests/FinOpsTopAvgRankingTests.cs diff --git a/Darling/Darling.Tests/FinOpsDatabaseResourcesViewTests.cs b/Darling/Darling.Tests/FinOpsDatabaseResourcesViewTests.cs index 225f011d51..5e0e02afde 100644 --- a/Darling/Darling.Tests/FinOpsDatabaseResourcesViewTests.cs +++ b/Darling/Darling.Tests/FinOpsDatabaseResourcesViewTests.cs @@ -51,7 +51,7 @@ public void GuideTail_CarriesTheViewsKeyFacts() Assert.Contains("top_by_total can include databases with 0 CPU that only had file I/O", tail, StringComparison.Ordinal); Assert.Contains("top_by_avg is empty when no database has executions in the window", tail, StringComparison.Ordinal); Assert.Contains("Above 72 hours, rows reads the per-database rollup and the top lists read the query-grain rollup", tail, StringComparison.Ordinal); - Assert.Contains("avg_cpu_ms is rounded to a whole millisecond, avg_io_mb has 4 decimals, io_total_mb has 2, and total_cpu_ms is the total behind the average", tail, StringComparison.Ordinal); + Assert.Contains("avg_cpu_ms has 2 decimals and top_by_avg is ranked on it before any rounding, avg_io_mb has 4 decimals, io_total_mb has 2, and total_cpu_ms is the total behind the average", tail, StringComparison.Ordinal); Assert.Contains("the desktop's top-consumer grids always cover 24 hours, so send hours_back=24 to match them", tail, StringComparison.Ordinal); Assert.Contains("database_count", tail, StringComparison.Ordinal); Assert.Contains("truncated", tail, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/FinOpsTopAvgRankingTests.cs b/Darling/Darling.Tests/FinOpsTopAvgRankingTests.cs new file mode 100644 index 0000000000..8d99957393 --- /dev/null +++ b/Darling/Darling.Tests/FinOpsTopAvgRankingTests.cs @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Runtime.CompilerServices; +using System.Text.Json; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage.FinOps; +using Xunit; + +namespace Darling.Tests; + +/// +/// #5644 W4: the "Top Databases by Avg CPU / Execution" ranking ran on avg_cpu_ms after Convert.ToInt64 had turned +/// 2.3, 1.95 and 1.56 into 2, so the tie broke on database name and the top-N cut dropped the busiest database. +/// The average now travels unrounded (AvgCpuMs) and is what the ranking and the wire row use. +/// +public sealed class FinOpsTopAvgRankingTests +{ + private static string Read(string relative, [CallerFilePath] string thisFile = "") => + File.ReadAllText(Path.Combine(Path.GetDirectoryName(thisFile)!, "..", "..", relative)).ReplaceLineEndings("\n"); + + [Fact] + public void TopByAvgRow_CarriesTheUnroundedAverage() + { + var row = new TopResourceConsumer("busiest", 2L, 1_066_691L, 10m, 0m, 0m, 2_498_148L, 0.5m, 2.34m); + + var json = JsonSerializer.Serialize(DarlingMcpFinOpsTools.TopByAvgRow(row)); + + Assert.Contains("\"avg_cpu_ms\":2.34", json, StringComparison.Ordinal); + } + + [Fact] + public void StorageReader_RanksTheAvgGridOnTheUnroundedAverage() + { + var source = Read("Darling/PerformanceMonitor.Darling.Storage/FinOps/DarlingFinOpsDatabaseResourcesReader.cs"); + + Assert.Contains("avgCandidates.OrderByDescending(r => r.AvgCpuMs)", source, StringComparison.Ordinal); + Assert.DoesNotContain("avgCandidates.OrderByDescending(r => r.CpuTimeMs)", source, StringComparison.Ordinal); + Assert.DoesNotContain("new TopResourceConsumer(dbName, Convert.ToInt64(reader.GetValue(6))", source, StringComparison.Ordinal); + } + + [Fact] + public void LiteTwin_RanksTheAvgGridOnTheUnroundedAverage() + { + var source = Read("Lite/Services/LocalDataService.FinOps.Workload.cs"); + + Assert.Contains("avgCandidates.OrderByDescending(r => r.AvgCpuMs)", source, StringComparison.Ordinal); + Assert.DoesNotContain("avgCandidates.OrderByDescending(r => r.CpuTimeMs)", source, StringComparison.Ordinal); + Assert.DoesNotContain("CpuTimeMs = ToInt64(reader.GetValue(6))", source, StringComparison.Ordinal); + } + + [Fact] + public void Grids_BindTheAverageNotTheRoundedField() + { + Assert.Contains("Header=\"Avg CPU (ms)\" Binding=\"{Binding AvgCpuMs", Read("Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml"), StringComparison.Ordinal); + Assert.Contains("Header=\"Avg CPU (ms)\" Binding=\"{Binding AvgCpuMs", Read("Lite/Controls/FinOpsTab.xaml"), StringComparison.Ordinal); + Assert.Contains("{ key: \"avg_cpu_ms\", label: \"Avg CPU (ms)\", format: \"num2\" }", + Read("Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js"), StringComparison.Ordinal); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFinOpsTools.DatabaseResources.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFinOpsTools.DatabaseResources.cs index ee74b6d2f0..3e33bf477b 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFinOpsTools.DatabaseResources.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFinOpsTools.DatabaseResources.cs @@ -25,7 +25,7 @@ public sealed partial class DarlingMcpFinOpsTools "database_resources: CPU, reads, writes and I/O per database."; internal const string DatabaseResourcesViewGuide = - "database_resources gives one row per database: CPU ms, logical and physical reads, logical writes and executions from the query-stats window, and read and write MB and stall ms from the file-I/O window. cpu_share_pct and io_share_pct are the database's share of the server's CPU and I/O, rounded to 0.01. rows lists every database up to 500, ordered by CPU, then I/O, then name; database_count is the number of databases and truncated says when there were more. Older windows read the hourly or daily rollups, chosen by the same rule as the desktop viewer. top_by_total and top_by_avg rank databases by total CPU and by CPU per execution, from the query-grain window, and limit caps only these two lists. top_by_total can include databases with 0 CPU that only had file I/O; a database with no executions is left out of top_by_avg, so top_by_avg is empty when no database has executions in the window; an unattributed one is left out of top_by_total. Above 72 hours, rows reads the per-database rollup and the top lists read the query-grain rollup, so one database's CPU can differ between them. avg_cpu_ms is rounded to a whole millisecond, avg_io_mb has 4 decimals, io_total_mb has 2, and total_cpu_ms is the total behind the average. The top lists follow hours_back, while the desktop's top-consumer grids always cover 24 hours, so send hours_back=24 to match them. No cost fields."; + "database_resources gives one row per database: CPU ms, logical and physical reads, logical writes and executions from the query-stats window, and read and write MB and stall ms from the file-I/O window. cpu_share_pct and io_share_pct are the database's share of the server's CPU and I/O, rounded to 0.01. rows lists every database up to 500, ordered by CPU, then I/O, then name; database_count is the number of databases and truncated says when there were more. Older windows read the hourly or daily rollups, chosen by the same rule as the desktop viewer. top_by_total and top_by_avg rank databases by total CPU and by CPU per execution, from the query-grain window, and limit caps only these two lists. top_by_total can include databases with 0 CPU that only had file I/O; a database with no executions is left out of top_by_avg, so top_by_avg is empty when no database has executions in the window; an unattributed one is left out of top_by_total. Above 72 hours, rows reads the per-database rollup and the top lists read the query-grain rollup, so one database's CPU can differ between them. avg_cpu_ms has 2 decimals and top_by_avg is ranked on it before any rounding, avg_io_mb has 4 decimals, io_total_mb has 2, and total_cpu_ms is the total behind the average. The top lists follow hours_back, while the desktop's top-consumer grids always cover 24 hours, so send hours_back=24 to match them. No cost fields."; /// The fixed ceiling on rows; limit caps only the two top lists. internal const int MaxDatabaseRows = 500; @@ -104,11 +104,11 @@ private static async Task ReadDatabaseResourcesAsync( io_share_pct = r.PctIo, }; - /// One top-by-CPU-per-execution row in the wire shape; CpuTimeMs holds the average on these rows. + /// One top-by-CPU-per-execution row in the wire shape; AvgCpuMs holds the unrounded average on these rows (#5644 W4). internal static object TopByAvgRow(TopResourceConsumer r) => new { database_name = r.DatabaseName, - avg_cpu_ms = r.CpuTimeMs, + avg_cpu_ms = r.AvgCpuMs, execution_count = r.ExecutionCount, total_cpu_ms = r.TotalCpuTimeMs, io_total_mb = r.IoTotalMb, diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js index 1065489b4a..0303b83691 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js @@ -100,7 +100,7 @@ const TOP_TOTAL_COLUMNS = [ const TOP_AVG_COLUMNS = [ { key: "database_name", label: "Database" }, - { key: "avg_cpu_ms", label: "Avg CPU (ms)", format: "int" }, + { key: "avg_cpu_ms", label: "Avg CPU (ms)", format: "num2" }, { key: "avg_io_mb", label: "Avg IO (MB)", format: "num2" }, { key: "execution_count", label: "Execs", format: "int" }, ]; diff --git a/Darling/PerformanceMonitor.Darling.Storage/FinOps/DarlingFinOpsDatabaseResourcesReader.cs b/Darling/PerformanceMonitor.Darling.Storage/FinOps/DarlingFinOpsDatabaseResourcesReader.cs index 0a2f81a673..b14ea73fe0 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/FinOps/DarlingFinOpsDatabaseResourcesReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/FinOps/DarlingFinOpsDatabaseResourcesReader.cs @@ -21,10 +21,12 @@ public sealed record DatabaseResourceUsage( decimal IoReadMb, decimal IoWriteMb, long IoStallMs, decimal PctCpuShare, decimal PctIoShare); /// One database in a top-consumer list. By-total rows carry CPU in ; by-average rows -/// carry the average CPU per execution there and the total in . +/// carry the total in and the average CPU per execution, unrounded, in +/// (#5644 W4: on those rows is only the whole-ms rounding of it, kept +/// for older callers; rank and display on ). public sealed record TopResourceConsumer( string DatabaseName, long CpuTimeMs, long ExecutionCount, decimal IoTotalMb, decimal PctCpu, decimal PctIo, - long TotalCpuTimeMs, decimal AvgIoMb); + long TotalCpuTimeMs, decimal AvgIoMb, decimal AvgCpuMs = 0m); /// /// The FinOps Database Resources reads (per-database usage and the top consumers), shared by the viewer and the @@ -363,9 +365,12 @@ public static async Task> GetDatabaseResourceUsageAs NULL exactly for those, so testing it here reproduces the old HAVING. */ if (!reader.IsDBNull(6)) { - avgCandidates.Add(new TopResourceConsumer(dbName, Convert.ToInt64(reader.GetValue(6)), executionCount, ioTotalMb, 0m, 0m, + /* #5644 W4: carry the average unrounded. Convert.ToInt64 here turned 2.3, 1.95 and 1.56 into 2, 2 and 2, + so the tie broke on database name and the top-N cut dropped the busiest database. */ + var avgCpuMs = Convert.ToDecimal(reader.GetValue(6)); + avgCandidates.Add(new TopResourceConsumer(dbName, (long)Math.Round(avgCpuMs), executionCount, ioTotalMb, 0m, 0m, cpuTimeMs, - reader.IsDBNull(7) ? 0m : Convert.ToDecimal(reader.GetValue(7)))); + reader.IsDBNull(7) ? 0m : Convert.ToDecimal(reader.GetValue(7)), avgCpuMs)); } } @@ -375,7 +380,7 @@ public static async Task> GetDatabaseResourceUsageAs tie's order to whichever plan Postgres picked (never a documented contract, and cpu_time_ms / avg_cpu_ms are sums of independent per-database deltas, so an exact tie is not expected in practice). */ var byTotal = totalCandidates.OrderByDescending(r => r.CpuTimeMs).Take(topN).ToList(); - var byAvg = avgCandidates.OrderByDescending(r => r.CpuTimeMs).Take(topN).ToList(); + var byAvg = avgCandidates.OrderByDescending(r => r.AvgCpuMs).Take(topN).ToList(); return (byTotal, byAvg); } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml b/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml index 99db5904ee..1dafbf18bd 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml @@ -456,7 +456,7 @@ - + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.cs index 91d3d99c17..f234140b08 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.cs @@ -97,11 +97,13 @@ public sealed class TopResourceConsumerRow public decimal PctIo { get; set; } public long TotalCpuTimeMs { get; set; } public decimal AvgIoMb { get; set; } + /// The by-average rows' CPU per execution, unrounded (#5644 W4). + public decimal AvgCpuMs { get; set; } public static TopResourceConsumerRow From(PerformanceMonitor.Darling.Storage.FinOps.TopResourceConsumer d) => new() { DatabaseName = d.DatabaseName, CpuTimeMs = d.CpuTimeMs, ExecutionCount = d.ExecutionCount, IoTotalMb = d.IoTotalMb, - PctCpu = d.PctCpu, PctIo = d.PctIo, TotalCpuTimeMs = d.TotalCpuTimeMs, AvgIoMb = d.AvgIoMb + PctCpu = d.PctCpu, PctIo = d.PctIo, TotalCpuTimeMs = d.TotalCpuTimeMs, AvgIoMb = d.AvgIoMb, AvgCpuMs = d.AvgCpuMs }; } diff --git a/Lite.Tests/PvsStatsCollectorDefinitionTests.cs b/Lite.Tests/PvsStatsCollectorDefinitionTests.cs index ede2b2d024..7aa3e7e0a6 100644 --- a/Lite.Tests/PvsStatsCollectorDefinitionTests.cs +++ b/Lite.Tests/PvsStatsCollectorDefinitionTests.cs @@ -153,10 +153,25 @@ public void OnPrem_IsInstanceWide_AndUsesMasterFiles() var query = PvsStatsCollector.Instance.BuildQuery(MakeContext()); Assert.Contains("sys.master_files", query.Text, StringComparison.Ordinal); - Assert.DoesNotContain("sys.database_files", query.Text, StringComparison.Ordinal); + /* Only the tempdb-qualified form (#5644 W5, below) may appear; the unqualified view is the Azure path's. */ + Assert.DoesNotContain("sys.database_files", query.Text.Replace("tempdb.sys.database_files", "", StringComparison.Ordinal), StringComparison.Ordinal); Assert.DoesNotContain("DB_ID()", query.Text, StringComparison.Ordinal); } + [Fact] + public void OnPrem_SizesTempdbFromItsLiveFiles_NotTheStartupSizeInMasterFiles() + { + /* #5644 W5: sys.master_files carries tempdb's size at startup (32 GB against a live 44 GB), so the + denominator for database_id 2 comes from tempdb.sys.database_files. */ + var text = PvsStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("FROM tempdb.sys.database_files AS tdbf", text, StringComparison.Ordinal); + Assert.Equal(1, CountOccurrences(text, "OUTER APPLY")); + Assert.Contains("AND mf.database_id <> 2", text, StringComparison.Ordinal); + Assert.Contains("WHERE pvss.database_id = 2", text, StringComparison.Ordinal); + Assert.DoesNotContain("tempdb.sys.database_files", PvsStatsCollector.Instance.BuildQuery(MakeContext(isAzureSqlDb: true)).Text, StringComparison.Ordinal); + } + [Fact] public void ManagedInstance_TakesTheOnPremPath() { diff --git a/Lite/Controls/FinOpsTab.xaml b/Lite/Controls/FinOpsTab.xaml index a5c5149796..35fafece78 100644 --- a/Lite/Controls/FinOpsTab.xaml +++ b/Lite/Controls/FinOpsTab.xaml @@ -569,7 +569,7 @@ LoadingRow="MarkedGrid_LoadingRow"> - + diff --git a/Lite/Services/LocalDataService.FinOps.Workload.cs b/Lite/Services/LocalDataService.FinOps.Workload.cs index effcff2661..3d05f34217 100644 --- a/Lite/Services/LocalDataService.FinOps.Workload.cs +++ b/Lite/Services/LocalDataService.FinOps.Workload.cs @@ -293,7 +293,9 @@ GetTopResourceConsumersByTotalAsync statement's WHERE c.database_name IS NOT NUL avgCandidates.Add(new TopResourceConsumerRow { DatabaseName = dbName, - CpuTimeMs = ToInt64(reader.GetValue(6)), + /* #5644 W4: the average stays unrounded for the ranking; CpuTimeMs is only its whole-ms rounding. */ + CpuTimeMs = (long)Math.Round(Convert.ToDecimal(reader.GetValue(6))), + AvgCpuMs = Convert.ToDecimal(reader.GetValue(6)), ExecutionCount = executionCount, IoTotalMb = ioTotalMb, TotalCpuTimeMs = cpuTimeMs, @@ -307,7 +309,7 @@ GetTopResourceConsumersByTotalAsync statement's WHERE c.database_name IS NOT NUL statement's ORDER BY c.database_name — deterministic, where the old per-grid statements left a tie's order to whichever plan DuckDB picked (never a documented contract). */ var byTotal = totalCandidates.OrderByDescending(r => r.CpuTimeMs).Take(topN).ToList(); - var byAvg = avgCandidates.OrderByDescending(r => r.CpuTimeMs).Take(topN).ToList(); + var byAvg = avgCandidates.OrderByDescending(r => r.AvgCpuMs).Take(topN).ToList(); return (byTotal, byAvg); } diff --git a/Lite/Services/LocalDataService.FinOps.cs b/Lite/Services/LocalDataService.FinOps.cs index f8bbeae0bf..6ddaffc019 100644 --- a/Lite/Services/LocalDataService.FinOps.cs +++ b/Lite/Services/LocalDataService.FinOps.cs @@ -52,6 +52,8 @@ public class TopResourceConsumerRow public decimal PctIo { get; set; } public long TotalCpuTimeMs { get; set; } public decimal AvgIoMb { get; set; } + /// The by-average rows' CPU per execution, unrounded (#5644 W4). + public decimal AvgCpuMs { get; set; } } public class DatabaseSizeSummaryRow diff --git a/PerformanceMonitor.Collectors/PvsStatsCollector.cs b/PerformanceMonitor.Collectors/PvsStatsCollector.cs index 038653eb69..0b551c9f16 100644 --- a/PerformanceMonitor.Collectors/PvsStatsCollector.cs +++ b/PerformanceMonitor.Collectors/PvsStatsCollector.cs @@ -205,7 +205,7 @@ public readonly record struct Row( online_index_version_store_size_mb = CONVERT(decimal(19,2), pvss.online_index_version_store_size_kb / 1024.0), database_data_size_mb = - CONVERT(decimal(19,2), df.total_db_size_kb / 1024.0), + CONVERT(decimal(19,2), df.total_db_size_kb / 1024.0),, current_aborted_transaction_count = pvss.current_aborted_transaction_count, oldest_active_transaction_id = @@ -243,13 +243,31 @@ JOIN sys.databases AS d ON d.database_id = pvss.database_id OUTER APPLY ( + /* #5644 W5: sys.master_files holds tempdb's size at startup (32 GB against a live 44 GB in the audit), so + database_id 2 reads its live size from tempdb.sys.database_files instead. The two legs are exclusive + on database_id, and the aggregate stays scalar: exactly one row, so this cannot multiply the PVS rows. */ SELECT total_db_size_kb = - SUM(mf.size * 8.0) - FROM sys.master_files AS mf - WHERE mf.database_id = pvss.database_id - AND mf.state = 0 /*ONLINE*/ - AND mf.type = 0 /*ROWS*/ + SUM(sz.size * 8.0) + FROM + ( + SELECT + mf.size + FROM sys.master_files AS mf + WHERE mf.database_id = pvss.database_id + AND mf.database_id <> 2 + AND mf.state = 0 /*ONLINE*/ + AND mf.type = 0 /*ROWS*/ + + UNION ALL + + SELECT + tdbf.size + FROM tempdb.sys.database_files AS tdbf + WHERE pvss.database_id = 2 + AND tdbf.state = 0 /*ONLINE*/ + AND tdbf.type = 0 /*ROWS*/ + ) AS sz ) AS df WHERE 1 = 1 /*EXCLUSION_FILTER*/ From 3139747fd4c66aa8ce5a2fc07e9488c43bdbb188 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:14:22 -0400 Subject: [PATCH 03/42] Web audit labels W9, W14, W16, W19: peak_granted_at, Triage/Mute first, Deadlock trend, honest labels (#5644) --- .../FinOpsTabUtilizationPageTests.cs | 2 +- .../McpLatestSnapshotStampTests.cs | 7 +- Darling/Darling.Tests/MuteRulesPageTests.cs | 2 +- .../Darling.Tests/WebAuditLaneDLabelTests.cs | 74 +++++++++++++++++++ .../Mcp/DarlingMcpMemoryGrantTools.cs | 5 +- .../Mcp/DarlingMemoryGrantReader.cs | 37 ++++++++-- .../wwwroot/js/pages/alerts.js | 8 +- .../js/pages/finops/server-inventory.js | 2 +- .../wwwroot/js/pages/finops/utilization.js | 12 +-- .../wwwroot/js/pages/fleet.js | 13 +++- .../wwwroot/js/pages/server-tabs.js | 24 +++--- Lite.Tests/McpLatestSnapshotStampTests.cs | 8 +- Lite/Mcp/McpMemoryTools.cs | 5 +- .../Services/LocalDataService.MemoryGrants.cs | 36 +++++++-- 14 files changed, 192 insertions(+), 43 deletions(-) create mode 100644 Darling/Darling.Tests/WebAuditLaneDLabelTests.cs diff --git a/Darling/Darling.Tests/FinOpsTabUtilizationPageTests.cs b/Darling/Darling.Tests/FinOpsTabUtilizationPageTests.cs index 48e1f0e9f1..dc4afae583 100644 --- a/Darling/Darling.Tests/FinOpsTabUtilizationPageTests.cs +++ b/Darling/Darling.Tests/FinOpsTabUtilizationPageTests.cs @@ -106,7 +106,7 @@ public void TheSummaryNamesItsWindowAndFormatsMaxCpuAsAnInteger() var tab = Tab(); Assert.Contains("const SUMMARY_SUBTITLE = \"Last 24 hours\";", tab); Assert.Contains("el(\"span\", { class: \"panel-sub\", text: \" \" + SUMMARY_SUBTITLE })", tab); - Assert.Contains("{ key: \"cpu.max_cpu_pct\", label: \"Max CPU %\", format: \"int\" }", tab); + Assert.Contains("{ key: \"cpu.max_cpu_pct\", label: \"Max SQL Server CPU %\", format: \"int\" }", tab); Assert.True(tab.IndexOf("SUMMARY_SUBTITLE })", System.StringComparison.Ordinal) < tab.IndexOf("VIZ.stat(view", System.StringComparison.Ordinal)); } diff --git a/Darling/Darling.Tests/McpLatestSnapshotStampTests.cs b/Darling/Darling.Tests/McpLatestSnapshotStampTests.cs index 111777c2e9..587bbb1a6a 100644 --- a/Darling/Darling.Tests/McpLatestSnapshotStampTests.cs +++ b/Darling/Darling.Tests/McpLatestSnapshotStampTests.cs @@ -925,8 +925,8 @@ public async Task LatestReads_SayWhenTheyWereCaptured_AgainstLivePostgres() var anchor = @base.AddMinutes(5).ToString("o") + "Z"; var runningJobsLatest = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow).AddMinutes(-2); - /* ── memory grants: a storm 30 minutes before a calm latest snapshot ── */ - foreach (var (t, waiters, timeouts, granted) in new[] { (@base.AddMinutes(-30), 12, 3L, 6000m), (@base, 0, 0L, 500m) }) + /* ── memory grants: a storm 30 minutes before a calm latest snapshot; the largest grant (#5644 W9) is the calm one ── */ + foreach (var (t, waiters, timeouts, granted) in new[] { (@base.AddMinutes(-30), 12, 3L, 500m), (@base, 0, 0L, 6000m) }) { await DarlingMcpTestData.ExecAsync(connection, ct, @"INSERT INTO memory_grant_stats (collection_id, collection_time, server_id, server_name, resource_semaphore_id, pool_id, target_memory_mb, max_target_memory_mb, total_memory_mb, available_memory_mb, granted_memory_mb, used_memory_mb, grantee_count, waiter_count, timeout_error_count, forced_grant_count, timeout_error_count_delta, forced_grant_count_delta) @@ -945,6 +945,7 @@ await DarlingMcpTestData.ExecAsync(connection, ct, Assert.Equal(Stamp(@base.AddMinutes(-30)), windowRow.GetProperty("peak_waiters_at").GetString()); Assert.Equal(3, windowRow.GetProperty("timeout_errors_in_window").GetInt64()); Assert.Equal(6000d, windowRow.GetProperty("peak_granted_memory_mb").GetDouble()); + Assert.Equal(Stamp(@base), windowRow.GetProperty("peak_granted_at").GetString()); Assert.Equal(2000d, windowRow.GetProperty("min_available_memory_mb").GetDouble()); Assert.Equal(Stamp(@base), windowRow.GetProperty("last_snapshot_at").GetString()); @@ -954,6 +955,8 @@ await DarlingMcpTestData.ExecAsync(connection, ct, var poolWindow = Assert.Single(grants.GetProperty("window").EnumerateArray()); Assert.Equal(JsonValueKind.Null, poolWindow.GetProperty("resource_semaphore_id").ValueKind); Assert.Equal(12, poolWindow.GetProperty("peak_waiter_count").GetInt64()); + Assert.Equal(Stamp(@base), poolWindow.GetProperty("peak_granted_at").GetString()); + Assert.Equal(Stamp(@base.AddMinutes(-30)), poolWindow.GetProperty("peak_waiters_at").GetString()); /* ── scheduler: one snapshot three hours before the anchor — inside a 4 h search, outside a 1 h one ── */ var schedulerAt = @base.AddMinutes(5).AddHours(-3); diff --git a/Darling/Darling.Tests/MuteRulesPageTests.cs b/Darling/Darling.Tests/MuteRulesPageTests.cs index c9dd48f37d..943ec409b3 100644 --- a/Darling/Darling.Tests/MuteRulesPageTests.cs +++ b/Darling/Darling.Tests/MuteRulesPageTests.cs @@ -64,7 +64,7 @@ public void AlertHistoryOffersMuteActionsOnlyToAnEditingSeat() { var alerts = Wwwroot("js", "pages", "alerts.js"); Assert.Contains("canMute = !!(await getSession()).can_edit;", alerts); - Assert.Contains("return canMute ? [SELECT_COLUMN].concat(ALERT_COLUMNS, [MUTE_COLUMN]) : ALERT_COLUMNS;", alerts); + Assert.Contains("return canMute ? [SELECT_COLUMN, TRIAGE_COLUMN, MUTE_COLUMN].concat(ALERT_COLUMNS) : [TRIAGE_COLUMN].concat(ALERT_COLUMNS);", alerts); Assert.Contains("link(\"Mute this alert\", mutePrefillParams(a))", alerts); Assert.DoesNotContain("server_name: a.server_name", alerts); Assert.Contains("stored_server_name = r.StoredServerName", File.ReadAllText(PathTo("Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingMcpAlertTools.cs"))); diff --git a/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs b/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs new file mode 100644 index 0000000000..ad51d2d750 --- /dev/null +++ b/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using Xunit; +using static Darling.Tests.RepoFile; + +namespace Darling.Tests; + +/// +/// Source pins for the #5644 web UI audit labels W9, W14, W16 and W19: each label says what its value is. No JS runner +/// is available to the C# tests, so these read the page sources, the way MuteRulesPageTests does. +/// +public sealed class WebAuditLaneDLabelTests +{ + private static string Js(params string[] parts) => + ReadRepoFile([.. new[] { "Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js" }, .. parts]).ReplaceLineEndings("\n"); + + [Fact] + public void W9_TheMemoryGrantWindowShowsTheTimeOfThePeakGrantBesideThePeakGrant() + { + var tabs = Js("pages", "server-tabs.js"); + Assert.Contains("{ key: \"peak_waiters_at\", label: \"Peak waiters at\", format: \"time\" },", tabs, StringComparison.Ordinal); + Assert.Contains( + "{ key: \"peak_granted_memory_mb\", label: \"Peak granted\", format: \"mb\" },\n { key: \"peak_granted_at\", label: \"Peak granted at\", format: \"time\" },", + tabs, StringComparison.Ordinal); + Assert.DoesNotContain("label: \"Peak at\"", tabs, StringComparison.Ordinal); + } + + [Fact] + public void W14_TriageAndMuteAreTheFirstAlertHistoryColumns() + { + var alerts = Js("pages", "alerts.js"); + Assert.Contains("[SELECT_COLUMN, TRIAGE_COLUMN, MUTE_COLUMN].concat(ALERT_COLUMNS)", alerts, StringComparison.Ordinal); + Assert.Contains(": [TRIAGE_COLUMN].concat(ALERT_COLUMNS)", alerts, StringComparison.Ordinal); + Assert.DoesNotContain("{ key: \"triage\", label: \"Triage\", csv: false, render: (a) => triageCell(a) },\n];", alerts, StringComparison.Ordinal); + } + + [Fact] + public void W16_TheDeadlockTrendChartAndTheDeadlocksTableAreNamedApartAndEmptyTheSameWay() + { + var tabs = Js("pages", "server-tabs.js"); + Assert.Contains("line(\"Deadlock trend\", \"get_deadlock_trend\", { server, hours: ctx.hours }, \"trend\", \"time\", COUNT_SERIES, {\n subtitle: ctx.label,\n format: \"int\",\n atTimeItem: \"deadlocks\",\n emptyText: DEADLOCKS_EMPTY_TEXT,", tabs, StringComparison.Ordinal); + Assert.Contains("const DEADLOCKS_EMPTY_TEXT = \"No deadlocks in this window.\";", tabs, StringComparison.Ordinal); + Assert.Contains(" ctx.label,\n DEADLOCKS_EMPTY_TEXT,\n 2,\n \"separately_monitored_note\"", tabs, StringComparison.Ordinal); + } + + [Fact] + public void W19_EachMisleadingLabelNamesItsValue() + { + var tabs = Js("pages", "server-tabs.js"); + Assert.Contains("{ key: \"hyperthread_ratio\", label: \"Logical CPUs per socket\"", tabs, StringComparison.Ordinal); + Assert.DoesNotContain("label: \"HT ratio\"", tabs, StringComparison.Ordinal); + Assert.Contains("{ key: \"plan_cache_mb\", label: \"Plan cache (memory clerk)\", format: \"mb\" },", tabs, StringComparison.Ordinal); + Assert.Contains("{ key: \"summary.total_size_mb\", label: \"Cached plans size\", format: \"mb\" },", tabs, StringComparison.Ordinal); + Assert.Contains("{ key: \"pct_of_sweep_budget_per_run\", label: \"% of run budget\"", tabs, StringComparison.Ordinal); + /* fmtMs prints seconds above a second ("32.9 s"), so the header carries no unit. */ + Assert.Contains("{ key: \"duration_ms\", label: \"Duration\", format: \"ms\" },\n { key: \"signal_duration_ms\", label: \"Signal duration\", format: \"ms\" },", tabs, StringComparison.Ordinal); + + var utilization = Js("pages", "finops", "utilization.js"); + foreach (var label in new[] { "Avg SQL Server CPU %", "P95 SQL Server CPU %", "Max SQL Server CPU %" }) + Assert.Contains("label: \"" + label + "\"", utilization, StringComparison.Ordinal); + Assert.Contains("label: \"Avg SQL Server CPU %\"", Js("pages", "finops", "server-inventory.js"), StringComparison.Ordinal); + + var fleet = Js("pages", "fleet.js"); + Assert.Contains("chip(c.total_cpu_percent != null ? \"Total CPU\" : \"CPU\", cpuValue, c.cpu_severity, cpuDetail)", fleet, StringComparison.Ordinal); + Assert.Contains("fmtInt(threadsInUse) + \" in use / \" + fmtInt(c.total_threads) + \" threads\"", fleet, StringComparison.Ordinal); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpMemoryGrantTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpMemoryGrantTools.cs index 9648511d24..0ac9055fb5 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpMemoryGrantTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpMemoryGrantTools.cs @@ -43,7 +43,7 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpMemoryGrantTools { - [McpServerTool(Name = "get_resource_semaphore"), Description("Resource semaphore stats: granted/available memory vs target/max-target ceiling, waiters, timeout/forced pressure, window ends at as_of. TWO READS, per semaphore+pool: grants[] is the NEWEST snapshot (moment: captured_at/age_seconds); window[] aggregates EVERY snapshot (peak waiters+when, peak/min memory, summed timeout/forced deltas). Read window[] for 'was there pressure', grants[] for 'is there pressure now'. No rows: unavailable (not_collected first). sample_interval_seconds/interval_known null/false on a restart-marker or pre-column row; its zero deltas aren't 'no timeouts'. <> Gets resource semaphore statistics showing granted vs available workspace memory against the target/max-target ceiling, waiter counts, and timeout/forced grant pressure indicators. High waiter counts or rising timeout/forced deltas indicate memory grant pressure affecting query performance. TWO READS UNDER ONE WINDOW: grants[] is the NEWEST snapshot in the window (one row per resource semaphore and pool), stamped once as captured_at with age_seconds against the window's end - it is a moment, not the window. window[] aggregates EVERY snapshot in the window per (resource_semaphore_id, pool_id): peak_waiter_count and peak_waiters_at (the most sessions ever seen waiting for a grant and when), peak_granted_memory_mb, min_available_memory_mb, and timeout_errors_in_window / forced_grants_in_window (the SUM of the per-interval deltas across the window). Each grants[] row also carries sample_interval_seconds, the measured seconds its two deltas accrued over; it is null with interval_known false when the row is a restart marker (no delta was knowable, so the zero deltas beside it are not 'no timeouts') or predates the column. interval_seconds is the interval these deltas cover, null when it can't be known - the same name the latch and spinlock tools use; sample_interval_seconds carries the same value and is kept for existing readers. " + McpToolGuideTopics.MemoryGrantWindowReadOrder)] + [McpServerTool(Name = "get_resource_semaphore"), Description("Resource semaphore stats: granted/available memory vs target/max-target ceiling, waiters, timeout/forced pressure, window ends at as_of. TWO READS, per semaphore+pool: grants[] is the NEWEST snapshot (moment: captured_at/age_seconds); window[] aggregates EVERY snapshot (peak waiters+when, peak/min memory, summed timeout/forced deltas). Read window[] for 'was there pressure', grants[] for 'is there pressure now'. No rows: unavailable (not_collected first). sample_interval_seconds/interval_known null/false on a restart-marker or pre-column row; its zero deltas aren't 'no timeouts'. <> Gets resource semaphore statistics showing granted vs available workspace memory against the target/max-target ceiling, waiter counts, and timeout/forced grant pressure indicators. High waiter counts or rising timeout/forced deltas indicate memory grant pressure affecting query performance. TWO READS UNDER ONE WINDOW: grants[] is the NEWEST snapshot in the window (one row per resource semaphore and pool), stamped once as captured_at with age_seconds against the window's end - it is a moment, not the window. window[] aggregates EVERY snapshot in the window per (resource_semaphore_id, pool_id): peak_waiter_count and peak_waiters_at (the most sessions ever seen waiting for a grant and when), peak_granted_memory_mb and peak_granted_at (the largest granted memory and when - not the time of peak waiters), min_available_memory_mb, and timeout_errors_in_window / forced_grants_in_window (the SUM of the per-interval deltas across the window). Each grants[] row also carries sample_interval_seconds, the measured seconds its two deltas accrued over; it is null with interval_known false when the row is a restart marker (no delta was knowable, so the zero deltas beside it are not 'no timeouts') or predates the column. interval_seconds is the interval these deltas cover, null when it can't be known - the same name the latch and spinlock tools use; sample_interval_seconds carries the same value and is kept for existing readers. " + McpToolGuideTopics.MemoryGrantWindowReadOrder)] public static async Task GetResourceSemaphore( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -121,7 +121,7 @@ the original sample_interval_seconds rather than in place of it. */ } } - [McpServerTool(Name = "get_memory_grants"), Description("Gets resource semaphore stats: granted vs available workspace memory per resource pool, waiters, timeout/forced grant deltas, over a window ending at as_of (default 1h). TWO READS: grants[] is the NEWEST snapshot in the window (a moment: captured_at/age_seconds), one row per pool; window[] aggregates EVERY snapshot in it, per pool (peak waiters+when, peak/min memory, summed timeout/forced deltas). Read window[] for 'was there pressure', grants[] for 'is there pressure now'. No rows: unavailable (or not_collected first). <> Gets resource semaphore statistics showing granted vs available workspace memory per resource pool, waiter counts, and timeout/forced grant deltas. High waiter counts or rising timeout deltas indicate memory grant pressure affecting query performance. TWO READS UNDER ONE WINDOW: grants[] is the NEWEST snapshot in the window (one row per pool, summed across its semaphores), stamped once as captured_at with age_seconds against the window's end - it is a moment, not the window. window[] aggregates EVERY snapshot in the window per pool: peak_waiter_count and peak_waiters_at (the most sessions ever seen waiting on the pool at one instant and when), peak_granted_memory_mb, min_available_memory_mb, and timeout_errors_in_window / forced_grants_in_window (the SUM of the per-interval deltas across the window). " + McpToolGuideTopics.MemoryGrantWindowReadOrder)] + [McpServerTool(Name = "get_memory_grants"), Description("Gets resource semaphore stats: granted vs available workspace memory per resource pool, waiters, timeout/forced grant deltas, over a window ending at as_of (default 1h). TWO READS: grants[] is the NEWEST snapshot in the window (a moment: captured_at/age_seconds), one row per pool; window[] aggregates EVERY snapshot in it, per pool (peak waiters+when, peak/min memory, summed timeout/forced deltas). Read window[] for 'was there pressure', grants[] for 'is there pressure now'. No rows: unavailable (or not_collected first). <> Gets resource semaphore statistics showing granted vs available workspace memory per resource pool, waiter counts, and timeout/forced grant deltas. High waiter counts or rising timeout deltas indicate memory grant pressure affecting query performance. TWO READS UNDER ONE WINDOW: grants[] is the NEWEST snapshot in the window (one row per pool, summed across its semaphores), stamped once as captured_at with age_seconds against the window's end - it is a moment, not the window. window[] aggregates EVERY snapshot in the window per pool: peak_waiter_count and peak_waiters_at (the most sessions ever seen waiting on the pool at one instant and when), peak_granted_memory_mb and peak_granted_at (the largest granted memory and when - not the time of peak waiters), min_available_memory_mb, and timeout_errors_in_window / forced_grants_in_window (the SUM of the per-interval deltas across the window). " + McpToolGuideTopics.MemoryGrantWindowReadOrder)] public static async Task GetMemoryGrants( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -195,6 +195,7 @@ public static async Task GetMemoryGrants( peak_waiter_count = w.PeakWaiterCount, peak_waiters_at = w.PeakWaitersAt.ToString("o"), peak_granted_memory_mb = Math.Round(w.PeakGrantedMemoryMb, 2), + peak_granted_at = w.PeakGrantedAt.ToString("o"), min_available_memory_mb = Math.Round(w.MinAvailableMemoryMb, 2), timeout_errors_in_window = w.TimeoutErrorsInWindow, forced_grants_in_window = w.ForcedGrantsInWindow diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMemoryGrantReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMemoryGrantReader.cs index aa22acced8..ae3e0e73d2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMemoryGrantReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMemoryGrantReader.cs @@ -74,13 +74,14 @@ public sealed record MemoryGrantRow( /// One (resource_semaphore_id, pool_id) — or, for the pool lens, one pool with /// null — aggregated over EVERY snapshot in the window (#3541 A10). / /// is the storm detector: the most sessions ever seen waiting for a grant in the - /// window and the snapshot it happened at. The two *InWindow figures are SUMs of the stored per-interval + /// window and the snapshot it happened at. is the snapshot of the largest + /// granted memory (), a different moment when nothing waited then (#5644 W9). The two *InWindow figures are SUMs of the stored per-interval /// deltas — how many grants timed out / were forced across the whole window, not just the last interval. /// public sealed record MemoryGrantWindowRow( short? ResourceSemaphoreId, int PoolId, long SnapshotsInWindow, DateTime FirstSnapshotAt, DateTime LastSnapshotAt, long PeakWaiterCount, DateTime PeakWaitersAt, double PeakGrantedMemoryMb, double MinAvailableMemoryMb, - long TimeoutErrorsInWindow, long ForcedGrantsInWindow); + long TimeoutErrorsInWindow, long ForcedGrantsInWindow, DateTime PeakGrantedAt); /* ─────────────────────────── resource semaphore (latest snapshot, per semaphore) ─────────────────────────── */ @@ -275,6 +276,15 @@ SELECT DISTINCT ON (resource_semaphore_id, pool_id) collection_time AS peak_waiters_at FROM windowed ORDER BY resource_semaphore_id, pool_id, waiter_count DESC, collection_time DESC + ), + peak_grant AS + ( + SELECT DISTINCT ON (resource_semaphore_id, pool_id) + resource_semaphore_id, + pool_id, + collection_time AS peak_granted_at + FROM windowed + ORDER BY resource_semaphore_id, pool_id, granted_memory_mb DESC NULLS LAST, collection_time DESC ) SELECT a.resource_semaphore_id, @@ -287,11 +297,15 @@ FROM windowed a.peak_granted_memory_mb, a.min_available_memory_mb, a.timeout_errors_in_window, - a.forced_grants_in_window + a.forced_grants_in_window, + pg.peak_granted_at FROM agg AS a JOIN peak AS p ON p.resource_semaphore_id = a.resource_semaphore_id AND p.pool_id = a.pool_id + JOIN peak_grant AS pg + ON pg.resource_semaphore_id = a.resource_semaphore_id + AND pg.pool_id = a.pool_id ORDER BY a.pool_id, a.resource_semaphore_id """; @@ -341,6 +355,14 @@ SELECT DISTINCT ON (pool_id) collection_time AS peak_waiters_at FROM per_snapshot ORDER BY pool_id, waiter_count DESC, collection_time DESC + ), + peak_grant AS + ( + SELECT DISTINCT ON (pool_id) + pool_id, + collection_time AS peak_granted_at + FROM per_snapshot + ORDER BY pool_id, granted_memory_mb DESC NULLS LAST, collection_time DESC ) SELECT CAST(NULL AS smallint) AS resource_semaphore_id, @@ -353,9 +375,11 @@ FROM per_snapshot a.peak_granted_memory_mb, a.min_available_memory_mb, a.timeout_errors_in_window, - a.forced_grants_in_window + a.forced_grants_in_window, + pg.peak_granted_at FROM agg AS a JOIN peak AS p ON p.pool_id = a.pool_id + JOIN peak_grant AS pg ON pg.pool_id = a.pool_id ORDER BY a.pool_id """; @@ -367,7 +391,7 @@ public static Task> GetMemoryGrantsWindowAsync( NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) => ReadWindowAsync(postgres, MemoryGrantsWindowSql, serverId, startUtc, endUtc, cancellationToken); - /// The two window reads project the SAME eleven columns in the same order (the pool lens fills + /// The two window reads project the SAME twelve columns in the same order (the pool lens fills /// resource_semaphore_id with a typed NULL), so one materialiser serves both. private static async Task> ReadWindowAsync( NpgsqlDataSource postgres, string sql, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken) @@ -390,7 +414,8 @@ private static async Task> ReadWindowAsync( reader.IsDBNull(7) ? 0 : reader.GetDouble(7), reader.IsDBNull(8) ? 0 : reader.GetDouble(8), reader.IsDBNull(9) ? 0 : reader.GetInt64(9), - reader.IsDBNull(10) ? 0 : reader.GetInt64(10))); + reader.IsDBNull(10) ? 0 : reader.GetInt64(10), + reader.GetDateTime(11))); } return rows; diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js index e9f4c26e9d..c13a7647ab 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js @@ -50,9 +50,13 @@ const ALERT_COLUMNS = [ { key: "threshold_value", label: "Threshold", format: "num1" }, { key: "status", label: "Status", render: (a) => statusCell(a) }, { key: "detail_text", label: "Detail", render: (a) => detailCell(a), copyValue: (a) => detailFullText(a) }, - { key: "triage", label: "Triage", csv: false, render: (a) => triageCell(a) }, ]; +/* W14 (#5644): Triage and Mute are the grid's FIRST columns. At 1600 px the 1,791 px grid sat in a 1,298 px box and the + two actions were the last things reached by horizontal scroll. The checkbox column (a seat that can edit) stays + ahead of them. */ +const TRIAGE_COLUMN = { key: "triage", label: "Triage", csv: false, render: (a) => triageCell(a) }; + /* The Mute column exists only for a seat whose session reports can_edit (the server enforces the write gate; this is only the affordance). Both links open the Mute Rules create form pre-filled from the row. "Mute this alert" keys on the row's store id when it has one and otherwise on the stored server spelling, and fills the @@ -61,7 +65,7 @@ const ALERT_COLUMNS = [ let canMute = false; const MUTE_COLUMN = { key: "mute", label: "Mute", csv: false, render: (a) => muteCell(a) }; function alertColumns() { - return canMute ? [SELECT_COLUMN].concat(ALERT_COLUMNS, [MUTE_COLUMN]) : ALERT_COLUMNS; + return canMute ? [SELECT_COLUMN, TRIAGE_COLUMN, MUTE_COLUMN].concat(ALERT_COLUMNS) : [TRIAGE_COLUMN].concat(ALERT_COLUMNS); } /* Dismiss (the web twin of the Viewer's Dismiss Selected / Dismiss All), for a seat that can edit. The checked diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/server-inventory.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/server-inventory.js index d80641d0e3..cdab8b594b 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/server-inventory.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/server-inventory.js @@ -37,7 +37,7 @@ const COLUMNS = [ { key: "cores_per_socket", label: "Cores/Socket", format: "int" }, { key: "hardware_note", label: "Hardware note", wrap: true }, { key: "provisioning_status", label: "Status", sevKey: "provisioning_sev" }, - { key: "avg_cpu_pct", label: "Avg CPU %", format: "num1" }, + { key: "avg_cpu_pct", label: "Avg SQL Server CPU %", format: "num1" }, { key: "storage_total_gb", label: "Storage GB", format: "num1" }, { key: "idle_db_count", label: "Idle DBs", format: "int" }, { key: "sqlserver_start_time_local", label: "Start time (server clock)" }, diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js index 1065489b4a..aef6cf7405 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js @@ -62,9 +62,9 @@ const SUMMARY_STATS = [ { key: "health_score", label: "Health", format: "int" }, { key: "monthly_cost_text", label: "Budget", format: "text" }, { key: "annual_cost_text", label: "Annual", format: "text" }, - { key: "cpu.avg_cpu_pct", label: "Avg CPU %", format: "num1" }, - { key: "cpu.p95_cpu_pct", label: "P95 CPU %", format: "num1" }, - { key: "cpu.max_cpu_pct", label: "Max CPU %", format: "int" }, + { key: "cpu.avg_cpu_pct", label: "Avg SQL Server CPU %", format: "num1" }, + { key: "cpu.p95_cpu_pct", label: "P95 SQL Server CPU %", format: "num1" }, + { key: "cpu.max_cpu_pct", label: "Max SQL Server CPU %", format: "int" }, { key: "cpu.cpu_count", label: "vCores", format: "int", nullKey: "cpu.cpu_count_reason", showWhen: { key: "cpu.cpu_count_unit", equals: "vcores" } }, { key: "cpu.cpu_count", label: "CPUs", format: "int", nullKey: "cpu.cpu_count_reason", hideWhen: { key: "cpu.cpu_count_unit", equals: "vcores" } }, { key: "cpu.max_workers", label: "Max workers", format: "int" }, @@ -82,9 +82,9 @@ const SUMMARY_STATS = [ const TREND_COLUMNS = [ { key: "day_label", label: "Day" }, { key: "verdict_label", label: "Status", sevKey: "verdict_sev" }, - { key: "avg_cpu_pct", label: "Avg CPU %", format: "num1" }, - { key: "p95_cpu_pct", label: "P95 CPU %", format: "num1" }, - { key: "max_cpu_pct", label: "Max CPU %", format: "int" }, + { key: "avg_cpu_pct", label: "Avg SQL Server CPU %", format: "num1" }, + { key: "p95_cpu_pct", label: "P95 SQL Server CPU %", format: "num1" }, + { key: "max_cpu_pct", label: "Max SQL Server CPU %", format: "int" }, { key: "memory_ratio", label: "Mem Ratio", format: "num2" }, ]; /* utilization-keys:end */ diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js index 61550e8235..067d68f852 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js @@ -725,10 +725,15 @@ export function metricBands(c) { : c.available_threads != null ? fmtInt(c.available_threads) + " free" : "ok"; - const threadsDetail = + /* W19 (#5644): the tile value reads "N free", so the line under it must not repeat N over the total (that read as + N in use). It says how many are in use, of the total. */ + const threadsInUse = c.total_threads != null - ? fmtInt(c.available_threads ?? c.total_threads - (c.current_workers ?? 0)) + " / " + fmtInt(c.total_threads) + " threads" + ? c.available_threads != null + ? c.total_threads - c.available_threads + : c.current_workers ?? 0 : null; + const threadsDetail = c.total_threads != null ? fmtInt(threadsInUse) + " in use / " + fmtInt(c.total_threads) + " threads" : null; const cpuValue = c.total_cpu_percent != null || c.cpu_percent != null ? fmtPct(c.total_cpu_percent ?? c.cpu_percent) : "n/a"; @@ -804,7 +809,9 @@ export function metricBands(c) { } return el("div", { class: "metric-bands" }, [ - chip("CPU", cpuValue, c.cpu_severity, cpuDetail), + /* W19 (#5644): total_cpu_percent is the whole box (SQL Server plus other processes); FinOps reads the SQL Server + process alone, so this chip says which one it shows. The cpu_percent fallback (a PostgreSQL target) stays "CPU". */ + chip(c.total_cpu_percent != null ? "Total CPU" : "CPU", cpuValue, c.cpu_severity, cpuDetail), chip("Threads", threadsValue, c.threads_severity, threadsDetail), chip("Memory", memValue, c.memory_severity, memDetail), chip("Blocking", fmtInt(c.blocking_count), c.blocking_severity, blockingDetail), diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js index 97cb90a3a9..c957ebc7ef 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js @@ -1322,11 +1322,11 @@ export const SERVER_TABS = [ sourceKey: "source", emptyText: "No blocking events in this window — an empty trend here means none happened, not that nothing was collected.", }), - line("Deadlocks", "get_deadlock_trend", { server, hours: ctx.hours }, "trend", "time", COUNT_SERIES, { + line("Deadlock trend", "get_deadlock_trend", { server, hours: ctx.hours }, "trend", "time", COUNT_SERIES, { subtitle: ctx.label, format: "int", atTimeItem: "deadlocks", - emptyText: "No deadlocks in this window — an empty trend here means none happened, not that nothing was collected.", + emptyText: DEADLOCKS_EMPTY_TEXT, }), table( "Blocking", @@ -1348,7 +1348,7 @@ export const SERVER_TABS = [ "deadlocks", [...DEADLOCK_COLUMNS, deadlockPlanColumn(server)], ctx.label, - "No deadlocks in this window.", + DEADLOCKS_EMPTY_TEXT, 2, "separately_monitored_note" ), @@ -2956,7 +2956,7 @@ const PROPERTY_STATS = [ { key: "cpu_count", label: "Logical CPUs", format: "int" }, { key: "socket_count", label: "Sockets", format: "int", hideWhen: AZURE_SQL_DATABASE }, { key: "cores_per_socket", label: "Cores/socket", format: "int", hideWhen: AZURE_SQL_DATABASE }, - { key: "hyperthread_ratio", label: "HT ratio", format: "int", hideWhen: AZURE_SQL_DATABASE }, + { key: "hyperthread_ratio", label: "Logical CPUs per socket", format: "int", hideWhen: AZURE_SQL_DATABASE }, { key: "physical_memory_mb", label: "Physical memory", format: "mb", hideWhen: AZURE_SQL_DATABASE }, { key: "is_clustered", label: "Clustered", format: "bool" }, { key: "is_hadr_enabled", label: "Always On", format: "bool" }, @@ -3033,17 +3033,20 @@ const MEMORY_STATS = [ { key: "total_server_memory_mb", label: "Total server", format: "mb" }, { key: "target_server_memory_mb", label: "Target server", format: "mb" }, { key: "buffer_pool_mb", label: "Buffer pool", format: "mb" }, - { key: "plan_cache_mb", label: "Plan cache", format: "mb" }, + { key: "plan_cache_mb", label: "Plan cache (memory clerk)", format: "mb" }, { key: "system_memory_state", label: "System state", format: "text", small: true, nullKey: "system_memory_state_note" }, { key: "sql_memory_model", label: "Memory model", format: "text", small: true }, ]; +/* W16 (#5644): the Blocking tab's Deadlock trend chart and Deadlocks table say the same thing when empty. */ +const DEADLOCKS_EMPTY_TEXT = "No deadlocks in this window."; + const PLAN_CACHE_STATS = [ { key: "summary.bloat_level", label: "Bloat", format: "text", small: true }, { key: "summary.total_plans", label: "Plans", format: "int" }, { key: "summary.single_use_plans", label: "Single-use", format: "int" }, { key: "summary.single_use_percent", label: "Single-use %", format: "num1" }, - { key: "summary.total_size_mb", label: "Cache size", format: "mb" }, + { key: "summary.total_size_mb", label: "Cached plans size", format: "mb" }, { key: "summary.single_use_size_mb", label: "Single-use size", format: "mb" }, { key: "summary.wasted_percent", label: "Wasted %", format: "num1" }, { key: "summary.bloat_recommendation", label: "Recommendation", format: "text", small: true }, @@ -3898,8 +3901,9 @@ const GRANT_WINDOW_COLUMNS = [ { key: "pool_id", label: "Pool", format: "int" }, { key: "snapshots_in_window", label: "Snapshots", format: "int" }, { key: "peak_waiter_count", label: "Peak waiters", format: "int" }, - { key: "peak_waiters_at", label: "Peak at", format: "time" }, + { key: "peak_waiters_at", label: "Peak waiters at", format: "time" }, { key: "peak_granted_memory_mb", label: "Peak granted", format: "mb" }, + { key: "peak_granted_at", label: "Peak granted at", format: "time" }, { key: "min_available_memory_mb", label: "Min available", format: "mb" }, { key: "timeout_errors_in_window", label: "Timeouts", format: "int" }, { key: "forced_grants_in_window", label: "Forced grants", format: "int" }, @@ -4712,8 +4716,8 @@ const MEMORY_BROKER_COLUMNS = [ const SIGNIFICANT_WAIT_COLUMNS = [ { key: "event_time", label: "Event Time", format: "time" }, { key: "wait_type", label: "Wait Type" }, - { key: "duration_ms", label: "Duration ms", format: "ms" }, - { key: "signal_duration_ms", label: "Signal ms", format: "ms" }, + { key: "duration_ms", label: "Duration", format: "ms" }, + { key: "signal_duration_ms", label: "Signal duration", format: "ms" }, { key: "wait_resource", label: "Wait Resource", wrap: true }, { key: "session_id", label: "Session", format: "int" }, { key: "query_text", label: "Query", valueList: false, render: (r) => codeDisclosure(r.query_text) }, @@ -4926,7 +4930,7 @@ const HEAVIEST_COLUMNS = [ row has one, so a server with no run times shows no column of dashes. */ { key: "run_at", label: "Run at", hideWhenEmpty: true }, { key: "amortized_ms_per_minute", label: "ms/min", format: "num1" }, - { key: "pct_of_sweep_budget_per_run", label: "% of sweep", format: "num1" }, + { key: "pct_of_sweep_budget_per_run", label: "% of run budget", format: "num1" }, ]; /* ─────────────────────────── PostgreSQL stat descriptors ─────────────────────────── */ diff --git a/Lite.Tests/McpLatestSnapshotStampTests.cs b/Lite.Tests/McpLatestSnapshotStampTests.cs index 1b320072cd..e2a443469a 100644 --- a/Lite.Tests/McpLatestSnapshotStampTests.cs +++ b/Lite.Tests/McpLatestSnapshotStampTests.cs @@ -87,8 +87,10 @@ public void Dispose() public async Task GetResourceSemaphore_ReadsTheWindowBesideTheStampedSnapshot() { var (@base, anchor) = Past(); - await SeedGrantAsync(@base.AddMinutes(-30), waiters: 12, timeoutsDelta: 3, grantedMb: 6000); - await SeedGrantAsync(@base, waiters: 0, timeoutsDelta: 0, grantedMb: 500); + /* #5644 W9: the largest grant (6000) is the calm latest snapshot, the storm (12 waiters) is 30 minutes + earlier at 500, so the two peaks are two different moments. */ + await SeedGrantAsync(@base.AddMinutes(-30), waiters: 12, timeoutsDelta: 3, grantedMb: 500); + await SeedGrantAsync(@base, waiters: 0, timeoutsDelta: 0, grantedMb: 6000); var root = Parse(await McpMemoryTools.GetResourceSemaphore(_dataService, _serverManager, ServerName, 1, anchor)); @@ -109,6 +111,7 @@ public async Task GetResourceSemaphore_ReadsTheWindowBesideTheStampedSnapshot() Assert.Equal(3, window.GetProperty("timeout_errors_in_window").GetInt64()); Assert.Equal(0, window.GetProperty("forced_grants_in_window").GetInt64()); Assert.Equal(6000d, window.GetProperty("peak_granted_memory_mb").GetDouble()); + Assert.Equal(Stamp(@base), window.GetProperty("peak_granted_at").GetString()); Assert.Equal(2000d, window.GetProperty("min_available_memory_mb").GetDouble()); Assert.Equal(Stamp(@base.AddMinutes(-30)), window.GetProperty("first_snapshot_at").GetString()); /* The window's last snapshot IS the stamped one — one span, one set of rows, two halves. */ @@ -144,6 +147,7 @@ public async Task GetMemoryGrants_ReadsThePoolWindow_WithANullSemaphoreId() Assert.Equal(12, window.GetProperty("peak_waiter_count").GetInt64()); Assert.Equal(3, window.GetProperty("timeout_errors_in_window").GetInt64()); Assert.Equal(6000d, window.GetProperty("peak_granted_memory_mb").GetDouble()); + Assert.Equal(Stamp(@base.AddMinutes(-30)), window.GetProperty("peak_granted_at").GetString()); Assert.Equal(2, window.GetProperty("snapshots_in_window").GetInt64()); } diff --git a/Lite/Mcp/McpMemoryTools.cs b/Lite/Mcp/McpMemoryTools.cs index e3c1caf82c..09a6604d26 100644 --- a/Lite/Mcp/McpMemoryTools.cs +++ b/Lite/Mcp/McpMemoryTools.cs @@ -257,7 +257,7 @@ public static async Task GetMemoryPressureEvents( } } - [McpServerTool(Name = "get_resource_semaphore"), Description("Resource semaphore stats: granted/available memory vs target/max-target ceiling, waiters, timeout/forced pressure, window ends at as_of. TWO READS, per semaphore+pool: grants[] is the NEWEST snapshot (moment: captured_at/age_seconds); window[] aggregates EVERY snapshot (peak waiters+when, peak/min memory, summed timeout/forced deltas). Read window[] for 'was there pressure', grants[] for 'is there pressure now'. No rows: unavailable (not_collected first). sample_interval_seconds/interval_known null/false on a restart-marker or pre-column row; its zero deltas aren't 'no timeouts'. <> Gets resource semaphore statistics showing granted vs available workspace memory against the target/max-target ceiling, waiter counts, and timeout/forced grant pressure indicators. High waiter counts or rising timeout/forced deltas indicate memory grant pressure affecting query performance. TWO READS UNDER ONE WINDOW: grants[] is the NEWEST snapshot in the window (one row per resource semaphore and pool), stamped once as captured_at with age_seconds against the window's end - it is a moment, not the window. window[] aggregates EVERY snapshot in the window per (resource_semaphore_id, pool_id): peak_waiter_count and peak_waiters_at (the most sessions ever seen waiting for a grant and when), peak_granted_memory_mb, min_available_memory_mb, and timeout_errors_in_window / forced_grants_in_window (the SUM of the per-interval deltas across the window). Each grants[] row also carries sample_interval_seconds, the measured seconds its two deltas accrued over; it is null with interval_known false when the row is a restart marker (no delta was knowable, so the zero deltas beside it are not 'no timeouts') or predates the column. interval_seconds is the interval these deltas cover, null when it can't be known - the same name the latch and spinlock tools use; sample_interval_seconds carries the same value and is kept for existing readers. " + McpToolGuideTopics.MemoryGrantWindowReadOrder)] + [McpServerTool(Name = "get_resource_semaphore"), Description("Resource semaphore stats: granted/available memory vs target/max-target ceiling, waiters, timeout/forced pressure, window ends at as_of. TWO READS, per semaphore+pool: grants[] is the NEWEST snapshot (moment: captured_at/age_seconds); window[] aggregates EVERY snapshot (peak waiters+when, peak/min memory, summed timeout/forced deltas). Read window[] for 'was there pressure', grants[] for 'is there pressure now'. No rows: unavailable (not_collected first). sample_interval_seconds/interval_known null/false on a restart-marker or pre-column row; its zero deltas aren't 'no timeouts'. <> Gets resource semaphore statistics showing granted vs available workspace memory against the target/max-target ceiling, waiter counts, and timeout/forced grant pressure indicators. High waiter counts or rising timeout/forced deltas indicate memory grant pressure affecting query performance. TWO READS UNDER ONE WINDOW: grants[] is the NEWEST snapshot in the window (one row per resource semaphore and pool), stamped once as captured_at with age_seconds against the window's end - it is a moment, not the window. window[] aggregates EVERY snapshot in the window per (resource_semaphore_id, pool_id): peak_waiter_count and peak_waiters_at (the most sessions ever seen waiting for a grant and when), peak_granted_memory_mb and peak_granted_at (the largest granted memory and when - not the time of peak waiters), min_available_memory_mb, and timeout_errors_in_window / forced_grants_in_window (the SUM of the per-interval deltas across the window). Each grants[] row also carries sample_interval_seconds, the measured seconds its two deltas accrued over; it is null with interval_known false when the row is a restart marker (no delta was knowable, so the zero deltas beside it are not 'no timeouts') or predates the column. interval_seconds is the interval these deltas cover, null when it can't be known - the same name the latch and spinlock tools use; sample_interval_seconds carries the same value and is kept for existing readers. " + McpToolGuideTopics.MemoryGrantWindowReadOrder)] public static async Task GetResourceSemaphore( LocalDataService dataService, ServerManager serverManager, @@ -331,7 +331,7 @@ the original sample_interval_seconds rather than in place of it. */ } } - [McpServerTool(Name = "get_memory_grants"), Description("Gets resource semaphore stats: granted vs available workspace memory per resource pool, waiters, timeout/forced grant deltas, over a window ending at as_of (default 1h). TWO READS: grants[] is the NEWEST snapshot in the window (a moment: captured_at/age_seconds), one row per pool; window[] aggregates EVERY snapshot in it, per pool (peak waiters+when, peak/min memory, summed timeout/forced deltas). Read window[] for 'was there pressure', grants[] for 'is there pressure now'. No rows: unavailable (or not_collected first). <> Gets resource semaphore statistics showing granted vs available workspace memory per resource pool, waiter counts, and timeout/forced grant deltas. High waiter counts or rising timeout deltas indicate memory grant pressure affecting query performance. TWO READS UNDER ONE WINDOW: grants[] is the NEWEST snapshot in the window (one row per pool, summed across its semaphores), stamped once as captured_at with age_seconds against the window's end - it is a moment, not the window. window[] aggregates EVERY snapshot in the window per pool: peak_waiter_count and peak_waiters_at (the most sessions ever seen waiting on the pool at one instant and when), peak_granted_memory_mb, min_available_memory_mb, and timeout_errors_in_window / forced_grants_in_window (the SUM of the per-interval deltas across the window). " + McpToolGuideTopics.MemoryGrantWindowReadOrder)] + [McpServerTool(Name = "get_memory_grants"), Description("Gets resource semaphore stats: granted vs available workspace memory per resource pool, waiters, timeout/forced grant deltas, over a window ending at as_of (default 1h). TWO READS: grants[] is the NEWEST snapshot in the window (a moment: captured_at/age_seconds), one row per pool; window[] aggregates EVERY snapshot in it, per pool (peak waiters+when, peak/min memory, summed timeout/forced deltas). Read window[] for 'was there pressure', grants[] for 'is there pressure now'. No rows: unavailable (or not_collected first). <> Gets resource semaphore statistics showing granted vs available workspace memory per resource pool, waiter counts, and timeout/forced grant deltas. High waiter counts or rising timeout deltas indicate memory grant pressure affecting query performance. TWO READS UNDER ONE WINDOW: grants[] is the NEWEST snapshot in the window (one row per pool, summed across its semaphores), stamped once as captured_at with age_seconds against the window's end - it is a moment, not the window. window[] aggregates EVERY snapshot in the window per pool: peak_waiter_count and peak_waiters_at (the most sessions ever seen waiting on the pool at one instant and when), peak_granted_memory_mb and peak_granted_at (the largest granted memory and when - not the time of peak waiters), min_available_memory_mb, and timeout_errors_in_window / forced_grants_in_window (the SUM of the per-interval deltas across the window). " + McpToolGuideTopics.MemoryGrantWindowReadOrder)] public static async Task GetMemoryGrants( LocalDataService dataService, ServerManager serverManager, @@ -406,6 +406,7 @@ public static async Task GetMemoryGrants( peak_waiter_count = w.PeakWaiterCount, peak_waiters_at = w.PeakWaitersAt.ToString("o"), peak_granted_memory_mb = Math.Round(w.PeakGrantedMemoryMb, 2), + peak_granted_at = w.PeakGrantedAt.ToString("o"), min_available_memory_mb = Math.Round(w.MinAvailableMemoryMb, 2), timeout_errors_in_window = w.TimeoutErrorsInWindow, forced_grants_in_window = w.ForcedGrantsInWindow diff --git a/Lite/Services/LocalDataService.MemoryGrants.cs b/Lite/Services/LocalDataService.MemoryGrants.cs index 928a86fce9..9ecf4f0b59 100644 --- a/Lite/Services/LocalDataService.MemoryGrants.cs +++ b/Lite/Services/LocalDataService.MemoryGrants.cs @@ -368,6 +368,15 @@ SELECT DISTINCT ON (resource_semaphore_id, pool_id) collection_time AS peak_waiters_at FROM windowed ORDER BY resource_semaphore_id, pool_id, waiter_count DESC, collection_time DESC +), +peak_grant AS +( + SELECT DISTINCT ON (resource_semaphore_id, pool_id) + resource_semaphore_id, + pool_id, + collection_time AS peak_granted_at + FROM windowed + ORDER BY resource_semaphore_id, pool_id, granted_memory_mb DESC NULLS LAST, collection_time DESC ) SELECT a.resource_semaphore_id, @@ -380,11 +389,15 @@ FROM windowed a.peak_granted_memory_mb, a.min_available_memory_mb, a.timeout_errors_in_window, - a.forced_grants_in_window + a.forced_grants_in_window, + pg.peak_granted_at FROM agg AS a JOIN peak AS p ON p.resource_semaphore_id = a.resource_semaphore_id AND p.pool_id = a.pool_id +JOIN peak_grant AS pg + ON pg.resource_semaphore_id = a.resource_semaphore_id + AND pg.pool_id = a.pool_id ORDER BY a.pool_id, a.resource_semaphore_id"; /// @@ -393,7 +406,7 @@ JOIN peak AS p /// serves), THEN the window's peak / floor / total is taken over /// those per-snapshot pool figures, so peak waiters is the most sessions waiting on the pool at one instant, /// not the largest single semaphore's count. resource_semaphore_id is a typed NULL so both reads - /// project the same eleven columns. $1 server_id, $2 window start, $3 window end. + /// project the same twelve columns. $1 server_id, $2 window start, $3 window end. /// public const string MemoryGrantsWindowSql = @" WITH per_snapshot AS @@ -434,6 +447,14 @@ SELECT DISTINCT ON (pool_id) collection_time AS peak_waiters_at FROM per_snapshot ORDER BY pool_id, waiter_count DESC, collection_time DESC +), +peak_grant AS +( + SELECT DISTINCT ON (pool_id) + pool_id, + collection_time AS peak_granted_at + FROM per_snapshot + ORDER BY pool_id, granted_memory_mb DESC NULLS LAST, collection_time DESC ) SELECT CAST(NULL AS INTEGER) AS resource_semaphore_id, @@ -446,9 +467,11 @@ FROM per_snapshot a.peak_granted_memory_mb, a.min_available_memory_mb, a.timeout_errors_in_window, - a.forced_grants_in_window + a.forced_grants_in_window, + pg.peak_granted_at FROM agg AS a JOIN peak AS p ON p.pool_id = a.pool_id +JOIN peak_grant AS pg ON pg.pool_id = a.pool_id ORDER BY a.pool_id"; /// Every snapshot in the window aggregated per (resource_semaphore_id, pool_id) — the window half @@ -463,7 +486,7 @@ public Task> GetResourceSemaphoreWindowAsync(int serv public Task> GetMemoryGrantsWindowAsync(int serverId, int hoursBack = 24, DateTime? fromDate = null, DateTime? toDate = null, DateTime? asOfUtc = null) => ReadMemoryGrantWindowAsync(MemoryGrantsWindowSql, "GetMemoryGrantsWindowAsync", serverId, hoursBack, fromDate, toDate, asOfUtc); - /// The two window reads project the SAME eleven columns in the same order, so one materialiser + /// The two window reads project the SAME twelve columns in the same order, so one materialiser /// serves both. DuckDB widens SUM(INTEGER) to HUGEINT and COUNT(*) to BIGINT, so every count goes through /// ToInt64 and every MB figure through ToDouble, like the readers above. private async Task> ReadMemoryGrantWindowAsync( @@ -496,7 +519,8 @@ private async Task> ReadMemoryGrantWindowAsync( PeakGrantedMemoryMb = reader.IsDBNull(7) ? 0 : ToDouble(reader.GetValue(7)), MinAvailableMemoryMb = reader.IsDBNull(8) ? 0 : ToDouble(reader.GetValue(8)), TimeoutErrorsInWindow = reader.IsDBNull(9) ? 0 : ToInt64(reader.GetValue(9)), - ForcedGrantsInWindow = reader.IsDBNull(10) ? 0 : ToInt64(reader.GetValue(10)) + ForcedGrantsInWindow = reader.IsDBNull(10) ? 0 : ToInt64(reader.GetValue(10)), + PeakGrantedAt = reader.GetDateTime(11) }); } return items; @@ -521,6 +545,8 @@ public class MemoryGrantWindowRow public long PeakWaiterCount { get; set; } public DateTime PeakWaitersAt { get; set; } public double PeakGrantedMemoryMb { get; set; } + /// The snapshot of the largest granted memory (#5644 W9) - not the time of peak waiters. + public DateTime PeakGrantedAt { get; set; } public double MinAvailableMemoryMb { get; set; } public long TimeoutErrorsInWindow { get; set; } public long ForcedGrantsInWindow { get; set; } From 43f54355b9bee6784a008cad0bb07057fde2fc87 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:16:12 -0400 Subject: [PATCH 04/42] CPU collector no longer stores the newest ring-buffer record twice when SYSDATETIME and ms_ticks drift (#5644 W3) --- .../CpuUtilizationCollectorDefinitionTests.cs | 30 +++++++++++++++++++ .../CpuUtilizationCollector.cs | 22 ++++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/Lite.Tests/CpuUtilizationCollectorDefinitionTests.cs b/Lite.Tests/CpuUtilizationCollectorDefinitionTests.cs index fa8e95a8c7..6a909eb1b8 100644 --- a/Lite.Tests/CpuUtilizationCollectorDefinitionTests.cs +++ b/Lite.Tests/CpuUtilizationCollectorDefinitionTests.cs @@ -261,6 +261,36 @@ public async Task ReadAsync_RingBuffer_UnderAUtcWatermark_DedupsAtOrBelowItOnThe Assert.True(row.SampleTime < watermarkUtc, "the fixture must put every LOCAL stamp below the UTC watermark, or this pin cannot tell the two frames apart"); } + /// + /// #5644 W3: the ring buffer record that is already stored comes back on the next poll with a recomputed + /// stamp a few milliseconds ABOVE the stored one (SYSDATETIME() and ms_ticks drift apart: 8 ms in 68 s on the + /// test box), so a strict `<= watermark` test stored it again, 82-87 samples an hour against 60 records. + /// The same record, drifted 8 ms and then 3 s above the watermark, is dropped in either frame; the next + /// record, 60 s later, still lands. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReadAsync_RingBuffer_AlreadyStoredRecordWithDriftedStamp_IsNotStoredAgain(bool fromUtc) + { + var utcBase = new DateTime(2026, 7, 1, 16, 0, 0, DateTimeKind.Utc); + var localBase = utcBase.AddHours(-4); /* the server sits at UTC-4 */ + object[] Row(TimeSpan at, int cpu) => new object[] { localBase + at, cpu, 1, utcBase + at }; + using var reader = new FakeCollectorDataReader( + Row(TimeSpan.FromSeconds(60.008), 50), /* the genuinely new record */ + Row(TimeSpan.FromMilliseconds(8), 40), /* the stored record, recomputed 8 ms later */ + Row(TimeSpan.FromSeconds(3), 41), /* a larger drift still inside the window */ + Row(TimeSpan.Zero, 30)); /* the stored record exactly */ + + var rows = await CpuUtilizationCollector.Instance.ReadAsync( + reader, + CollectorTestContext.Make(s_deltas, isAzureSqlDb: false, watermark: fromUtc ? utcBase : localBase, watermarkFromUtcColumn: fromUtc), + CancellationToken.None); + + var row = Assert.Single(rows); + Assert.Equal(50, row.SqlServerCpuUtilization); + } + /* ── #3778: the three acceptance sequences, through the REAL ReadAsync dedup rather than WritePayload ── */ /// diff --git a/PerformanceMonitor.Collectors/CpuUtilizationCollector.cs b/PerformanceMonitor.Collectors/CpuUtilizationCollector.cs index fb4602e50b..f1fe956c5a 100644 --- a/PerformanceMonitor.Collectors/CpuUtilizationCollector.cs +++ b/PerformanceMonitor.Collectors/CpuUtilizationCollector.cs @@ -181,7 +181,7 @@ EXEC sys.sp_executesql sub-second remainder of the elapsed-ticks offset, and that remainder is different on every poll (SYSDATETIME() keeps moving while a given ring-buffer entry's own timestamp does not). Since this query re-reads the ring buffer's whole retained history every cycle and dedups client-side on this - COMPUTED value (a physical entry has no stable key), a recomputed sample_time that drifts forward + COMPUTED value (the store keeps no column for the record id; see RingBufferResampleTolerance), a recomputed sample_time that drifts forward by a fraction of a second on a later poll can land just above the watermark and re-insert the same entry as a fresh row - a near-duplicate a moment after the original. Issue #2749: this duplicate pattern (two samples ~200ms-1s apart, then a real ~60s gap to the next) collapses @@ -304,6 +304,23 @@ public override CollectorQuery BuildQuery(CollectorContext context) : new CollectorQuery(RingBufferQueryText); } + /// + /// #5644 W3: how far past the watermark a ring-buffer stamp may sit and still be the record the watermark + /// already holds. A SCHEDULER_MONITOR record IS stable across reads (measured on SQL Server 2022, a minute + /// apart: the same Record/@id and the same ring-buffer timestamp), but the query's + /// sample_time is recomputed every poll from SYSDATETIME() minus ms_ticks - timestamp, two + /// clocks that run slightly apart (8 ms in 68 s on the test box). Re-reading the newest stored record + /// therefore lands a few milliseconds ABOVE its stored stamp, clears a strict <= watermark test, + /// and is stored again: 82-87 samples an hour where the ring buffer holds 60. The store has no column for the + /// record id, so the watermark is the only memory; records are 60 s apart, so anything within this window of + /// it is the same record. 20 s is far above any clock drift between polls and far below the spacing, so a + /// genuinely new record still lands. + /// + internal static readonly TimeSpan RingBufferResampleTolerance = TimeSpan.FromSeconds(20); + + private static DateTime WithRingBufferTolerance(DateTime watermark) => + watermark > DateTime.MaxValue - RingBufferResampleTolerance ? watermark : watermark + RingBufferResampleTolerance; + public override async ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) { var rows = new List(); @@ -328,7 +345,8 @@ old rule would not have. A row with no twin under a UTC watermark cannot be plac than compared across frames or dropped on a guess. No arm produces one — the pins say so — and if one ever did, a visible duplicate beats a silent hole. */ var stampInWatermarkFrame = context.WatermarkFromUtcColumn ? sampleTimeUtc : sampleTime; - if (!context.Target.IsAzureSqlDb && context.Watermark.HasValue && stampInWatermarkFrame <= context.Watermark.Value) + if (!context.Target.IsAzureSqlDb && context.Watermark.HasValue + && stampInWatermarkFrame <= WithRingBufferTolerance(context.Watermark.Value)) { continue; } From de985f18baf74bd0030599d2aa78b8118bca3328 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:24:47 -0400 Subject: [PATCH 05/42] Blocking events count waits, not blocked-process reports (#5644 W1, W2) --- .../PgFactCollector.Waits.cs | 34 ++-- .../Mcp/DarlingFleetReader.cs | 30 +++- .../Mcp/DarlingHealthReader.cs | 15 +- .../ViewerDataService.AzureMasterScope.cs | 15 +- .../ViewerDataService.Fleet.cs | 27 ++- .../ViewerDataService.Overview.cs | 18 +- Lite/Analysis/DuckDbFactCollector.Waits.cs | 13 +- Lite/Services/LocalDataService.Overview.cs | 16 +- .../AlertContextBuilders.cs | 29 +++- PerformanceMonitor.Alerting/AlertEngine.cs | 36 ++-- PerformanceMonitor.Common/BlockingWaits.cs | 163 ++++++++++++++++++ 11 files changed, 325 insertions(+), 71 deletions(-) create mode 100644 PerformanceMonitor.Common/BlockingWaits.cs diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs index e63493ad0b..d65281769d 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs @@ -13,6 +13,7 @@ using Npgsql; using PerformanceMonitor.Analysis; using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; namespace PerformanceMonitor.Darling.Analysis; @@ -264,19 +265,28 @@ the window is when the report HAPPENED (event_time), the same column the blockin EventWindowFloor for $2: blocked_process_reports is partitioned on collection_time, which an event_time bound gives the planner nothing to exclude chunks on. The bucket moves with the window: a bucket origin on one column over rows windowed on the other would split the peak across buckets. */ - private const string BlockingSqlHead = @" -WITH reports AS ( + /* #5644 W1: the rows the fact counts are WAITS, not blocked-process reports. SQL Server writes a report for + a still-blocked session every threshold seconds, so one one-minute wait was about eleven events and the + graded rate overstated blocking by the same factor. BlockingWaits.WaitsCteSql is the one definition (a + blocked session, its blocker and the wait's derived start; the longest wait time kept) that the alert + engine and the fleet tile share, so this fact's event_count, the alert value and the Blocking tile agree. + The wait lands in the 4-hour bucket of its longest-wait (latest) report. The (10, 50) grading pair was + measured on reports; it now meets waits, and the PR notes that. */ + private static string BuildBlockingSql(string extraWhere) => @" +WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms, blocking_status + FROM blocked_process_reports + WHERE server_id = $1 + AND event_time >= $2 + AND event_time <= $3" + extraWhere) + @", +reports AS ( SELECT wait_time_ms, blocking_spid, blocking_status, date_bin('4 hours', event_time, $2) AS bucket_start - FROM blocked_process_reports - WHERE server_id = $1 - AND event_time >= $2 - AND event_time <= $3"; - - private const string BlockingSqlTail = @" + FROM waits ), buckets AS ( SELECT COUNT(*) AS bucket_event_count @@ -292,16 +302,16 @@ GROUP BY bucket_start (SELECT COALESCE(MAX(bucket_event_count), 0) FROM buckets) AS peak_4h_event_count FROM reports"; - public const string BlockingSql = BlockingSqlHead + @" - AND collection_time >= $4" + BlockingSqlTail; + public static string BlockingSql => BuildBlockingSql(@" + AND collection_time >= $4"); /// The same read for an Azure SQL Database master target: $4 is the names of the databases monitored /// as their own targets, whose events are skipped (a NULL database still counts), and $5 the event-window /// floor. The list keeps its number from the scoped read and the floor takes the next one, so the floor is /// $4 in and $5 here. - public const string BlockingSqlSkippingSeparate = BlockingSqlHead + @" + public static string BlockingSqlSkippingSeparate => BuildBlockingSql(@" AND collection_time >= $5 - AND (database_name IS NULL OR NOT (lower(database_name) = ANY(SELECT lower(x) FROM unnest($4::text[]) x)))" + BlockingSqlTail; + AND (database_name IS NULL OR NOT (lower(database_name) = ANY(SELECT lower(x) FROM unnest($4::text[]) x)))"); /// The peak sub-window's width in hours — the grain the (10, 50) grading pair was measured /// on (#3871). The peak rate divides by min(this, observed hours), never by the constant diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index 2134982224..c13fd04fd4 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -342,7 +342,25 @@ WHERE s.is_enabled /// $3 is the partition-column bound the event window cannot supply (#3895): without it both scans /// open every retained chunk to count the last hour. It cannot drop a row the window wants — /// says why. - public const string FleetBlockingSql = @" + /* #5644 W1: each side counts WAITS (a blocked session, its blocker and the wait's derived start), not report + rows, with the longest wait time as its max: SQL Server writes a report for a still-blocked session every + threshold seconds, so one one-minute wait was about eleven tile counts. BlockingWaits.WaitsCteSql is the + one definition the alert engine and the analysis fact share. The DMV side applies the same rule to its + snapshot rows (a wait seen in several snapshots is one wait). */ + public static readonly string FleetBlockingSql = @" +WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "xe_waits", + @"SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms + FROM v_blocked_process_reports + WHERE event_time >= $1 + AND event_time <= $2 + AND collection_time >= $3", new[] { "server_id" }) + "," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "dmv_waits", + @"SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms + FROM v_dmv_blocking_snapshots + WHERE event_time >= $1 + AND event_time <= $2 + AND collection_time >= $3", new[] { "server_id" }) + @" SELECT COALESCE(xe.server_id, dmv.server_id) AS server_id, COALESCE(xe.cnt, 0) AS xe_count, @@ -352,19 +370,13 @@ WHERE s.is_enabled FROM ( SELECT server_id, COUNT(*) AS cnt, MAX(wait_time_ms) AS max_wait - FROM v_blocked_process_reports - WHERE event_time >= $1 - AND event_time <= $2 - AND collection_time >= $3 + FROM xe_waits GROUP BY server_id ) AS xe FULL OUTER JOIN ( SELECT server_id, COUNT(*) AS cnt, MAX(wait_time_ms) AS max_wait - FROM v_dmv_blocking_snapshots - WHERE event_time >= $1 - AND event_time <= $2 - AND collection_time >= $3 + FROM dmv_waits GROUP BY server_id ) AS dmv ON xe.server_id = dmv.server_id"; diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs index a5a07921bc..56d4f0f241 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs @@ -87,10 +87,19 @@ ORDER BY collection_time DESC /// $3 the for $2 — the partition-column bound the event window cannot supply /// (#3895, the fleet card's DarlingFleetReader.FleetBlockingSql carries the same one), so the counts /// open the window's chunks rather than every retained one. - public const string ServerSummaryBlockingSql = @" + /// #5644 W1: both counts are WAITS, not report rows (, the + /// definition the alert engine, the analysis fact and the fleet tile share), so the summary agrees with the tile. + public static readonly string ServerSummaryBlockingSql = @" +WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "xe_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms FROM v_blocked_process_reports + WHERE server_id = $1 AND event_time >= $2 AND collection_time >= $3") + "," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "dmv_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms FROM v_dmv_blocking_snapshots + WHERE server_id = $1 AND event_time >= $2 AND collection_time >= $3") + @" SELECT - (SELECT COUNT(*) FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND collection_time >= $3), - (SELECT COUNT(*) FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND event_time >= $2 AND collection_time >= $3)"; + (SELECT COUNT(*) FROM xe_waits), + (SELECT COUNT(*) FROM dmv_waits)"; /// Deadlock count in the window. $1 server_id, $2 window start, $3 the /// for $2 (#3895). diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AzureMasterScope.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AzureMasterScope.cs index b38afbe44a..9b64ec35ff 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AzureMasterScope.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AzureMasterScope.cs @@ -12,6 +12,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; +using PerformanceMonitor.Common; using PerformanceMonitor.Alerting; using PerformanceMonitor.Darling.Analysis; using PerformanceMonitor.Darling.Storage; @@ -182,10 +183,18 @@ private async Task ReadScopedDeadlocksAsync( serverId, start, end, separate, cancellationToken, ViewerCommandDeadlines.CurrentInteractiveReadSeconds); } - private const string FleetMasterUnscopedSql = @" + /* #5644 W1: waits, not report rows (BlockingWaits.WaitsCteSql), so it subtracts like for like from the fleet totals. */ + private static readonly string FleetMasterUnscopedSql = @" +WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "xe_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms FROM v_blocked_process_reports + WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $4") + "," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "dmv_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms FROM v_dmv_blocking_snapshots + WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $4") + @" SELECT - (SELECT COUNT(*) FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $4), - (SELECT COUNT(*) FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $4), + (SELECT COUNT(*) FROM xe_waits), + (SELECT COUNT(*) FROM dmv_waits), (SELECT COUNT(*) FROM v_deadlocks WHERE server_id = $1 AND deadlock_time >= $2 AND deadlock_time <= $3 AND collection_time >= $4)"; /// diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Fleet.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Fleet.cs index 3b799881e8..b092904080 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Fleet.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Fleet.cs @@ -86,7 +86,22 @@ public sealed partial class ViewerDataService /// chunk to count the last hour — 69.8 ms on DARLING01, 1.1 ms with it. The PostgreSQL arm is already /// windowed on collection_time and needs no floor. /// - public const string FleetTotalsSql = @" + /* #5644 W1: the blocking total sums WAITS (BlockingWaits.WaitsCteSql), not report rows, as the service's fleet + tile does, so the viewer and the web UI show the same number. */ + public static readonly string FleetTotalsSql = @" +WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "xe_waits", + @"SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms + FROM v_blocked_process_reports + WHERE event_time >= $1 + AND event_time <= $2 + AND collection_time >= $3", new[] { "server_id" }) + "," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "dmv_waits", + @"SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms + FROM v_dmv_blocking_snapshots + WHERE event_time >= $1 + AND event_time <= $2 + AND collection_time >= $3", new[] { "server_id" }) + @" SELECT ( SELECT COALESCE(SUM(CASE WHEN per_server.xe_count > 0 THEN per_server.xe_count ELSE per_server.dmv_count END), 0) @@ -99,19 +114,13 @@ SELECT COALESCE(SUM(CASE WHEN per_server.xe_count > 0 THEN per_server.xe_count E FROM ( SELECT server_id, COUNT(*) AS cnt - FROM v_blocked_process_reports - WHERE event_time >= $1 - AND event_time <= $2 - AND collection_time >= $3 + FROM xe_waits GROUP BY server_id ) AS xe FULL OUTER JOIN ( SELECT server_id, COUNT(*) AS cnt - FROM v_dmv_blocking_snapshots - WHERE event_time >= $1 - AND event_time <= $2 - AND collection_time >= $3 + FROM dmv_waits GROUP BY server_id ) AS dmv ON xe.server_id = dmv.server_id ) AS per_server diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs index 58ab9675b8..e58cf58639 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs @@ -159,12 +159,20 @@ FROM v_cpu_scheduler_stats /// and probe the window's chunks instead of every retained one. The two "ever" reads keep no bound: /// they ask for the newest event in all of history, which a floor would change the answer to. /// - public const string ServerSummaryBlockingSql = @" + /* #5644 W1: the two counts are WAITS (BlockingWaits.WaitsCteSql), not report rows, as the fleet tile's are. */ + public static readonly string ServerSummaryBlockingSql = @" +WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "xe_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms FROM v_blocked_process_reports + WHERE server_id = $1 AND event_time >= $2 AND collection_time >= $3") + "," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "dmv_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms FROM v_dmv_blocking_snapshots + WHERE server_id = $1 AND event_time >= $2 AND collection_time >= $3") + @" SELECT - (SELECT COUNT(*) FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND collection_time >= $3), - (SELECT MAX(wait_time_ms) FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND collection_time >= $3), - (SELECT COUNT(*) FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND event_time >= $2 AND collection_time >= $3), - (SELECT MAX(wait_time_ms) FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND event_time >= $2 AND collection_time >= $3), + (SELECT COUNT(*) FROM xe_waits), + (SELECT MAX(wait_time_ms) FROM xe_waits), + (SELECT COUNT(*) FROM dmv_waits), + (SELECT MAX(wait_time_ms) FROM dmv_waits), (SELECT MAX(event_time) FROM v_blocked_process_reports WHERE server_id = $1), (SELECT MAX(event_time) FROM v_dmv_blocking_snapshots WHERE server_id = $1)"; diff --git a/Lite/Analysis/DuckDbFactCollector.Waits.cs b/Lite/Analysis/DuckDbFactCollector.Waits.cs index e65251cbf8..5e1b32869a 100644 --- a/Lite/Analysis/DuckDbFactCollector.Waits.cs +++ b/Lite/Analysis/DuckDbFactCollector.Waits.cs @@ -6,6 +6,7 @@ using DuckDB.NET.Data; using PerformanceMonitor.Analysis; using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; using PerformanceMonitor.PlanAnalysis; using PerformanceMonitorLite.Database; @@ -246,14 +247,22 @@ private async Task CollectBlockingFactsAsync(AnalysisContext context, List using var command = connection.CreateCommand(); var scopeList = context.SeparatelyMonitoredDatabases; var scopeFilter = SeparatelyMonitoredScope.BprFilter(scopeList, 4); + /* #5644 W1: the rows the fact counts are WAITS (BlockingWaits.WaitsCteSql: a blocked session, its blocker + and the wait's derived start, the longest wait time kept), not blocked-process reports, which SQL + Server writes for a still-blocked session every threshold seconds. The PG collector counts the same + way, from the same definition. */ command.CommandText = (@" -WITH reports AS ( +WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.DuckDb, "waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms, blocking_status + FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3{SCOPE}") + @" AS ev") + @", +reports AS ( SELECT wait_time_ms, blocking_spid, blocking_status, time_bucket(INTERVAL '4 hours', event_time, $2) AS bucket_start - FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3{SCOPE}") + @" AS ev + FROM waits ), buckets AS ( SELECT COUNT(*) AS bucket_event_count diff --git a/Lite/Services/LocalDataService.Overview.cs b/Lite/Services/LocalDataService.Overview.cs index e9d361ba64..68100e79aa 100644 --- a/Lite/Services/LocalDataService.Overview.cs +++ b/Lite/Services/LocalDataService.Overview.cs @@ -119,11 +119,21 @@ A null or empty list leaves today's SQL untouched. */ /* Blocking count in last hour - uses XE blocked process reports */ using (var cmd = connection.CreateCommand()) { - /* Prefer the blocked-process-report; fall back to the always-on DMV snapshot (AWS RDS). */ + /* Prefer the blocked-process-report; fall back to the always-on DMV snapshot (AWS RDS). + #5644 W1: both sides count WAITS, not report rows (BlockingWaits.WaitsCteSql, the definition the + alert engine, the analysis fact and Darling's fleet tile share): SQL Server writes a report for a + still-blocked session every threshold seconds, so one one-minute wait counted about eleven times. */ cmd.CommandText = @" +WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.DuckDb, "xe_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms + FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2" + SeparatelyMonitoredScope.BprFilter(separate, 3)) + @" AS ev") + "," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.DuckDb, "dmv_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms + FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND event_time >= $2" + SeparatelyMonitoredScope.BprFilter(separate, 3)) + @" SELECT COALESCE(NULLIF( - (SELECT COUNT(*) FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2" + SeparatelyMonitoredScope.BprFilter(separate, 3)) + @" AS ev), 0), - (SELECT COUNT(*) FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND event_time >= $2" + SeparatelyMonitoredScope.BprFilter(separate, 3) + @"))"; + (SELECT COUNT(*) FROM xe_waits), 0), + (SELECT COUNT(*) FROM dmv_waits))"; cmd.Parameters.Add(new DuckDBParameter { Value = serverId }); cmd.Parameters.Add(new DuckDBParameter { Value = DateTime.UtcNow.AddHours(-1) }); SeparatelyMonitoredScope.AddParameters(cmd, separate); diff --git a/PerformanceMonitor.Alerting/AlertContextBuilders.cs b/PerformanceMonitor.Alerting/AlertContextBuilders.cs index 32985abb21..3592010296 100644 --- a/PerformanceMonitor.Alerting/AlertContextBuilders.cs +++ b/PerformanceMonitor.Alerting/AlertContextBuilders.cs @@ -61,7 +61,7 @@ public static class AlertContextBuilders { if (events == null || events.Count == 0) return null; - var filtered = FilterBlocking(events, excludedDatabases); + var filtered = BlockingWaitRows(events, excludedDatabases); if (filtered.Count == 0) return null; var groups = GroupBlocking(serverName, filtered); @@ -222,7 +222,7 @@ public static IReadOnlyList BlockingIncidents( { if (events == null || events.Count == 0) return Array.Empty(); - var filtered = FilterBlocking(events, excludedDatabases); + var filtered = BlockingWaitRows(events, excludedDatabases); if (filtered.Count == 0) return Array.Empty(); return GroupBlocking(serverName, filtered).Select(g => g.Incident).ToList(); @@ -466,6 +466,31 @@ alert on it. */ return context; } + /// + /// #5644 W1/W2: THE set of blocking events a blocking alert is made of, so the alert's value and its detail + /// cannot disagree. Excluded databases drop their rows, the reports of one wait collapse to one row carrying + /// the longest wait time (), and the count is XE-preferred as Lite's overview has + /// always been: the blocked-process-report waits when there are any, else the DMV-snapshot waits. The alert + /// value is this set's Count; the card's "Blocking chain (xN)" groups are drawn from the same list, so + /// the chain counts add up to the value. + /// + public static IReadOnlyList BlockingWaitRows( + IReadOnlyList? events, IReadOnlyList excludedDatabases) + { + if (events == null || events.Count == 0) return Array.Empty(); + + var waits = BlockingWaits.Collapse( + FilterBlocking(events, excludedDatabases), + r => r.Source ?? "", + r => r.BlockedSpid, + r => r.BlockingSpid, + r => r.EventTime, + r => r.WaitTimeMs); + + var xeWaits = waits.Where(r => r.Source == BlockedProcessAlertRow.XeReportSource).ToList(); + return xeWaits.Count > 0 ? xeWaits : waits; + } + /* Excluded databases drop their rows; rows with no database always pass. Shared by the render path and #2216's observation path so the two can never disagree about which rows exist. */ private static IReadOnlyList FilterBlocking( diff --git a/PerformanceMonitor.Alerting/AlertEngine.cs b/PerformanceMonitor.Alerting/AlertEngine.cs index 2d43d084ea..0d8e69924f 100644 --- a/PerformanceMonitor.Alerting/AlertEngine.cs +++ b/PerformanceMonitor.Alerting/AlertEngine.cs @@ -1196,29 +1196,19 @@ private async Task CheckBlockingAsync( and the fired alert's context (:172) — see class remarks adaptation (1). */ blockingRows = await _readAdapter.GetRecentBlockedProcessReportsAsync(key, hoursBack: RollingCountWindowHours, ct); - /* Lite's overview count semantics (LocalDataService.Overview.cs:74-77): prefer the - XE blocked-process-report count; fall back to the DMV snapshot count when the XE - count is zero (AWS RDS / unset blocked-process threshold). The merged adapter - list contains all XE rows plus only uncovered DMV rows, so when no XE row exists - the merged count IS the DMV count. */ - int xeCount = blockingRows.Count(r => r.Source == BlockedProcessAlertRow.XeReportSource); - effectiveBlockingCount = xeCount > 0 ? xeCount : blockingRows.Count; - - /* :118-127 — with excluded databases configured and the raw count at/over the - threshold, recount only rows outside the excluded set (no-database rows pass). */ - if (excludedDatabases.Count > 0 - && effectiveBlockingCount >= _settings.BlockingCountThreshold) - { - /* The recount applies the raw count's own rule to the same row set: XE rows when any - exist, otherwise every row. */ - var recountRows = xeCount > 0 - ? blockingRows.Where(r => r.Source == BlockedProcessAlertRow.XeReportSource) - : blockingRows; - effectiveBlockingCount = recountRows - .Count(r => string.IsNullOrEmpty(r.DatabaseName) || - !excludedDatabases.Any(e => - string.Equals(e, r.DatabaseName, StringComparison.OrdinalIgnoreCase))); - } + /* #5644 W1/W2: the value is the number of blocking WAITS, not blocked-process reports. + SQL Server reports a still-blocked session every threshold seconds, so a one-minute wait + is about eleven report rows; counting rows alerted on one wait eleven times over. One + event is one blocked session, blocker and wait start, with its longest wait time. + BlockingWaitRows is the one place that set is built, and the card's "Blocking chain (xN)" + groups are drawn from the SAME list, so the value is the sum of the chain counts. It also + carries Lite's overview semantics (LocalDataService.Overview.cs:74-77): the XE + blocked-process-report waits when there are any, else the DMV-snapshot waits (AWS RDS / + unset blocked-process threshold), and the excluded-database filter (:118-127), whose rows + with no database always pass. The old shape counted every row, then recounted outside the + excluded set only once the raw count reached the threshold; filtering before counting + gives the same fire decision and a watermark that tracks the filtered count throughout. */ + effectiveBlockingCount = AlertContextBuilders.BlockingWaitRows(blockingRows, excludedDatabases).Count; } catch (OperationCanceledException) when (ct.IsCancellationRequested) { diff --git a/PerformanceMonitor.Common/BlockingWaits.cs b/PerformanceMonitor.Common/BlockingWaits.cs new file mode 100644 index 0000000000..11a7f07dd1 --- /dev/null +++ b/PerformanceMonitor.Common/BlockingWaits.cs @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace PerformanceMonitor.Common; + +/// +/// #5644 W1: what ONE blocking event is, defined once for the alert engine, the fleet "Blocking N" tile and the +/// analysis BLOCKING_EVENTS fact, in Darling and in Lite. +/// +/// SQL Server writes a blocked-process report for a still-blocked session every +/// blocked process threshold seconds (5 at the lowest), so one one-minute wait is about eleven report rows. +/// Counting rows counted that wait eleven times. Erik's ruling: one event per blocked session, blocker and wait +/// start, reporting the longest wait time. +/// +/// The wait start is derived: the report's time minus its wait_time_ms. Every report of one +/// wait derives the same instant, give or take the milliseconds between the event timestamp and the wait time +/// being read, so two reports of one (blocked, blocker) pair belong to the same wait when their derived starts are +/// within of the previous one (reports sorted by derived start). A different +/// wait of the same pair that was reported at all lasted at least the threshold (5 s) before its first report, +/// so its start is at least that far from the previous wait's start; the tolerance sits well below it. +/// +/// The same rule is written twice because the counts live in two places: for rows +/// already in memory (the alert engine's read), and for the counts that run in the +/// store (the fleet tile and the analysis fact). A test runs both over the same rows and checks they agree. +/// +public static class BlockingWaits +{ + /// How far apart (milliseconds) two reports' derived wait starts may be and still be one wait. The + /// threshold is at least 5 s, so 2 s cannot merge two waits and is far above the few milliseconds of + /// timestamp jitter between reports of one wait (and the whole-second rounding of a store that keeps only + /// seconds). + public const long WaitStartToleranceMs = 2000; + + /// The wait start of one report in epoch milliseconds: event time minus its wait time. + public static double WaitStartEpochMs(DateTime eventTime, long waitTimeMs) => + (eventTime - DateTime.UnixEpoch).TotalMilliseconds - waitTimeMs; + + /// + /// Collapses reports to one row per wait: the report with the longest wait time (the latest one on a tie). + /// keeps rows that must never merge apart (the report source, for one). A row with + /// no event time cannot be placed in a wait, so it stays a wait of its own. The result is ordered by the + /// representative's event time, newest first, as the alert reads are. + /// + public static List Collapse( + IEnumerable reports, + Func partition, + Func blockedSpid, + Func blockingSpid, + Func eventTime, + Func waitTimeMs) + { + var result = new List(); + var placeable = new List<(T Row, double Start, DateTime Time, long Wait)>(); + foreach (var row in reports) + { + var time = eventTime(row); + if (time is null) + { + result.Add(row); + continue; + } + + placeable.Add((row, WaitStartEpochMs(time.Value, waitTimeMs(row)), time.Value, waitTimeMs(row))); + } + + foreach (var pair in placeable.GroupBy(p => (partition(p.Row), blockedSpid(p.Row), blockingSpid(p.Row)))) + { + var ordered = pair.OrderBy(p => p.Start).ThenBy(p => p.Time).ToList(); + var best = ordered[0]; + var previousStart = ordered[0].Start; + for (var i = 1; i < ordered.Count; i++) + { + var current = ordered[i]; + if (current.Start - previousStart > WaitStartToleranceMs) + { + result.Add(best.Row); + best = current; + } + else if (current.Wait > best.Wait || (current.Wait == best.Wait && current.Time >= best.Time)) + { + best = current; + } + + previousStart = current.Start; + } + + result.Add(best.Row); + } + + return result + .OrderByDescending(r => eventTime(r) ?? DateTime.MinValue) + .ToList(); + } + + /// Which store's SQL dialect writes the epoch conversion in. + public enum SqlDialect + { + /// PostgreSQL (Darling's store). + Postgres, + + /// DuckDB (Lite's store). + DuckDb, + } + + /// + /// The CTE chain that turns a report relation into one row per wait, written for both stores from this one + /// text. is a SELECT that must return blocked_spid, blocking_spid, + /// event_time and wait_time_ms, plus whatever else the caller reads (kept as the representative + /// report's values), plus any column named in (for example + /// server_id). The chain ends in a CTE named , one row per wait, whose + /// wait_time_ms is the longest wait of that wait's reports and whose other columns are that + /// report's. The caller continues the WITH list after it (the text ends with the closing parenthesis, no + /// trailing comma). + /// + public static string WaitsCteSql( + SqlDialect dialect, string name, string source, IReadOnlyList? partitionColumns = null) + { + var partition = string.Join( + ", ", (partitionColumns ?? Array.Empty()).Concat(new[] { "blocked_spid", "blocking_spid" })); + var epochMs = dialect == SqlDialect.DuckDb + ? "CAST(epoch_ms(s.event_time) AS DOUBLE)" + : "(date_part('epoch', s.event_time) * 1000.0)"; + var tolerance = WaitStartToleranceMs.ToString(System.Globalization.CultureInfo.InvariantCulture); + return $@" +{name}_marked AS ( + SELECT s.*, {epochMs} - s.wait_time_ms AS wait_start_ms + FROM ({source}) AS s +), +{name}_gapped AS ( + SELECT m.*, + CASE + WHEN LAG(m.wait_start_ms) OVER (PARTITION BY {partition} ORDER BY m.wait_start_ms, m.event_time) IS NULL + OR m.wait_start_ms - LAG(m.wait_start_ms) OVER (PARTITION BY {partition} ORDER BY m.wait_start_ms, m.event_time) > {tolerance} + THEN 1 ELSE 0 + END AS starts_wait + FROM {name}_marked AS m +), +{name}_numbered AS ( + SELECT g.*, + SUM(g.starts_wait) OVER (PARTITION BY {partition} ORDER BY g.wait_start_ms, g.event_time ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS wait_no + FROM {name}_gapped AS g +), +{name}_ranked AS ( + SELECT n.*, + ROW_NUMBER() OVER (PARTITION BY {partition}, n.wait_no ORDER BY n.wait_time_ms DESC, n.event_time DESC) AS wait_rank + FROM {name}_numbered AS n +), +{name} AS ( + SELECT r.* + FROM {name}_ranked AS r + WHERE r.wait_rank = 1 +)"; + } +} From de6cf7abea88221703437445056817284eed79bf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:30:51 -0400 Subject: [PATCH 06/42] Web wording: caller-only clauses, latch/spinlock rate labels, plural, zero-normal collectors (#5644 W8 W10 W13 W15) --- .../CollectionOutputBesideCostTests.cs | 5 +- Darling/Darling.Tests/WebPlainTextTests.cs | 51 ++++++++++ Darling/Darling.Tests/WebWordingFixesTests.cs | 50 ++++++++++ .../ZeroIsNormalCollectorTests.cs | 92 +++++++++++++++++++ .../Mcp/DarlingDataReader.cs | 3 +- .../wwwroot/js/pages/finops/index-analysis.js | 3 +- .../wwwroot/js/pages/finops/utilization.js | 3 +- .../wwwroot/js/pages/server-tabs.js | 21 +++-- .../wwwroot/js/plain-text.js | 24 +++++ .../wwwroot/js/util.js | 11 ++- Lite.Tests/CollectionOutputBesideCostTests.cs | 5 +- Lite.Tests/ZeroIsNormalCollectorTests.cs | 43 +++++++++ .../LocalDataService.CollectionHealth.cs | 3 +- .../ServerHealthBands.cs | 50 +++++++++- 14 files changed, 343 insertions(+), 21 deletions(-) create mode 100644 Darling/Darling.Tests/WebWordingFixesTests.cs create mode 100644 Darling/Darling.Tests/ZeroIsNormalCollectorTests.cs create mode 100644 Lite.Tests/ZeroIsNormalCollectorTests.cs diff --git a/Darling/Darling.Tests/CollectionOutputBesideCostTests.cs b/Darling/Darling.Tests/CollectionOutputBesideCostTests.cs index ce6ce8ab9b..d3930130ee 100644 --- a/Darling/Darling.Tests/CollectionOutputBesideCostTests.cs +++ b/Darling/Darling.Tests/CollectionOutputBesideCostTests.cs @@ -220,10 +220,11 @@ public void TheFindingSignature_TakesTheNoteCount_AndNoNoteText() /* The precondition, named so a signature change reports itself rather than turning the assertions below into a vacuous pass over a list that no longer means what this test thinks. */ - Assert.Equal(6, parameters.Length); + /* #5644: the zero-is-normal bool joined the event bool, computed from a name list the same way. */ + Assert.Equal(7, parameters.Length); Assert.Equal( - new[] { "rowsStored", "totalRuns", "deniedSinceLastSuccess", "noteCount", "faultedRuns", "isEventCollector" }, + new[] { "rowsStored", "totalRuns", "deniedSinceLastSuccess", "noteCount", "faultedRuns", "isEventCollector", "isZeroNormal" }, parameters.Select(p => p.Name!).ToArray()); Assert.DoesNotContain(parameters, p => p.ParameterType == typeof(string)); diff --git a/Darling/Darling.Tests/WebPlainTextTests.cs b/Darling/Darling.Tests/WebPlainTextTests.cs index f87c80755f..eae57e27bc 100644 --- a/Darling/Darling.Tests/WebPlainTextTests.cs +++ b/Darling/Darling.Tests/WebPlainTextTests.cs @@ -97,6 +97,57 @@ public void CollectorMeasurementCounts_ReadAsWords() Assert.Equal("latest run: shred gated: 1, events read: 0, report xml empty: 0", o[0]); } + /* #5644 W8: the clauses that point an MCP caller at a parameter the page has no control for. */ + [Fact] + public void ParameterClauses_ForAnMcpCaller_AreDroppedOrSaidInThePagesWords() + { + var o = Run( + [ + "The width was chosen to keep this answer near 1500 points; pass bucket_minutes (1-1440) for another width, or narrow hours_back for finer points.", + "TRUNCATED: 300 indexes match and 200 were returned. Rows are ordered by total wait time descending; raise limit to see more.", + "TRUNCATED: 9 indexes match database 'x' and 5 were returned. This says nothing about databases outside the filter — raise limit to see more.", + "TRUNCATED: 9 indexes match and 5 were returned. This is not evidence that a database was not collected — pass database_name to ask about one, or raise limit.", + " truncated: 5 of 9 returned; raise limit (now 5) for more.", + "The row limit of 50 was REACHED: databases[] holds the 50 biggest spillers and the totals above still cover every database - raise limit to see the rest of the rows.", + "At least one database's statistics WERE reset in this window - see stats_reset_count per database, and raise limit if none of the databases returned shows one - so its totals are lower bounds.", + "No active query snapshots on SQL2025 in the last 1 hour(s) matched blocking_only. The filters were applied in SQL over the whole window.", + "No active query snapshots on SQL2025 in the last 1 hour(s) matched database_name 'Sales' with blocking_only with wait_type 'LCK_M_X'.", + "3 sp_server_diagnostics event(s) were captured for SQL2025 in the last 1 hour(s) and none qualified. Events ARE being captured, so this is the healthy answer for this read rather than missing data.", + "TRUNCATED: 40 groups were in scope; only the top 25 (by name) are returned. Scope by server_name, or raise limit, to see the rest.", + ], out _); + + Assert.Equal("The width was chosen to keep this answer near 1500 points; narrow the time range for finer points.", o[0]); + Assert.Equal("TRUNCATED: 300 indexes match and 200 were returned. Rows are ordered by total wait time descending.", o[1]); + Assert.EndsWith("outside the filter.", o[2]); + Assert.EndsWith("filter by database to ask about one.", o[3]); + Assert.Equal(" truncated: 5 of 9 returned.", o[4]); + Assert.EndsWith("still cover every database.", o[5]); + Assert.Contains("per database - so its totals", o[6]); + Assert.Equal("No active query snapshots on SQL2025 in the last 1 hour(s) matched blocking queries only. The filters were applied in SQL over the whole window.", o[7]); + Assert.Contains("matched database 'Sales' with blocking queries only with wait type 'LCK_M_X'.", o[8]); + Assert.Contains("so this is a normal result, not missing data.", o[9]); + Assert.EndsWith("Select a single server to see the rest.", o[10]); + foreach (var line in o) + { + Assert.DoesNotContain("raise limit", line); + Assert.DoesNotContain("bucket_minutes", line); + Assert.DoesNotContain("blocking_only", line); + Assert.DoesNotContain("healthy answer for this read", line); + } + } + + [Fact] + public void AParameterNamedInRealText_IsLeftAlone() + { + string[] same = + [ + "Column limit_value was reset", + "relation database_name_idx does not exist", + "blocking_only_tbl", + ]; + Assert.Equal(same, Run(same, out _)); + } + [Fact] public void UserDataAndRealErrors_ReachThePageAsWritten() { diff --git a/Darling/Darling.Tests/WebWordingFixesTests.cs b/Darling/Darling.Tests/WebWordingFixesTests.cs new file mode 100644 index 0000000000..bdc3c3f843 --- /dev/null +++ b/Darling/Darling.Tests/WebWordingFixesTests.cs @@ -0,0 +1,50 @@ +using System; +using System.Text.RegularExpressions; +using Xunit; +using static Darling.Tests.RepoFile; +namespace Darling.Tests; + +/// +/// #5644 W8, W10 and W15: wording fixes in the shipped page script, pinned against the source (the page itself needs a browser). +/// +public sealed class WebWordingFixesTests +{ + private static string ServerTabs() => + ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js", "pages", "server-tabs.js").ReplaceLineEndings("\n"); + + [Fact] + public void ThePickerPlurals_AreWrittenOut_NotBuiltWithAnS() + { + var js = ServerTabs(); + + /* W15: "latch class" + "s" read "latch classs" in the panel subtitle and the picker's defaults label. */ + Assert.Contains("plural: \"latch classes\"", js, StringComparison.Ordinal); + Assert.Contains("plural: \"spinlocks\"", js, StringComparison.Ordinal); + Assert.DoesNotContain("spec.noun + \"s\"", js, StringComparison.Ordinal); + Assert.Contains("\"Top \" + spec.plural", js, StringComparison.Ordinal); + Assert.Contains("spec.plural + \" you check\"", js, StringComparison.Ordinal); + } + + [Fact] + public void TheLatchAndSpinlockRateColumns_SayTheyAreTheLatestInterval() + { + var js = ServerTabs(); + + /* W10: the panels read "last 24 hours" and so do their totals; the per-second rate is the latest interval's. */ + Assert.Matches(new Regex(@"key: ""wait_ms_per_second"", label: ""ms/s, latest interval"""), js); + Assert.Matches(new Regex(@"key: ""collisions_per_second"", label: ""Coll/s, latest interval"""), js); + } + + [Fact] + public void TheNoticeStrips_AndTheTrendNotes_RunThroughPlainText() + { + var util = ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js", "util.js").ReplaceLineEndings("\n"); + var tabs = ServerTabs(); + + /* W8: a payload's own note (aggregate_note, a truncation note) is written for an MCP caller. */ + Assert.Contains("[plainText(message)]", util, StringComparison.Ordinal); + Assert.Contains("text: plainText(text)", util, StringComparison.Ordinal); + Assert.DoesNotContain("class: \"mp-metric-note\", text: res.data.aggregate_note", tabs, StringComparison.Ordinal); + Assert.DoesNotContain("class: \"mp-metric-note\", text: trend.data.aggregate_note", tabs, StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/ZeroIsNormalCollectorTests.cs b/Darling/Darling.Tests/ZeroIsNormalCollectorTests.cs new file mode 100644 index 0000000000..4ab84ca00d --- /dev/null +++ b/Darling/Darling.Tests/ZeroIsNormalCollectorTests.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; +namespace Darling.Tests; + +/// +/// #5644 W13: Collection Health said a persistent zero "needs a look" for a trace-flag list with none enabled and for a +/// running-job snapshot with no job running, sources whose zero is the normal answer. They now get their own sentence, and +/// the set that earns it is pinned against the catalog. +/// +public sealed class ZeroIsNormalCollectorTests +{ + private static CollectorHealth Zero(string collector, long notes = 0, long errors = 0) => new() + { + CollectorName = collector, + TotalRuns = 400, + SuccessCount = 400 - errors, + ErrorCount = errors, + RowsStored = 0, + NoteCount = notes, + LastSuccessTime = DateTime.UtcNow.AddMinutes(-1), + LastRunTime = DateTime.UtcNow.AddMinutes(-1), + }; + + [Theory] + [InlineData("trace_flags")] + [InlineData("running_jobs")] + public void AZeroFromASourceThatIsEmptyByDefault_IsNotSentToALook(string collector) + { + var finding = Zero(collector).OutputFinding; + + Assert.NotNull(finding); + Assert.Contains("normal answer", finding, StringComparison.Ordinal); + Assert.Contains("needs no action", finding, StringComparison.Ordinal); + Assert.DoesNotContain("needs a look", finding, StringComparison.Ordinal); + /* It is not an event capture, so the event sentence's claim is not made of it. */ + Assert.DoesNotContain("only when an event occurs", finding, StringComparison.Ordinal); + } + + [Fact] + public void ASnapshotCollectorNotOnTheList_StillNeedsALook() + { + /* The positive control for the test above: the same row shape on a collector whose zero is not a normal answer. */ + var finding = Zero("database_scoped_config").OutputFinding; + + Assert.NotNull(finding); + Assert.Contains("needs a look", finding, StringComparison.Ordinal); + Assert.DoesNotContain("normal answer", finding, StringComparison.Ordinal); + } + + [Fact] + public void ANoteOrAFault_KeepsItsOwnSentence_OverTheNormalZero() + { + var noted = Zero("trace_flags", notes: 3).OutputFinding; + Assert.NotNull(noted); + Assert.DoesNotContain("normal answer", noted, StringComparison.Ordinal); + Assert.Contains("recorded a note", noted, StringComparison.Ordinal); + + var faulted = Zero("running_jobs", errors: 5).OutputFinding; + Assert.NotNull(faulted); + Assert.DoesNotContain("normal answer", faulted, StringComparison.Ordinal); + Assert.Contains("UNABLE to read", faulted, StringComparison.Ordinal); + } + + [Fact] + public void TheZeroIsNormalSet_IsReal_AndDisjointFromTheEventSet() + { + var catalog = new HashSet(CollectorCatalog.All.Select(c => c.Name), StringComparer.OrdinalIgnoreCase); + var set = CollectorHealthClassifier.ZeroIsNormalCollectorNamesForPinning; + + Assert.NotEmpty(set); + foreach (var name in set) + { + Assert.Contains(name, catalog); + Assert.True(CollectorHealthClassifier.IsZeroNormalCollector(name), name); + /* The event sentence is checked first and says something else, so an overlap would never reach this one. */ + Assert.False(CollectorHealthClassifier.IsEventCollector(name), name); + } + + foreach (var described in new[] { "trace_flags", "running_jobs" }) + { + Assert.True(CollectorHealthClassifier.IsZeroNormalCollector(described), described); + } + + Assert.False(CollectorHealthClassifier.IsZeroNormalCollector("wait_stats")); + Assert.False(CollectorHealthClassifier.IsZeroNormalCollector(null)); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs index 78c7ae2458..96c2f2cce2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs @@ -5324,7 +5324,8 @@ defaulted zero reaching a surface as "stored nothing" is the #2804 hazard exactl : CollectorHealthClassifier.FormatOutputFinding( RowsStored, TotalRuns, DeniedSinceLastSuccess, NoteCount, faultedRuns: ErrorCount + SessionMissingCount, - isEventCollector: CollectorHealthClassifier.IsEventCollector(CollectorName)) + isEventCollector: CollectorHealthClassifier.IsEventCollector(CollectorName), + isZeroNormal: CollectorHealthClassifier.IsZeroNormalCollector(CollectorName)) is { Length: > 0 } finding ? finding : null; diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/index-analysis.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/index-analysis.js index b1917ef698..6d1c255b1e 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/index-analysis.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/index-analysis.js @@ -20,6 +20,7 @@ import { VIZ } from "../../panels.js"; import { databaseBox, newBoxChoice } from "./database-box.js"; import { el, mount, loadingStrip, noticeStrip, readErrorStrip, errorStrip, readTool, fmtInt, fmtNum } from "../../util.js"; +import { plainText } from "../../plain-text.js"; import { gatedEmptyStrip } from "./gate.js"; // The most recommendations the view lists (#5238). The desktop grid shows every finding, so the tab asks for the ceiling, as the @@ -163,7 +164,7 @@ export const tab = { if (more.length) { parts.push(el("details", {}, [ el("summary", { text: "Analyzer notes (" + more.length + ")" }), - ...more.map((t) => el("p", { class: "finops-note", text: t })), + ...more.map((t) => el("p", { class: "finops-note", text: plainText(t) })), ])); } parts.push(el("h3", { text: "Reclaimable Space (overall + per database)" })); diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js index 1065489b4a..e172987775 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/finops/utilization.js @@ -17,6 +17,7 @@ import { renderPanel, VIZ } from "../../panels.js"; import { READ_FIELDS } from "../../read-fields.js"; import { el, mount, loadingStrip, readErrorStrip, errorStrip, readTool, fmtInt } from "../../util.js"; +import { plainText } from "../../plain-text.js"; import { gatedEmptyStrip, PG_NOT_COLLECTED } from "./gate.js"; const CPU_HOURS = 24; const TOP_HOURS = 24; @@ -147,7 +148,7 @@ function loadUtilization(body, server, ctx) { mount(body, [ el("h3", {}, [SUMMARY_TITLE, el("span", { class: "panel-sub", text: " " + SUMMARY_SUBTITLE })]), VIZ.stat(view, { stats }), - data.health_score_note ? el("p", { class: "finops-note", title: data.health_score_note, text: data.health_score_note }) : null, + data.health_score_note ? el("p", { class: "finops-note", title: plainText(data.health_score_note), text: plainText(data.health_score_note) }) : null, el("p", { class: "finops-reason", text: data.verdict_reason }), el("details", {}, [ el("summary", { text: "7-Day Provisioning Trend" }), diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js index 97cb90a3a9..364c6522af 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js @@ -39,7 +39,7 @@ * touches innerHTML. */ -import { el, makeActivatable, readTool, readToolWithinKeptHistory, keptWindowStrip, windowFloorStrip, mount, truncate, loadingStrip, errorStrip, readErrorStrip, emptyStrip, disclosure, noticeStrip, getPath, fmtMs, fmtRate, localTime, parseUtc, windowFromHours, daysText, dbScopeChip, dbFilteredEmptyText, sourceStrip, getActiveDatabaseFilter, queryWaitFilter, setQueryWaitFilter, waitIsLinked } from "../util.js"; +import { el, makeActivatable, readTool, readToolWithinKeptHistory, keptWindowStrip, windowFloorStrip, mount, truncate, loadingStrip, errorStrip, readErrorStrip, emptyStrip, disclosure, noticeStrip, metricNote, getPath, fmtMs, fmtRate, localTime, parseUtc, windowFromHours, daysText, dbScopeChip, dbFilteredEmptyText, sourceStrip, getActiveDatabaseFilter, queryWaitFilter, setQueryWaitFilter, waitIsLinked } from "../util.js"; import { renderPanel, setPanelSignal, getPanelSignal, VIZ, worstFirst, newestFirst } from "../panels.js"; import { zoomableLineChart, chartZoomScope, SERIES_COLORS, CATEGORICAL_COLORS } from "../charts.js"; import { multiPicker, mergeSeriesRows } from "../multi-picker.js"; @@ -3306,7 +3306,8 @@ const LATCH_COLUMNS = [ { key: "total_delta_wait_time_ms", label: "Wait", format: "ms" }, { key: "total_delta_waiting_requests", label: "Requests", format: "int" }, { key: "avg_wait_ms_per_request", label: "Avg/req", format: "num2" }, - { key: "wait_ms_per_second", label: "ms/s", format: "num2" }, + /* #5644 W10: the panel reads "last 24 hours" and the totals are, but this rate is the latest interval's only. */ + { key: "wait_ms_per_second", label: "ms/s, latest interval", format: "num2" }, { key: "description", label: "What it means", wrap: true }, ]; @@ -3316,7 +3317,7 @@ const SPINLOCK_COLUMNS = [ { key: "total_delta_spins", label: "Spins", format: "int" }, { key: "total_delta_backoffs", label: "Backoffs", format: "int" }, { key: "spins_per_collision", label: "Spins/coll", format: "num1" }, - { key: "collisions_per_second", label: "Coll/s", format: "num2" }, + { key: "collisions_per_second", label: "Coll/s, latest interval", format: "num2" }, { key: "description", label: "What it means", wrap: true }, ]; @@ -4053,7 +4054,7 @@ export function serverTrendPanel(server, ctx, kind) { unit: spec.unit, ...windowFromHours(res.keptHours || ctx.hours), }, "server-trend|" + server + "|" + spec.metric, chartZoomScope(ctx.hours)), - typeof res.data.aggregate_note === "string" && res.data.aggregate_note ? el("div", { class: "mp-metric-note", text: res.data.aggregate_note }) : null, + typeof res.data.aggregate_note === "string" && res.data.aggregate_note ? metricNote(res.data.aggregate_note) : null, ]); })(); return panel; @@ -4157,7 +4158,7 @@ export async function drawClerkTrends(slot, server, ctx, checked) { unit: "MB", ...windowFromHours(trend.keptHours || ctx.hours), }, "clerk-trend|" + server, chartZoomScope(ctx.hours)), - typeof trend.data.aggregate_note === "string" && trend.data.aggregate_note ? el("div", { class: "mp-metric-note", text: trend.data.aggregate_note }) : null, + typeof trend.data.aggregate_note === "string" && trend.data.aggregate_note ? metricNote(trend.data.aggregate_note) : null, ]); } @@ -4176,6 +4177,7 @@ const NAMED_TRENDS = { seriesKey: "latch_class", unit: "ms/s", noun: "latch class", + plural: "latch classes", label: "Latch classes", note: "Wait ms/s, one line per checked latch class.", noOptions: "No latch classes accumulated waits in this window.", @@ -4188,6 +4190,7 @@ const NAMED_TRENDS = { seriesKey: "spinlock_name", unit: "collisions/s", noun: "spinlock", + plural: "spinlocks", label: "Spinlocks", note: "Collisions/s, one line per checked spinlock.", noOptions: "No spinlocks recorded collisions in this window.", @@ -4198,7 +4201,7 @@ const DEFAULT_NAMES_CHECKED = 5; export function namedTrendPanel(server, ctx, kind) { const spec = NAMED_TRENDS[kind]; - const { panel, body } = panelShell(spec.title, ctx.label + ", with a trend for the " + spec.noun + "s you check", 2, spec.optionsTool); + const { panel, body } = panelShell(spec.title, ctx.label + ", with a trend for the " + spec.plural + " you check", 2, spec.optionsTool); (async () => { const res = await readToolWithinKeptHistory(spec.optionsTool, { server, hours: ctx.hours, top: MAX_NAMES_CHARTED }, ctx && ctx.signal); if (res.kind === "error") return mount(body, readErrorStrip(res.message)); @@ -4213,7 +4216,7 @@ export function namedTrendPanel(server, ctx, kind) { options, max: MAX_NAMES_CHARTED, noun: spec.noun, - defaultsLabel: "Top " + spec.noun + "s", + defaultsLabel: "Top " + spec.plural, defaults: (o) => o.slice(0, DEFAULT_NAMES_CHECKED), onChange: (checked) => drawNamedTrends(chartSlot, server, ctx, kind, checked), }); @@ -4280,7 +4283,7 @@ export async function drawNamedTrends(slot, server, ctx, kind, checked) { ...windowFromHours(trend.keptHours || ctx.hours), }, kind + "-trend|" + server, chartZoomScope(ctx.hours)), el("div", { class: "mp-metric-note", text: spec.note }), - typeof trend.data.aggregate_note === "string" && trend.data.aggregate_note ? el("div", { class: "mp-metric-note", text: trend.data.aggregate_note }) : null, + typeof trend.data.aggregate_note === "string" && trend.data.aggregate_note ? metricNote(trend.data.aggregate_note) : null, ]); } @@ -4323,7 +4326,7 @@ export function sessionStatsTrendPanel(server, ctx) { ...windowFromHours(res.keptHours || ctx.hours), }, "server-trend|" + server + "|session_stats", chartZoomScope(ctx.hours)), top.length ? el("div", { class: "mp-metric-note", text: "Top in the newest bucket: " + top.join("; ") + "." }) : null, - typeof res.data.aggregate_note === "string" && res.data.aggregate_note ? el("div", { class: "mp-metric-note", text: res.data.aggregate_note }) : null, + typeof res.data.aggregate_note === "string" && res.data.aggregate_note ? metricNote(res.data.aggregate_note) : null, ]); })(); return panel; diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js index 0cd536ed73..a165249b54 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js @@ -179,6 +179,30 @@ export function plainText(text) { t = t.replace(/\banswered_only: true\b/g, "the answered-only ranking"); t = t.replace(/\banswered_only\b/g, "the answered-only ranking"); + /* #5644 W8: clauses that tell an MCP client which parameter to pass. The page has no such control, so a clause that points + at one is dropped (the sentence before it still stands) and a filter name is said as the page names it. hours_back and + days_back are already put into words by namedTokenText, which runs first. */ + /* A trend's width note: "...near 1500 points; pass bucket_minutes (1-1440) for another width, or narrow hours_back for finer + points." The page picks the width, so only the narrower range is left to say. */ + t = t.replace(/pass bucket_minutes \(1-\d+\) for another width, or /g, ""); + /* "raise limit": the pages have no row-limit control. "...filter - raise limit to see more." and "...returned; raise limit + (now 50) for more." lose the clause; ", or raise limit," and ", and raise limit if none ... shows one" lose theirs. */ + t = t.replace(/\s*[;—–-]+\s*raise (?:the )?limit(?: \(now [\d,]+\))?(?: (?:to see|for) (?:more|the rest(?: of the rows)?))?(?=\.)/g, ""); + t = t.replace(/Scope by server_name(?:, or raise limit,)? (to see the rest|to see them)\./g, "Select a single server $1."); + t = t.replace(/, or raise (?:the )?limit(?=[.,])/g, ""); + t = t.replace(/, and raise limit if none of the databases returned shows one/g, ""); + t = t.replace(/\bpass database_name to ask about one\b/g, "filter by database to ask about one"); + t = t.replace(/\bpass database_name to chart one database per file\b/g, "choose a database to chart its files"); + t = t.replace(/\bomit database_name for every database\b/g, "clear the database filter for every database"); + /* The filters a "matched ..." sentence names: "matched database_name 'x' with blocking_only with wait_type 'y'". */ + t = t.replace(/\bdatabase_name '/g, "database '"); + t = t.replace(/\bwait_type '/g, "wait type '"); + t = t.replace(/\bcollector_name '/g, "collector '"); + t = t.replace(/\bmin_duration_ms (?=\d)/g, "minimum duration (ms) "); + t = t.replace(/\bblocking_only\b/g, "blocking queries only"); + /* The system-health "events were captured, none qualified" closing. */ + t = t.replace(/so this is the healthy answer for this read rather than missing data/g, "so this is a normal result, not missing data"); + /* A tool name: use get_collection_health -> use Collection Health. Only a real tool name; any other get_ word is the reader's own text (an object, a server) and stays as written. */ t = t.replace(/(? (TOOL_NAME_SET.has(m) ? toolWords(rest) : m)); diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js index dd6c69f97a..6d712e80e7 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js @@ -135,11 +135,16 @@ export function errorStrip(message) { return el("div", { class: "strip error", role: "alert" }, [message]); } export function emptyStrip(message) { - return el("div", { class: "strip empty" }, [message]); + return el("div", { class: "strip empty" }, [plainText(message)]); } -/** A non-fatal caveat about otherwise-good data (e.g. the #1665 partial-window notice). */ +/** A non-fatal caveat about otherwise-good data (e.g. the #1665 partial-window notice). The sentence may be one the service + wrote for an MCP caller (a width note, a truncation note), so it passes through plainText like every message (#5644 W8). */ export function noticeStrip(message) { - return el("div", { class: "strip notice", role: "status" }, [message]); + return el("div", { class: "strip notice", role: "status" }, [plainText(message)]); +} +/** A note line under a chart, from a payload's own `aggregate_note`: written for an MCP caller, said in the page's words (#5644 W8). */ +export function metricNote(text) { + return el("div", { class: "mp-metric-note", text: plainText(text) }); } /** * The two collectors the blocking reads draw from (#5244): the blocked-process-report XE session when it holds a row in the window diff --git a/Lite.Tests/CollectionOutputBesideCostTests.cs b/Lite.Tests/CollectionOutputBesideCostTests.cs index c038dc6534..69d5f4cae9 100644 --- a/Lite.Tests/CollectionOutputBesideCostTests.cs +++ b/Lite.Tests/CollectionOutputBesideCostTests.cs @@ -200,10 +200,11 @@ below into a vacuous pass over a list that no longer means what this test thinks #3754: the faulted-run count (a RUN-CLASS count, like the note count beside it) and the event-collector bool, computed by the caller from the name on IsEventCollector's own name-list pattern, precisely so that this method still takes no string. */ - Assert.Equal(6, parameters.Length); + /* #5644: the zero-is-normal bool joined the event bool, computed from a name list the same way. */ + Assert.Equal(7, parameters.Length); Assert.Equal( - new[] { "rowsStored", "totalRuns", "deniedSinceLastSuccess", "noteCount", "faultedRuns", "isEventCollector" }, + new[] { "rowsStored", "totalRuns", "deniedSinceLastSuccess", "noteCount", "faultedRuns", "isEventCollector", "isZeroNormal" }, parameters.Select(p => p.Name!).ToArray()); Assert.DoesNotContain(parameters, p => p.ParameterType == typeof(string)); diff --git a/Lite.Tests/ZeroIsNormalCollectorTests.cs b/Lite.Tests/ZeroIsNormalCollectorTests.cs new file mode 100644 index 0000000000..b28bb1a7e5 --- /dev/null +++ b/Lite.Tests/ZeroIsNormalCollectorTests.cs @@ -0,0 +1,43 @@ +using System; +using PerformanceMonitorLite.Services; +using Xunit; +namespace Lite.Tests; + +/// +/// #5644 W13, the Lite twin of the Darling suite's test of the same name: a trace-flag list with none enabled and a +/// running-job snapshot with no job running are sources whose zero is the normal answer, so the collection-health row does not +/// say a persistent zero "needs a look". Lite's row reads the same shared classifier. +/// +public sealed class ZeroIsNormalCollectorTests +{ + private static CollectorHealthRow Zero(string collector) => new() + { + CollectorName = collector, + TotalRuns = 400, + SuccessCount = 400, + RowsStored = 0, + LastSuccessTime = DateTime.UtcNow.AddMinutes(-1), + LastRunTime = DateTime.UtcNow.AddMinutes(-1), + }; + + [Theory] + [InlineData("trace_flags")] + [InlineData("running_jobs")] + public void AZeroFromASourceThatIsEmptyByDefault_IsNotSentToALook(string collector) + { + var finding = Zero(collector).OutputFinding; + + Assert.NotNull(finding); + Assert.Contains("normal answer", finding, StringComparison.Ordinal); + Assert.DoesNotContain("needs a look", finding, StringComparison.Ordinal); + } + + [Fact] + public void ASnapshotCollectorNotOnTheList_StillNeedsALook() + { + var finding = Zero("database_scoped_config").OutputFinding; + + Assert.NotNull(finding); + Assert.Contains("needs a look", finding, StringComparison.Ordinal); + } +} diff --git a/Lite/Services/LocalDataService.CollectionHealth.cs b/Lite/Services/LocalDataService.CollectionHealth.cs index 39cf032083..810bd0ea54 100644 --- a/Lite/Services/LocalDataService.CollectionHealth.cs +++ b/Lite/Services/LocalDataService.CollectionHealth.cs @@ -1402,7 +1402,8 @@ hazard exactly. */ CollectorHealthClassifier.FormatOutputFinding( RowsStored, TotalRuns, DeniedSinceLastSuccess, NoteCount, faultedRuns: ErrorCount + SessionMissingCount, - isEventCollector: CollectorHealthClassifier.IsEventCollector(CollectorName)) + isEventCollector: CollectorHealthClassifier.IsEventCollector(CollectorName), + isZeroNormal: CollectorHealthClassifier.IsZeroNormalCollector(CollectorName)) is { Length: > 0 } finding ? finding : null; diff --git a/PerformanceMonitor.Common/ServerHealthBands.cs b/PerformanceMonitor.Common/ServerHealthBands.cs index ac0ee18b74..840881d50d 100644 --- a/PerformanceMonitor.Common/ServerHealthBands.cs +++ b/PerformanceMonitor.Common/ServerHealthBands.cs @@ -1719,6 +1719,35 @@ public static bool ExpectsUserDatabases(string? collectorName) => public static bool IsEventCollector(string? collectorName) => collectorName is not null && EventCollectorNames.Contains(collectorName); + /// + /// The collectors whose source holds nothing UNLESS something is enabled, running or configured (#5644 W13): a + /// trace-flag list with no flag turned on, a SQL Agent running-job snapshot with no job in flight, a plan-correction + /// list with no tuning recommendation, a replication-slot or standby list on a primary with none. A zero is the + /// normal answer for them, so must not call a persistent zero "a look" (the + /// generic snapshot sentence the web audit flagged). Not event captures: nothing "occurred" - there is simply + /// nothing set up to report - which is why they are not in and why the + /// event sentence's "stores a row only when an event occurs" is not said of them. Same polarity rule as the event + /// list: a name is added only for a source that is genuinely empty by default, so an omission fails loud (the + /// "needs a look" sentence) rather than reassuring over a broken zero. Pinned against the catalog's real names by + /// both suites. The AG state collectors are handled separately (a not-applicable row where Always On is off). + /// + private static readonly HashSet ZeroIsNormalCollectorNames = new(StringComparer.OrdinalIgnoreCase) + { + "trace_flags", + "running_jobs", + "plan_correction", + "pg_replication_slots", + "pg_replication_stats", + }; + + /// True when a zero-row window is the normal answer for this collector's source (, #5644). + /// False for every other collector and for a null name. + public static bool IsZeroNormalCollector(string? collectorName) => + collectorName is not null && ZeroIsNormalCollectorNames.Contains(collectorName); + + /// The names in , so both suites can pin them against the catalog. + public static IReadOnlyCollection ZeroIsNormalCollectorNamesForPinning => ZeroIsNormalCollectorNames; + /// /// The names in , exposed read-only so both suites can pin the set /// against the catalog's real collector names (every entry must be a collector that exists) and @@ -2767,7 +2796,8 @@ public static string FormatOutputFinding( bool deniedSinceLastSuccess, long noteCount, long faultedRuns, - bool isEventCollector) + bool isEventCollector, + bool isZeroNormal = false) { if (rowsStored > 0 || totalRuns <= 0) { @@ -2863,6 +2893,24 @@ FormatCollectionNote already renders and which would then exist twice. */ totalRuns); } + /* #5644 W13: a source that is empty unless something is enabled or running (ZeroIsNormalCollectorNames). Same + preconditions as the event sentence above (no denial, no fault, no note), so the reading rests on the same + facts; what differs is the reason a zero is fine - there is nothing set up to report - and it does not + claim the collector waits for an event. */ + if (isZeroNormal) + { + return string.Format( + CultureInfo.InvariantCulture, + "Stored 0 rows across {0:N0} runs with no current denial (denied_since_last_success is " + + "false), so this collector read and found nothing rather than being unable to read. " + + "Its source is empty unless something is enabled or running - a trace flag turned on, a " + + "job in progress, a tuning recommendation, a replication slot - so zero is a normal answer " + + "and needs no action. No run recorded a note or a fault, which is what that reading rests " + + "on. " + + ZeroOutputCaveat, + totalRuns); + } + /* #3754: NOT an event collector, and nothing on the row explains the zero. A configuration or snapshot read returns rows whenever its source has any, so every run returning nothing is not a resting state - it is the source coming back empty, and the sentence says so. Worded From 58c25c3d2bdff9132aea2f0c4864dbda0d04022a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:36:11 -0400 Subject: [PATCH 07/42] Long text cells clip to one line, capped panels say so, ISO times print local (#5644 W11 W12 W20) --- Darling/Darling.Tests/JobHistoryPageTests.cs | 3 +- .../WebClampCellBehaviourTests.cs | 142 ++++++++++++++++++ .../WebPartialWindowBehaviourTests.cs | 136 +++++++++++++++++ .../Darling.Tests/web-clamp-cell-harness.mjs | 89 +++++++++++ .../web-partial-window-harness.mjs | 97 ++++++++++++ .../wwwroot/css/app.css | 7 +- .../wwwroot/js/pages/job-history.js | 11 +- .../wwwroot/js/pages/server-tabs.js | 23 +-- .../wwwroot/js/pages/views.js | 4 +- .../wwwroot/js/panels.js | 18 ++- .../wwwroot/js/util.js | 136 ++++++++++++++++- 11 files changed, 636 insertions(+), 30 deletions(-) create mode 100644 Darling/Darling.Tests/WebClampCellBehaviourTests.cs create mode 100644 Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs create mode 100644 Darling/Darling.Tests/web-clamp-cell-harness.mjs create mode 100644 Darling/Darling.Tests/web-partial-window-harness.mjs diff --git a/Darling/Darling.Tests/JobHistoryPageTests.cs b/Darling/Darling.Tests/JobHistoryPageTests.cs index 9ad0072e86..b2e6387402 100644 --- a/Darling/Darling.Tests/JobHistoryPageTests.cs +++ b/Darling/Darling.Tests/JobHistoryPageTests.cs @@ -355,7 +355,8 @@ public void ARowClickOpensTheWholeMessage_AndTheCellClampsIt() Assert.Contains("el(\"div\", { class: \"jh-message\"", tabs, StringComparison.Ordinal); var css = Wwwroot("css", "app.css"); - Assert.Matches("\\.jh-message \\{[^}]*min-width: 2[0-9]rem;[^}]*-webkit-line-clamp: 3;", css); + Assert.Matches(@"\.jh-message \{[^}]*min-width: 2[0-9]rem;", css); + Assert.Matches(@"\.clamp-text \{[^}]*text-overflow: ellipsis;[^}]*white-space: nowrap;", css); // one line (#5644 W11), was 3 lines } [Fact] diff --git a/Darling/Darling.Tests/WebClampCellBehaviourTests.cs b/Darling/Darling.Tests/WebClampCellBehaviourTests.cs new file mode 100644 index 0000000000..494ae1e03f --- /dev/null +++ b/Darling/Darling.Tests/WebClampCellBehaviourTests.cs @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Linq; +using System.Text.Json; +using System.Text.RegularExpressions; +using Xunit; +using static Darling.Tests.RepoFile; + +namespace Darling.Tests; + +/// +/// A long text cell (a collector's note, error or output; a job's message) shows on one line and opens the rest on a click +/// (#5644 W11): the rows were hundreds of pixels tall. These run the shipped panels.js and util.js under Node +/// (web-clamp-cell-harness.mjs), plus pins on the column lists and the stylesheet. Skipped when Node is not installed, +/// except the source pins. +/// +public sealed class WebClampCellBehaviourTests +{ + private static JsonElement Run() + { + var psi = new ProcessStartInfo("node") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false }; + psi.ArgumentList.Add(PathTo("Darling", "Darling.Tests", "web-clamp-cell-harness.mjs")); + psi.ArgumentList.Add(PathTo("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js")); + + Process proc; + try + { + proc = Process.Start(psi)!; + } + catch (Win32Exception) + { + Assert.Skip("Node is not installed, so the shipped table script cannot be run."); + return default; + } + + using (proc) + { + var error = proc.StandardError.ReadToEndAsync(); + var output = proc.StandardOutput.ReadToEnd().Trim(); + if (!proc.WaitForExit(20000)) + { + proc.Kill(entireProcessTree: true); + Assert.Fail("the clamp cell harness did not finish in 20 s"); + } + + Assert.True(proc.ExitCode == 0, "the clamp cell harness failed: " + error.Result); + using var doc = JsonDocument.Parse(output.Split('\n')[0]); + return doc.RootElement.Clone(); + } + } + + private static string Wwwroot(params string[] parts) => + System.IO.File.ReadAllText(PathTo(new[] { "Darling", "PerformanceMonitor.Darling.Service", "wwwroot" }.Concat(parts).ToArray())); + + [Fact] + public void AClampColumn_KeepsTheFullText_AndHasTheClampClass() + { + var r = Run(); + Assert.Equal("clamp", r.GetProperty("noteTdClass").GetString()); + Assert.Equal("wrap", r.GetProperty("plainTdClass").GetString()); + Assert.True(r.GetProperty("longHasFullText").GetBoolean()); + Assert.True(r.GetProperty("longTitleFull").GetBoolean()); + } + + [Fact] + public void ALongCell_OpensAndClosesOnAClick_WithoutActivatingTheRow() + { + var r = Run(); + Assert.Equal("button", r.GetProperty("longRole").GetString()); + Assert.Equal("false", r.GetProperty("longExpandedBefore").GetString()); + var first = r.GetProperty("afterClick"); + Assert.Equal("true", first.GetProperty("expanded").GetString()); + Assert.True(first.GetProperty("open").GetBoolean()); + Assert.True(first.GetProperty("stopped").GetBoolean()); + Assert.Equal(0, first.GetProperty("rowClicks").GetInt32()); + var second = r.GetProperty("afterSecondClick"); + Assert.Equal("false", second.GetProperty("expanded").GetString()); + Assert.False(second.GetProperty("open").GetBoolean()); + } + + [Fact] + public void EnterOpensTheCell_AndOtherKeysDoNot() + { + var r = Run(); + var enter = r.GetProperty("afterEnter"); + Assert.Equal("true", enter.GetProperty("expanded").GetString()); + Assert.True(enter.GetProperty("prevented").GetBoolean()); + Assert.True(enter.GetProperty("stopped").GetBoolean()); + var other = r.GetProperty("otherKey"); + Assert.Equal("true", other.GetProperty("expanded").GetString()); + Assert.False(other.GetProperty("stopped").GetBoolean()); + } + + [Fact] + public void AShortCell_AnEmptyCell_AndANonExpandableCell_AreInert() + { + var r = Run(); + var s = r.GetProperty("short"); + Assert.Equal(JsonValueKind.Null, s.GetProperty("role").ValueKind); + Assert.Equal("short", s.GetProperty("text").GetString()); + var n = r.GetProperty("notExpandable"); + Assert.Equal(JsonValueKind.Null, n.GetProperty("role").ValueKind); + Assert.True(n.GetProperty("text").GetBoolean()); + Assert.True(n.GetProperty("title").GetBoolean()); + Assert.Equal("", r.GetProperty("empty").GetProperty("text").GetString()); + Assert.Equal(JsonValueKind.Null, r.GetProperty("empty").GetProperty("title").ValueKind); + Assert.Equal("button", r.GetProperty("multiline").GetString()); + } + + [Fact] + public void TheCollectorsLongTextColumns_AndBothJobMessageColumns_AreClamped() + { + var tabs = Wwwroot("js", "pages", "server-tabs.js").ReplaceLineEndings("\n"); + var columns = tabs[tabs.IndexOf("const COLLECTOR_COLUMNS = [", StringComparison.Ordinal)..]; + columns = columns[..columns.IndexOf("\n];", StringComparison.Ordinal)]; + foreach (var key in new[] { "run_time_note", "last_error", "note_summary", "recovered_note", "output_finding", "regression_finding", "message" }) + Assert.Matches("key: \"" + key + "\"[^\\n]*clamp: true", columns); + Assert.DoesNotMatch("key: \"(last_error|note_summary|output_finding)\"[^\\n]*wrap: true", columns); + + Assert.Matches("label: \"Message\", clamp: true, render: jobMessageCell", tabs); + var page = Wwwroot("js", "pages", "job-history.js"); + Assert.Matches("label: \"Message\", clamp: true, render: messageCell", page); + } + + [Fact] + public void TheStylesheet_ClipsAClampCellToOneLine_AndLetsAnOpenOneWrap() + { + var css = Wwwroot("css", "app.css"); + Assert.Matches(@"\.clamp-text \{[^}]*overflow: hidden;[^}]*text-overflow: ellipsis;[^}]*white-space: nowrap;", css); + Assert.Matches(@"\.clamp-text\.open \{[^}]*white-space: pre-wrap;", css); + Assert.DoesNotMatch(@"\.jh-message \{[^}]*line-clamp", css); + } +} diff --git a/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs b/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs new file mode 100644 index 0000000000..387c6d9291 --- /dev/null +++ b/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Text.Json; +using Xunit; +using static Darling.Tests.RepoFile; + +namespace Darling.Tests; + +/// +/// Every panel built from a row-capped read says so (#5644 W12): the System Events "system_health CPU" chart plotted only the +/// newest 50 events under a "last 24 hours" title with no notice, and the Active Queries grid showed about an hour under the same +/// title. These run the shipped util.js under Node (web-partial-window-harness.mjs). Skipped when Node is not installed. +/// +public sealed class WebPartialWindowBehaviourTests +{ + private static JsonElement Run() + { + var psi = new ProcessStartInfo("node") { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false }; + psi.ArgumentList.Add(PathTo("Darling", "Darling.Tests", "web-partial-window-harness.mjs")); + psi.ArgumentList.Add(PathTo("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js")); + + Process proc; + try + { + proc = Process.Start(psi)!; + } + catch (Win32Exception) + { + Assert.Skip("Node is not installed, so the shipped page script cannot be run."); + return default; + } + + using (proc) + { + var error = proc.StandardError.ReadToEndAsync(); + var output = proc.StandardOutput.ReadToEnd().Trim(); + if (!proc.WaitForExit(20000)) + { + proc.Kill(entireProcessTree: true); + Assert.Fail("the partial window harness did not finish in 20 s"); + } + + Assert.True(proc.ExitCode == 0, "the partial window harness failed: " + error.Result); + using var doc = JsonDocument.Parse(output.Split('\n')[0]); + return doc.RootElement.Clone(); + } + } + + private static string? S(JsonElement r, string name) => r.GetProperty(name).ValueKind == JsonValueKind.Null ? null : r.GetProperty(name).GetString(); + + [Fact] + public void ARowCappedChart_DrawsTheSameNoticeAsItsGrid_WorkedForAChart() + { + var r = Run(); + Assert.StartsWith("partial window: this grid shows only the newest rows", S(r, "gridServerNote")); + var chart = S(r, "chartServerNote")!; + Assert.StartsWith("partial window: this chart shows only the newest rows", chart); + Assert.Contains("The chart covers", chart); + Assert.DoesNotContain("grid", chart); + } + + [Fact] + public void AChart_DoesNotDrawADataStartNote_BecauseItsAxisShowsTheEmptySpan() + { + var r = Run(); + Assert.Null(S(r, "chartDataStart")); + Assert.Null(S(r, "chartWindowNoteOff")); + } + + [Fact] + public void ACutNewestFirstPage_GetsANoticeWithItsCountsAndOldestRow() + { + var r = Run(); + var oldest = S(r, "oldestLocal")!; + Assert.Equal("partial window: this grid shows only the newest 50 of 219 snapshots, back to " + oldest + ", because it stops at its row limit.", S(r, "activeGrid")); + Assert.StartsWith("partial window: this chart shows only the newest 50 of 219 snapshots", S(r, "activeChart")); + Assert.Equal("partial window: this grid shows only the newest rows, back to " + oldest + ", because it stops at its row limit.", S(r, "activeNoCounts")); + } + + [Fact] + public void ACutPage_ThatNamesNoTime_OrIsNotCut_OrOwnsItsNote_GetsNothing() + { + var r = Run(); + Assert.Null(S(r, "activeStat")); + Assert.Null(S(r, "notCut")); + Assert.Null(S(r, "rankedByOther")); + Assert.Null(S(r, "noOldest")); + Assert.Null(S(r, "ownNote")); + } + + [Fact] + public void TheServersOwnNote_IsNotDrawnTwice() + { + var r = Run(); + Assert.Contains("The window started at S.", S(r, "serverNoteWins")); + } + + [Fact] + public void AnIsoInstantInASentence_PrintsInTheGridsLocalTimeFormat() + { + // #5644 W20: server messages and status lines carried raw UTC stamps beside local-time grids. + var r = Run(); + var a = S(r, "isoLocal")!; + var b = S(r, "isoZLocal")!; + Assert.Equal("stored at " + a + " and not collected since " + b + ".", S(r, "sentence")); + Assert.Equal("last event " + a, S(r, "emptyStripText")); + Assert.Equal("since " + b, S(r, "noticeStripText")); + Assert.Equal("nothing to change 10/9/2026, 10:36:32 AM", S(r, "noStamp")); + Assert.Equal("null", S(r, "nonString")); + Assert.DoesNotContain("T", a); + } + + [Fact] + public void ACappedPanelsHeading_DropsItsWindowLabel_AndAnyOtherKeepsIt() + { + var r = Run(); + var oldest = S(r, "oldestLocal")!; + Assert.Equal("newest rows only, back to " + oldest, S(r, "subtitleActive")); + Assert.Equal("newest rows only, back to " + oldest, S(r, "subtitleServer")); + Assert.Null(S(r, "subtitleDataStart")); + Assert.Null(S(r, "subtitleCovered")); + Assert.Equal("Active Queries newest rows only, back to " + oldest, S(r, "headingCapped")); + Assert.Equal(" not the heading", S(r, "bodyUntouched")); + Assert.Equal("Active Queries last 24 hours", S(r, "headingCovered")); + Assert.Equal("No sub", S(r, "headingWithoutSub")); + } +} diff --git a/Darling/Darling.Tests/web-clamp-cell-harness.mjs b/Darling/Darling.Tests/web-clamp-cell-harness.mjs new file mode 100644 index 0000000000..b0edee3eba --- /dev/null +++ b/Darling/Darling.Tests/web-clamp-cell-harness.mjs @@ -0,0 +1,89 @@ +/* #5644 W11: runs the web dashboard's real shared table renderer (wwwroot/js/panels.js, VIZ.table) and the clampText helper + (util.js) on a small fake DOM, and prints what a long-text column drew as one line of JSON. WebClampCellBehaviourTests starts it as + node web-clamp-cell-harness.mjs + Only the DOM is a stand-in; panels.js and util.js are the shipped modules. */ +import { pathToFileURL } from "node:url"; + +class FakeNode { + constructor(tag) { + this.tag = tag; + this.children = []; + this.attrs = {}; + this.listeners = {}; + this.parent = null; + this._text = ""; + this.className = ""; + this.dataset = {}; + this.style = {}; + const set = new Set(); + this.classes = set; + this.classList = { add: (c) => set.add(c), remove: (c) => set.delete(c), toggle: (c, on) => (on ? set.add(c) : set.delete(c)), contains: (c) => set.has(c) }; + } + get firstChild() { return this.children[0] || null; } + get textContent() { return this._text + this.children.map((c) => c.textContent).join(""); } + set textContent(v) { this._text = String(v); this.children = []; } + setAttribute(k, v) { this.attrs[k] = String(v); if (k === "class") this.className = String(v); } + getAttribute(k) { return this.attrs[k]; } + appendChild(c) { if (c.parent) c.parent.children = c.parent.children.filter((x) => x !== c); c.parent = this; this.children.push(c); return c; } + removeChild(c) { this.children = this.children.filter((x) => x !== c); c.parent = null; } + addEventListener(t, fn) { (this.listeners[t] ||= []).push(fn); } + contains(n) { for (let x = n; x; x = x.parent) if (x === this) return true; return false; } + fire(t, e = {}) { + const ev = { type: t, stopped: false, defaultPrevented: false, stopPropagation() { this.stopped = true; }, preventDefault() { this.defaultPrevented = true; }, ...e }; + for (let n = this; n && !ev.stopped; n = n.parent) for (const fn of n.listeners[t] || []) fn(ev); + return ev; + } + findAll(pred, out = []) { if (pred(this)) out.push(this); this.children.forEach((c) => c.findAll(pred, out)); return out; } +} +class FakeText extends FakeNode { constructor(t) { super("#text"); this._text = t; } } +globalThis.Node = FakeNode; +globalThis.document = { createElement: (t) => new FakeNode(t), createTextNode: (t) => new FakeText(t) }; +globalThis.location = { hash: "#/server/a/health" }; + +const root = process.argv[2]; +const { VIZ } = await import(pathToFileURL(root + "/panels.js").href); +const { clampText } = await import(pathToFileURL(root + "/util.js").href); + +const longNote = "The collector read 0 items. " + "Nothing matched the filter on this server. ".repeat(20); +const cols = [ + { key: "collector", label: "Collector" }, + { key: "note", label: "Note", clamp: true, plain: true }, + { key: "plain", label: "Plain", wrap: true }, +]; +const rows = [ + { collector: "a", note: longNote, plain: longNote }, + { collector: "b", note: "short", plain: "short" }, +]; +const out = {}; + +const t = VIZ.table({ rows }, { rowsKey: "rows", columns: cols, title: "Collectors", tools: false }); +const tr = t.findAll((n) => n.tag === "tr").filter((n) => n.parent && n.parent.tag === "tbody"); +let rowClicks = 0; +tr[0].addEventListener("click", () => rowClicks++); +const tds = (r) => r.children; +const noteTd = tds(tr[0])[1]; +const clampOf = (td) => td.findAll((n) => String(n.className).includes("clamp-text"))[0]; +const longClamp = clampOf(noteTd); +out.noteTdClass = noteTd.className; +out.plainTdClass = tds(tr[0])[2].className; +out.longHasFullText = longClamp.textContent === longNote; +out.longTitleFull = longClamp.attrs.title === longNote; +out.longRole = longClamp.attrs.role; +out.longExpandedBefore = longClamp.attrs["aria-expanded"]; +const ev = longClamp.fire("click"); +out.afterClick = { expanded: longClamp.attrs["aria-expanded"], open: longClamp.classes.has("open"), stopped: ev.stopped, rowClicks }; +longClamp.fire("click"); +out.afterSecondClick = { expanded: longClamp.attrs["aria-expanded"], open: longClamp.classes.has("open") }; +const key = longClamp.fire("keydown", { key: "Enter" }); +out.afterEnter = { expanded: longClamp.attrs["aria-expanded"], prevented: key.defaultPrevented, stopped: key.stopped }; +const other = longClamp.fire("keydown", { key: "a" }); +out.otherKey = { expanded: longClamp.attrs["aria-expanded"], stopped: other.stopped }; + +const shortClamp = clampOf(tds(tr[1])[1]); +out.short = { role: shortClamp.attrs.role || null, tabindex: shortClamp.attrs.tabindex || null, text: shortClamp.textContent }; +const fixed = clampText(longNote, { expandable: false }); +out.notExpandable = { role: fixed.attrs.role || null, text: fixed.textContent === longNote, title: fixed.attrs.title === longNote }; +out.empty = { text: clampText(null).textContent, title: clampText(null).attrs.title || null }; +out.multiline = clampText("one\ntwo").attrs.role; + +console.log(JSON.stringify(out)); diff --git a/Darling/Darling.Tests/web-partial-window-harness.mjs b/Darling/Darling.Tests/web-partial-window-harness.mjs new file mode 100644 index 0000000000..fa91114298 --- /dev/null +++ b/Darling/Darling.Tests/web-partial-window-harness.mjs @@ -0,0 +1,97 @@ +/* #5644 W12: runs the web dashboard's real windowFloorStrip, cappedReadNote, partialWindowSubtitle and markPartialSubtitle (wwwroot/js/util.js) + on a small fake DOM and prints what each drew as one line of JSON. WebPartialWindowBehaviourTests starts it as + node web-partial-window-harness.mjs + Only the DOM is a stand-in; util.js is the shipped module. */ +import { pathToFileURL } from "node:url"; + +class FakeNode { + constructor(tag) { + this.tag = tag; + this.children = []; + this.attrs = {}; + this.listeners = {}; + this.parent = null; + this._text = ""; + this.className = ""; + this.dataset = {}; + this.style = {}; + } + get firstChild() { return this.children[0] || null; } + get textContent() { return this._text + this.children.map((c) => c.textContent).join(""); } + set textContent(v) { this._text = String(v); this.children = []; } + setAttribute(k, v) { this.attrs[k] = String(v); } + getAttribute(k) { return this.attrs[k]; } + appendChild(c) { if (c.parent) c.parent.children = c.parent.children.filter((x) => x !== c); c.parent = this; this.children.push(c); return c; } + removeChild(c) { this.children = this.children.filter((x) => x !== c); c.parent = null; } + addEventListener(t, fn) { (this.listeners[t] ||= []).push(fn); } +} +class FakeText extends FakeNode { constructor(t) { super("#text"); this._text = t; } } +globalThis.Node = FakeNode; +globalThis.document = { createElement: (t) => new FakeNode(t), createTextNode: (t) => new FakeText(t) }; +globalThis.location = { hash: "#/server/a/events" }; + +const root = process.argv[2]; +const { windowFloorStrip, cappedReadNote, partialWindowSubtitle, markPartialSubtitle, localizeIsoText, emptyStrip, noticeStrip, el, localTime } = await import(pathToFileURL(root + "/util.js").href); + +const text = (n) => (n ? n.textContent : null); +const OLDEST = "2026-10-12T13:29:59.5027090Z"; +const oldestLocal = localTime(OLDEST); +const out = { oldestLocal }; + +/* the server's own capped note (WebDataStartNote), as readTool leaves it: a sentence built for a grid */ +const serverCapped = { + window_truncated: true, + oldest_shown_utc: OLDEST, + truncation_note: "partial window: this grid shows only the newest rows, back to X, because it stops at its row limit. The window started at S. The grid covers X to E.", +}; +out.gridServerNote = text(windowFloorStrip(serverCapped, { viz: "table" })); +out.chartServerNote = text(windowFloorStrip(serverCapped, { viz: "line" })); +/* a chart does not draw a plain data-start note: its axis shows the empty span */ +out.chartDataStart = text(windowFloorStrip({ window_truncated: true, data_start_utc: OLDEST, truncation_note: "partial window: this panel's data starts at X." }, { viz: "line" })); +out.chartWindowNoteOff = text(windowFloorStrip(serverCapped, { viz: "line", windowNote: false })); + +/* the active-query page: cut at its limit, newest first, no window_truncated */ +const active = { truncated: true, total_snapshots: 219, snapshots_returned: 50, order: "collection_time_desc", oldest_returned_collection_time: OLDEST }; +out.activeGrid = text(windowFloorStrip(active, { viz: "table" })); +out.activeChart = text(windowFloorStrip(active, { viz: "line" })); +out.activeStat = text(windowFloorStrip(active, { viz: "stat" })); +out.activeNoCounts = cappedReadNote({ truncated: true, order: "collection_time_desc", oldest_returned_collection_time: OLDEST }, { viz: "table" }).text; +out.notCut = text(windowFloorStrip({ ...active, truncated: false }, { viz: "table" })); +out.rankedByOther = text(windowFloorStrip({ ...active, order: "total_elapsed" }, { viz: "table" })); +out.noOldest = text(windowFloorStrip({ truncated: true, order: "collection_time_desc" }, { viz: "table" })); +out.serverNoteWins = text(windowFloorStrip({ ...active, ...serverCapped }, { viz: "table" })); +out.ownNote = text(windowFloorStrip(active, { viz: "table", noteKey: "truncation_note" })); + +/* the heading: a capped panel drops its window label, any other keeps it */ +out.subtitleActive = partialWindowSubtitle(active, { viz: "table" }); +out.subtitleServer = partialWindowSubtitle(serverCapped, { viz: "line" }); +out.subtitleDataStart = partialWindowSubtitle({ window_truncated: true, data_start_utc: OLDEST, truncation_note: "x" }, { viz: "line" }); +out.subtitleCovered = partialWindowSubtitle({ truncated: false, rows: [] }, { viz: "table" }); +const panel = () => el("div", { class: "panel card" }, [ + el("h3", {}, ["Active Queries", el("span", { class: "panel-sub", text: " last 24 hours" })]), + el("div", { class: "panel-body" }, [el("span", { class: "panel-sub", text: " not the heading" })]), +]); +const capped = panel(); +markPartialSubtitle(capped, active, { viz: "table" }); +out.headingCapped = capped.children[0].textContent; +out.bodyUntouched = capped.children[1].textContent; +const covered = panel(); +markPartialSubtitle(covered, { truncated: false }, { viz: "table" }); +out.headingCovered = covered.children[0].textContent; +markPartialSubtitle(null, active, { viz: "table" }); +const noSub = el("div", { class: "panel card" }, [el("h3", {}, ["No sub"])]); +markPartialSubtitle(noSub, active, { viz: "table" }); +out.headingWithoutSub = noSub.children[0].textContent; + +/* W20: ISO instants inside a sentence print in the grids local-time format */ +const iso = "2026-10-09T14:37:35.0158630"; +const isoZ = "2026-10-08T18:11:14Z"; +out.isoLocal = localTime(iso); +out.isoZLocal = localTime(isoZ); +out.sentence = localizeIsoText("stored at " + iso + " and not collected since " + isoZ + "."); +out.noStamp = localizeIsoText("nothing to change 10/9/2026, 10:36:32 AM"); +out.nonString = String(localizeIsoText(null)); +out.emptyStripText = emptyStrip("last event " + iso).textContent; +out.noticeStripText = noticeStrip("since " + isoZ).textContent; + +console.log(JSON.stringify(out)); diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/css/app.css b/Darling/PerformanceMonitor.Darling.Service/wwwroot/css/app.css index 35886ef7f8..fce876122f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/css/app.css +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/css/app.css @@ -476,6 +476,11 @@ table.data th { table.data tbody tr:nth-child(even) { background: var(--row-alt); } table.data td.num { text-align: right; font-variant-numeric: tabular-nums; } table.data td.wrap { white-space: normal; max-width: 620px; } +/* A long text cell on one line, the rest on demand (#5644 W11). The full text stays in the node; the clip is only paint. */ +table.data td.clamp { max-width: 28rem; } +.clamp-text { display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.clamp-text[role="button"] { cursor: pointer; } +.clamp-text.open { white-space: pre-wrap; overflow-wrap: anywhere; } .plan-summary { display: flex; flex-wrap: wrap; gap: 0 1rem; margin: 4px 0; } table.data td.pre { white-space: pre-wrap; } table.data td.mono { font-family: var(--mono); font-size: 0.78rem; } @@ -1313,7 +1318,7 @@ pre.code { .locking-detail-counters dd { margin: 0; font-variant-numeric: tabular-nums; } /* Job History (W2): the Message cell keeps a readable width and shows the first 3 lines; the row's detail pane holds the whole message. */ -.jh-message { min-width: 24rem; max-width: 40rem; display: -webkit-box; -webkit-line-clamp: 3; line-clamp: 3; -webkit-box-orient: vertical; overflow: hidden; white-space: pre-wrap; overflow-wrap: anywhere; } +.jh-message { min-width: 24rem; max-width: 40rem; } .jh-detail:empty { display: none; } .jh-detail { background: var(--bg-dark); border: 1px solid var(--border); border-radius: 6px; padding: 10px 14px; margin: 8px 0; } .jh-detail-head { display: flex; align-items: center; justify-content: space-between; gap: 12px; } diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/job-history.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/job-history.js index f5120cf229..3dc5db84ca 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/job-history.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/job-history.js @@ -25,7 +25,7 @@ import { VIZ } from "../panels.js"; import { pageRangePicker, windowOfSpec } from "../page-range.js"; import { relativeSpec } from "../time-range.js"; import { orderServers } from "../server-order.js"; -import { el, mount, clear, loadingStrip, emptyStrip, noticeStrip, errorStrip, readErrorStrip, readTool, readToolWithinKeptHistory, keptWindowStrip, localTime, makeActivatable } from "../util.js"; +import { el, mount, clear, loadingStrip, emptyStrip, noticeStrip, errorStrip, readErrorStrip, readTool, readToolWithinKeptHistory, keptWindowStrip, localTime, makeActivatable, clampText } from "../util.js"; /** The default range, in hours. The range itself is the shared time range picker's (#5562). */ export const DEFAULT_HOURS = 24; @@ -53,15 +53,16 @@ export const JOB_HISTORY_COLUMNS = [ { key: "duration_formatted", label: "Duration", sortValue: (r) => r.duration_seconds }, { key: "retries", label: "Retries", format: "int" }, { key: "last_success", label: "Last Success", format: "time" }, - { key: "message", label: "Message", wrap: true, render: messageCell }, + { key: "message", label: "Message", clamp: true, render: messageCell }, ]; /* The Message column (the morning walk, W2). A failed step's message can run to thousands of characters and sat in a - column the table squeezed to about 90 px, so each row grew 150-250 px tall. The cell now has a readable width and - shows the first few lines only; the whole message opens in the detail pane when the row is clicked. */ + column the table squeezed to about 90 px, so each row grew 150-250 px tall. The cell has a readable width and is + clipped to ONE line (#5644 W11, was three), with the full text in the tooltip; + the whole message opens in the detail pane when the row is clicked. */ function messageCell(r) { const text = r && r.message != null ? String(r.message) : ""; - return el("div", { class: "jh-message", title: text || null, text }); + return el("div", { class: "jh-message" }, [clampText(text, { expandable: false })]); } /* A run has no id in the read, so the open row is found again after a rebuild by what identifies it. */ diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js index 97cb90a3a9..068d696555 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js @@ -39,7 +39,7 @@ * touches innerHTML. */ -import { el, makeActivatable, readTool, readToolWithinKeptHistory, keptWindowStrip, windowFloorStrip, mount, truncate, loadingStrip, errorStrip, readErrorStrip, emptyStrip, disclosure, noticeStrip, getPath, fmtMs, fmtRate, localTime, parseUtc, windowFromHours, daysText, dbScopeChip, dbFilteredEmptyText, sourceStrip, getActiveDatabaseFilter, queryWaitFilter, setQueryWaitFilter, waitIsLinked } from "../util.js"; +import { el, makeActivatable, readTool, readToolWithinKeptHistory, keptWindowStrip, windowFloorStrip, markPartialSubtitle, mount, truncate, loadingStrip, errorStrip, readErrorStrip, emptyStrip, disclosure, noticeStrip, getPath, fmtMs, fmtRate, localTime, parseUtc, windowFromHours, daysText, dbScopeChip, dbFilteredEmptyText, sourceStrip, clampText, getActiveDatabaseFilter, queryWaitFilter, setQueryWaitFilter, waitIsLinked } from "../util.js"; import { renderPanel, setPanelSignal, getPanelSignal, VIZ, worstFirst, newestFirst } from "../panels.js"; import { zoomableLineChart, chartZoomScope, SERIES_COLORS, CATEGORICAL_COLORS } from "../charts.js"; import { multiPicker, mergeSeriesRows } from "../multi-picker.js"; @@ -233,6 +233,7 @@ function fanout(read, params, specs) { opts.atTimeItem; Active Queries when it names none. */ const rendered = VIZ[spec.viz](res.data, { ...spec, read, windowHours: res.keptHours || (params && params.hours), atTime: spec.atTime === false || !params || !params.server ? null : { server: params.server, item: spec.atTimeItem || "queries" } }); + markPartialSubtitle(shells[i].panel, res.data, spec); mount(body, [keptWindowStrip(res), windowFloorStrip(res.data, spec), sourceStrip(res.data, spec), typeof note === "string" && note.trim() ? noticeStrip(note) : null, rendered]); } catch (e) { mount(body, errorStrip("Could not render this panel: " + (e && e.message ? e.message : String(e)))); @@ -4549,13 +4550,13 @@ const JOB_HISTORY_COLUMNS = [ { key: "duration_formatted", label: "Duration", sortValue: (r) => r.duration_seconds }, { key: "retries", label: "Retries", format: "int" }, { key: "last_success", label: "Last Success", format: "time" }, - { key: "message", label: "Message", wrap: true, render: jobMessageCell }, + { key: "message", label: "Message", clamp: true, render: jobMessageCell }, ]; -/* The same squeeze as the Job History page's Message column (W2): a readable width, the first 3 lines, and the whole text on hover. */ +/* The same squeeze as the Job History page's Message column (W2): a readable width, ONE line (#5644 W11), and the rest on a click (this tab has no detail pane). */ function jobMessageCell(r) { const text = r && r.message != null ? String(r.message) : ""; - return el("div", { class: "jh-message", title: text || null, text }); + return el("div", { class: "jh-message" }, [clampText(text)]); } const PERFMON_COLUMNS = [ @@ -4887,8 +4888,8 @@ const COLLECTOR_COLUMNS = [ time. Each column is left out unless a row fills it, so a server with no run times shows none of them. */ { key: "run_at", label: "Run at", hideWhenEmpty: true }, { key: "next_run_utc", label: "Next run", format: "time", hideWhenEmpty: true }, - { key: "run_time_note", label: "Run time", wrap: true, hideWhenEmpty: true }, - { key: "last_error", label: "Last Error", wrap: true, plain: "named" }, + { key: "run_time_note", label: "Run time", clamp: true, hideWhenEmpty: true }, + { key: "last_error", label: "Last Error", clamp: true, plain: "named" }, /* #1837: what a NON-failing run reported (an enumeration that came back with 0 items). Blank for a plainly healthy collector; the same column the two WPF grids carry, so the web view is not the one Collection Health surface that still hides it. note_summary, not the raw last_note: it carries the @@ -4896,24 +4897,24 @@ const COLLECTOR_COLUMNS = [ one, composed server-side from the shared formatter so this table cannot render it a third way. A collector's label=value counts read as its latest NOTED run's ("latest run:" only when every run carried them), never a window total. */ - { key: "note_summary", label: "Note", wrap: true, plain: true }, + { key: "note_summary", label: "Note", clamp: true, plain: true }, /* #5646: a collector that recovered (its newest 3 runs succeeded after failures) is HEALTHY, and this says so with the time of the last failure. Partial rows carry no note_summary, so the sentence rides in its own field. */ - { key: "recovered_note", label: "Recovered", wrap: true, plain: true, hideWhenEmpty: true }, + { key: "recovered_note", label: "Recovered", clamp: true, plain: true, hideWhenEmpty: true }, /* #3017: which of the two zero-output readings a collector that spent and stored nothing is — read and found nothing, or could not read. Blank whenever Rows is positive, for the same reason the Note column is blank on a plainly healthy collector. Composed server-side from the shared formatter, so this table cannot render the sentence a second way. */ - { key: "output_finding", label: "Output", wrap: true, plain: true }, + { key: "output_finding", label: "Output", clamp: true, plain: true }, /* #4620: why a collector that stopped doing what it used to do reads WARNING. The Status column already showed the floor, but no column showed the sentence behind it, so a regressed row read WARNING with every other cell blank. Composed server-side from the shared formatter, like the two columns above. */ - { key: "regression_finding", label: "Regression", wrap: true, plain: true }, + { key: "regression_finding", label: "Regression", clamp: true, plain: true }, /* #5249: a PostgreSQL collector this server's engine does not collect (a collector this engine kind can never run: an Aurora-only reader on stock PostgreSQL, the sampler on Aurora) has no log row, so the server lists it with status not_collected and the sentence saying why. The status paints with the neutral Unknown band, not an error one: a gate is not a fault. Left out unless a row has a message, so a server with no such collector (and every SQL Server tab) shows no column of dashes. */ - { key: "message", label: "Why not collected", wrap: true, hideWhenEmpty: true, plain: true }, + { key: "message", label: "Why not collected", clamp: true, hideWhenEmpty: true, plain: true }, ]; const HEAVIEST_COLUMNS = [ diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/views.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/views.js index 0e42a3f836..4b81c7ff9d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/views.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/views.js @@ -25,7 +25,7 @@ * descriptions) reaches the DOM through el()/textContent (R4 — never innerHTML). */ -import { el, mount, apiGetFleet, loadingStrip, errorStrip, emptyStrip, noticeStrip, relTime, localTime, fmtNum } from "../util.js"; +import { el, mount, apiGetFleet, loadingStrip, errorStrip, emptyStrip, noticeStrip, relTime, localTime, localizeIsoText, fmtNum } from "../util.js"; import { rollingHoursPicker } from "../page-range.js"; import { renderPanel, setPanelSignal, VIZ } from "../panels.js"; import { orderServers } from "../server-order.js"; @@ -803,7 +803,7 @@ function renderAlertCell(cell, index, readSet, sourceSet, scope, opts, limiter) if (cell.type === "status") { return el("div", { class: "notebook-panel panel card" }, [ el("h3", {}, [cell.title || "Status"]), - el("div", { class: "panel-body" }, [el("span", { class: "fv", text: cell.status || "Unknown" })]), + el("div", { class: "panel-body" }, [el("span", { class: "fv", text: localizeIsoText(cell.status || "Unknown") })]), ]); } if (cell.type === "read") { diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js index f8802410ad..a5ae8a0355 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js @@ -44,6 +44,9 @@ import { dbScopeChip, dbFilteredEmptyText, sourceStrip, + clampText, + markPartialSubtitle, + localizeIsoText, } from "./util.js"; import { zoomableLineChart, chartZoomScope, SERIES_COLORS } from "./charts.js"; import { plainText, plainTextNamed } from "./plain-text.js"; @@ -192,6 +195,8 @@ async function loadPanelBody(desc, body, signal) { const moreNotes = (desc.moreNoteKeys || []).map((k) => getPath(res.data, k)); /* #4966: a grid says where its table's data starts when that is after the window's start (windowFloorStrip). */ const floor = windowFloorStrip(res.data, desc); + /* #5644 W12: a row-capped panel stops claiming the window it was asked for in its heading. */ + markPartialSubtitle(body.parentNode, res.data, desc); /* A narrowed read draws its chart over the hours it answered for, not the Range it was asked for (#2802). A copy, so the caller's descriptor keeps the window it asked for. */ if (res.keptHours) desc = { ...desc, windowHours: res.keptHours }; @@ -233,7 +238,7 @@ export const VIZ = { */ const NO_FIELDS_MSG = "No fields configured — edit this view and run Auto-detect fields."; -/* table: desc = { rowsKey, columns:[{key,label,format,align,wrap,mono,pre,sevKey,statusSev,sortable,sortValue,csv,copy,copyValue}], sortable, sortId, onRow(row, tr), rowClass(row)|string, tools:false } +/* table: desc = { rowsKey, columns:[{key,label,format,align,wrap,clamp,mono,pre,sevKey,statusSev,sortable,sortValue,csv,copy,copyValue}], sortable, sortId, onRow(row, tr), rowClass(row)|string, tools:false } Column-header sort (#4843). A header click cycles ascending -> descending -> the server's own order, with a ▲/▼ indicator and aria-sort; Enter and Space on the focused header do the same. The comparison reads the RAW row value @@ -1211,7 +1216,8 @@ function cell(row, c) { query-text expander. It owns its own content; wrap/mono classes still apply if the column asks for them. */ if (typeof c.render === "function") { const rcls = []; - if (c.wrap) rcls.push("wrap"); + if (c.clamp) rcls.push("clamp"); + else if (c.wrap) rcls.push("wrap"); if (c.mono) rcls.push("mono"); if (c.pre) rcls.push("pre"); if (typeof c.cellClass === "function") { const k = c.cellClass(row); if (k) rcls.push(k); } @@ -1220,7 +1226,8 @@ function cell(row, c) { const raw = getPath(row, c.key); const cls = []; if (isNumericCol(c)) cls.push("num"); - if (c.wrap) cls.push("wrap"); + if (c.clamp) cls.push("clamp"); + else if (c.wrap) cls.push("wrap"); if (c.mono) cls.push("mono"); if (c.pre) cls.push("pre"); if (c.sevKey) cls.push(sevClass(getPath(row, c.sevKey))); @@ -1230,7 +1237,7 @@ function cell(row, c) { "n/a (log service)" for the log file of a Hyperscale database). The server wrote the sentence; the page only shows it in place of the bare em dash. */ const why = raw == null && c.nullKey ? getPath(row, c.nullKey) : null; - const text = + const shown = why != null && why !== "" ? String(why) : typeof c.display === "function" @@ -1244,6 +1251,9 @@ function cell(row, c) { : c.plain ? plainText(String(raw)) : String(raw); + /* #5644 W20: a column with no format shows any ISO instant in its text (a bare timestamp or one inside a sentence) in local time, like the grids beside it. */ + const text = c.format ? shown : localizeIsoText(shown); + if (c.clamp) return el("td", { class: cls.join(" ") || null }, [clampText(text)]); return el("td", { class: cls.join(" ") || null, text }); } diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js index dd6c69f97a..da50e01f0a 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js @@ -105,6 +105,46 @@ export function disclosure(summaryText, detail, opts = {}) { return el("details", { class: "disclosure" }, [summary, el("div", { class: "disc-body" }, detail)]); } +/* Text longer than this many characters (or holding a line break) gets the expand toggle; shorter text fits a cell as is. */ +const CLAMP_MIN_CHARS = 60; + +/** + * A long text cell shown on ONE line (#5644 W11): the full text stays in the node (so a selection, the tooltip and a + * screen reader read all of it) and the stylesheet clips it with an ellipsis (`.clamp-text`); a click, or Enter/Space, + * opens the rest in place (`.open`) and closes it again. Long collector notes, errors and job messages made rows + * hundreds of pixels tall. Copy and CSV export read the row data, not this node, so they still give the full text. + * `expandable: false` is for a cell whose row already opens the whole text elsewhere (the Job History detail pane). + * The toggle stops its event so a clickable row (Collectors) is not also activated. + */ +export function clampText(text, opts = {}) { + const full = String(text == null ? "" : text); + const node = el("span", { class: "clamp-text", title: full || null, text: full }); + const long = full.length > CLAMP_MIN_CHARS || /[\r\n]/.test(full); + if (opts.expandable === false || !long) return node; + node.setAttribute("role", "button"); + node.setAttribute("tabindex", "0"); + node.setAttribute("aria-expanded", "false"); + const toggle = (e) => { + /* A click that ends a text selection inside the cell is the reader copying, not opening it. */ + if (e && e.type === "click" && typeof window !== "undefined" && window.getSelection) { + const sel = window.getSelection(); + if (sel && !sel.isCollapsed && String(sel).length > 0 && sel.anchorNode && node.contains(sel.anchorNode)) return; + } + if (e && e.stopPropagation) e.stopPropagation(); + const open = node.getAttribute("aria-expanded") !== "true"; + node.setAttribute("aria-expanded", open ? "true" : "false"); + node.classList.toggle("open", open); + }; + node.addEventListener("click", toggle); + node.addEventListener("keydown", (e) => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + toggle(e); + } + }); + return node; +} + /* * #3031: a stable id for one of a rollup tile's text rows, so the tile's NUMBER can point at it. * @@ -135,11 +175,24 @@ export function errorStrip(message) { return el("div", { class: "strip error", role: "alert" }, [message]); } export function emptyStrip(message) { - return el("div", { class: "strip empty" }, [message]); + return el("div", { class: "strip empty" }, [localizeIsoText(message)]); } /** A non-fatal caveat about otherwise-good data (e.g. the #1665 partial-window notice). */ export function noticeStrip(message) { - return el("div", { class: "strip notice", role: "status" }, [message]); + return el("div", { class: "strip notice", role: "status" }, [localizeIsoText(message)]); +} + +/** + * Rewrites every ISO-8601 instant inside a sentence (a server message, a status line) into the grids' local-time format + * (#5644 W20): "stored at 2026-10-09T14:37:35.0158630" and "since 2026-10-12T19:18:38Z" printed raw UTC beside grids in local + * time. A stamp with no zone is UTC, as parseUtc reads it everywhere. Text with no such stamp, and a non-string, come back as given. + */ +export function localizeIsoText(text) { + if (typeof text !== "string") return text; + return text.replace(/\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?(?:Z|[+-]\d\d:?\d\d)?/g, (m) => { + const d = parseUtc(m); + return d ? d.toLocaleString() : m; + }); } /** * The two collectors the blocking reads draw from (#5244): the blocked-process-report XE session when it holds a row in the window @@ -248,12 +301,83 @@ export function keptWindowStrip(res) { * `floorKey`, and the fields are read from there instead of from the top level. The note is text, never markup (R4). */ export function windowFloorStrip(data, desc) { - if (!desc || desc.windowNote === false || (desc.viz !== "table" && desc.viz !== "stat")) return null; + if (!desc || desc.windowNote === false) return null; + const chart = desc.viz === "line"; + if (desc.viz !== "table" && desc.viz !== "stat" && !chart) return null; if (desc.noteKey === "truncation_note" || (desc.moreNoteKeys || []).includes("truncation_note")) return null; const source = desc.floorKey ? getPath(data, desc.floorKey) : data; - if (!source || source.window_truncated !== true) return null; - const note = source.truncation_note; - return typeof note === "string" && note.trim() ? noticeStrip(note) : null; + if (source && source.window_truncated === true) { + const note = source.truncation_note; + if (typeof note !== "string" || !note.trim()) return null; + if (!chart) return noticeStrip(note); + /* A chart draws only the row-capped note (#5644 W12): its time axis already shows the empty span of a data start that + falls after the window's start, but a chart built from the newest N rows plots nothing before its oldest row while + the axis and the title claim the whole window. That note names the oldest row shown (oldest_shown_utc). */ + if (typeof source.oldest_shown_utc !== "string") return null; + return noticeStrip(note.replace("this grid shows", "this chart shows").replace("The grid covers", "The chart covers")); + } + const capped = cappedReadNote(data, desc); + return capped ? noticeStrip(capped.text) : null; +} + +/* + * The partial-window facts of a read that stopped at its row limit (#5644 W12), or null. Two shapes carry them: the server's + * window-floor fields (`window_truncated` with `oldest_shown_utc`: the reads WebDataStartNote lists, rewritten by readTool), and a + * newest-first page that says it was cut (`truncated: true` with `oldest_returned_collection_time` and an `*_desc` order: the + * active-query page, whose cap of 50 rows covers about an hour of a "last 24 hours" panel). A cut that names no time (a list + * ranked by something other than time) is not a partial window, so it gets nothing. `{ text, oldest }`: the sentence for a + * strip above the panel, and the page's oldest instant. + */ +export function cappedReadNote(data, desc) { + if (!data || typeof data !== "object" || Array.isArray(data)) return null; + if (desc && (desc.windowNote === false || (desc.viz !== "table" && desc.viz !== "line"))) return null; + if (desc && (desc.noteKey === "truncation_note" || (desc.moreNoteKeys || []).includes("truncation_note"))) return null; + if (data.window_truncated === true) return null; + if (data.truncated !== true) return null; + const oldest = data.oldest_returned_collection_time; + if (typeof oldest !== "string" || !parseUtc(oldest)) return null; + if (!/_desc$|newest/.test(String(data.order == null ? "" : data.order))) return null; + const noun = desc && desc.viz === "line" ? "chart" : "grid"; + const shown = Number(data.snapshots_returned); + const total = Number(data.total_snapshots); + const counts = shown > 0 && total > shown ? shown.toLocaleString() + " of " + total.toLocaleString() + " snapshots" : "rows"; + return { + text: "partial window: this " + noun + " shows only the newest " + counts + ", back to " + localTime(oldest) + ", because it stops at its row limit.", + oldest, + }; +} + +/** + * The subtitle a row-capped panel gets in place of its window label (#5644 W12): "last 24 hours" over a page that reaches back + * one hour claimed a window the data does not cover. Null when the panel is not a row-capped one (the label stays). + */ +export function partialWindowSubtitle(data, desc) { + if (desc && desc.windowNote === false) return null; + if (data && typeof data === "object" && data.window_truncated === true) { + if (typeof data.oldest_shown_utc !== "string" || !parseUtc(data.oldest_shown_utc)) return null; + if (desc && desc.viz !== "table" && desc.viz !== "line") return null; + return "newest rows only, back to " + localTime(data.oldest_shown_utc); + } + const capped = cappedReadNote(data, desc); + return capped ? "newest rows only, back to " + localTime(capped.oldest) : null; +} + +/** Replaces the window label in a panel's heading (its `.panel-sub` span) when the read was row-capped. A no-op otherwise. */ +export function markPartialSubtitle(panel, data, desc) { + const text = partialWindowSubtitle(data, desc); + if (!text || !panel) return; + const find = (node) => { + for (const c of Array.from(node.children || [])) { + const k = c.className || (typeof c.getAttribute === "function" ? c.getAttribute("class") : null); + if (k === "panel-sub") return c; + const deeper = find(c); + if (deeper) return deeper; + } + return null; + }; + const head = Array.from(panel.children || [])[0]; + const sub = head ? find(head) : null; + if (sub) sub.textContent = " " + text; } /** From f84f5f19fdbd9d4a1f0d7739b089cf3675fa82c5 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:37:15 -0400 Subject: [PATCH 08/42] Pin the blocking wait count: one event per wait, SQL and C# agree (#5644 W1, W2) --- .../Darling.Tests/BlockingWaitCountTests.cs | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 Darling/Darling.Tests/BlockingWaitCountTests.cs diff --git a/Darling/Darling.Tests/BlockingWaitCountTests.cs b/Darling/Darling.Tests/BlockingWaitCountTests.cs new file mode 100644 index 0000000000..1024eba566 --- /dev/null +++ b/Darling/Darling.Tests/BlockingWaitCountTests.cs @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Common; +using Xunit; + +namespace Darling.Tests; + +/// +/// #5644 W1/W2: one blocking event is one blocked session, blocker and wait start, not one blocked-process report. +/// The reports below are built the way SQL Server writes them: one every 5 seconds while a session stays blocked, +/// each carrying the wait time so far, with a few milliseconds of jitter between the event timestamp and the wait +/// time (the wait start is derived, so it drifts by that much between reports of one wait). +/// +public sealed class BlockingWaitCountTests +{ + private static readonly DateTime T0 = new(2026, 10, 9, 9, 0, 0, DateTimeKind.Unspecified); + + /// The reports of one wait that began at : one every 5 s, wait times 5.0 s, + /// 10.0 s, ... with +/- 35 ms of jitter on the derived start. + internal static List OneWait( + int blocked, int blocker, DateTime start, int reports, string obj = "db.dbo.Users", + string source = BlockedProcessAlertRow.XeReportSource) + { + var rows = new List(); + for (var k = 1; k <= reports; k++) + { + var jitter = ((k * 37) % 71) - 35; + var wait = (5000L * k) + 900; + rows.Add(new BlockedProcessAlertRow + { + EventTime = start.AddMilliseconds(wait + jitter), + DatabaseName = "db", + BlockedSpid = blocked, + BlockingSpid = blocker, + WaitTimeMs = wait, + LockMode = "X", + BlockedSqlText = "UPDATE Users SET x = 1", + BlockingSqlText = "BEGIN TRAN UPDATE Users", + ContentiousObject = obj, + Source = source, + }); + } + + return rows; + } + + private static List Collapse(IEnumerable rows) => + BlockingWaits.Collapse(rows, r => r.Source, r => r.BlockedSpid, r => r.BlockingSpid, r => r.EventTime, r => r.WaitTimeMs); + + [Fact] + public void OneMinuteWait_ElevenReports_IsOneEvent_WithTheLongestWait() + { + var rows = OneWait(102, 100, T0, 11); + + var waits = Collapse(rows); + + var wait = Assert.Single(waits); + Assert.Equal(rows.Max(r => r.WaitTimeMs), wait.WaitTimeMs); + } + + [Fact] + public void IssueShape_FortyThreeReportsOfFourWaits_AreFourEvents() + { + /* The audit's table: blocked 102, blocker 100, 11 + 9 + 9 + 6 reports. */ + var rows = new List(); + rows.AddRange(OneWait(102, 100, T0, 11)); + rows.AddRange(OneWait(102, 100, T0.AddMinutes(10), 9)); + rows.AddRange(OneWait(102, 100, T0.AddMinutes(20), 9)); + rows.AddRange(OneWait(102, 100, T0.AddMinutes(30), 6)); + Assert.Equal(35, rows.Count); + + var waits = Collapse(rows); + + Assert.Equal(4, waits.Count); + Assert.Equal(new[] { 55900L, 45900L, 45900L, 30900L }.OrderBy(x => x), waits.Select(w => w.WaitTimeMs).OrderBy(x => x)); + } + + [Fact] + public void ReportOrder_DoesNotChangeTheCount() + { + var rows = new List(); + rows.AddRange(OneWait(102, 100, T0, 11)); + rows.AddRange(OneWait(102, 100, T0.AddMinutes(5), 4)); + + Assert.Equal(2, Collapse(rows).Count); + Assert.Equal(2, Collapse(Enumerable.Reverse(rows)).Count); + Assert.Equal(2, Collapse(rows.OrderBy(r => r.WaitTimeMs % 7)).Count); + } + + [Fact] + public void BackToBackWaits_OfTheSamePair_AreTwoEvents() + { + /* The second wait starts the instant the first one ends; its first report is 5 s later. */ + var first = OneWait(102, 100, T0, 3); + var secondStart = first[^1].EventTime!.Value; + var second = OneWait(102, 100, secondStart, 3); + + Assert.Equal(2, Collapse(first.Concat(second)).Count); + } + + [Fact] + public void DifferentSessions_DifferentBlockers_AndDifferentSources_NeverMerge() + { + var rows = new List(); + rows.AddRange(OneWait(102, 100, T0, 4)); + rows.AddRange(OneWait(103, 100, T0, 4)); + rows.AddRange(OneWait(102, 101, T0, 4)); + rows.AddRange(OneWait(102, 100, T0, 4, source: BlockedProcessAlertRow.DmvSnapshotSource)); + + Assert.Equal(4, Collapse(rows).Count); + } + + [Fact] + public void ARowWithNoEventTime_StaysAnEventOfItsOwn() + { + var rows = OneWait(102, 100, T0, 3); + rows.Add(new BlockedProcessAlertRow { BlockedSpid = 102, BlockingSpid = 100, WaitTimeMs = 9000, EventTime = null }); + + Assert.Equal(2, Collapse(rows).Count); + } + + [Fact] + public void AlertValue_IsTheSumOfTheChainCounts_NotOneHigher() + { + /* W2's shape: 42 waits of one chain and 1 of another, 5 reports each. The old value counted rows and the + card showed "Blocking chain (x42)" beside it; now both come from BlockingWaitRows. */ + var rows = new List(); + for (var i = 0; i < 42; i++) + { + rows.AddRange(OneWait(200 + i, 100, T0.AddMinutes(i * 3), 5)); + } + + rows.AddRange(OneWait(900, 901, T0.AddHours(3), 5, obj: "db.dbo.Orders")); + + var counted = AlertContextBuilders.BlockingWaitRows(rows, Array.Empty()); + var context = AlertContextBuilders.BuildBlockingContext("srv", rows, Array.Empty()); + + Assert.Equal(43, counted.Count); + Assert.NotNull(context); + var shown = context!.Details + .Where(d => d.Heading.StartsWith("Blocking chain", StringComparison.Ordinal)) + .Select(d => d.Heading == "Blocking chain" ? 1 : int.Parse(d.Heading[(d.Heading.IndexOf("(x", StringComparison.Ordinal) + 2)..^1], CultureInfo.InvariantCulture)) + .ToList(); + Assert.Contains(42, shown); + Assert.Equal(43, shown.Sum()); + } + + [Fact] + public async Task Engine_FiresOnTheWaitCount_AndTheValueMatchesTheCard() + { + var h = new AlertEngineTests.Harness(); + h.Settings.BlockingEnabled = true; + h.Settings.BlockingCountThreshold = 1; + var engine = h.Build(); + + /* One long wait, eleven reports; before #5644 this fired with a value of 11. */ + h.Adapter.Blocking.AddRange(OneWait(102, 100, T0, 11)); + await engine.EvaluateServerAsync(AlertEngineTests.Harness.Snapshot()); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("1", fired.CurrentValue); + Assert.Equal(1, fired.NumericCurrentValue); + var chain = Assert.Single(fired.Context!.Details, d => d.Heading.StartsWith("Blocking chain", StringComparison.Ordinal)); + Assert.Equal("Blocking chain", chain.Heading); + var waitRange = chain.Fields.Single(f => f.Label == "Wait Range").Value; + Assert.Contains("55", waitRange, StringComparison.Ordinal); + } + + /* ---- the SQL form of the same rule (PostgreSQL): the fleet tile, the analysis fact and the summaries run it ---- */ + + private static string ValuesSource(IReadOnlyList rows, int server) => + "SELECT " + server.ToString(CultureInfo.InvariantCulture) + " AS server_id, v.blocked_spid, v.blocking_spid, v.event_time, v.wait_time_ms FROM (VALUES " + + string.Join(",", rows.Select(r => string.Format( + CultureInfo.InvariantCulture, "({0},{1},TIMESTAMP '{2:yyyy-MM-dd HH:mm:ss.ffffff}',{3}::bigint)", + r.BlockedSpid, r.BlockingSpid, r.EventTime, r.WaitTimeMs))) + + ") AS v(blocked_spid, blocking_spid, event_time, wait_time_ms)"; + + [Fact] + public async Task PostgresSql_AgreesWithTheCSharpRule_OnTheSameReports() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrWhiteSpace(connectionString), "Set DARLING_TEST_PG to run the live wait-count SQL test."); + + var rows = new List(); + rows.AddRange(OneWait(102, 100, T0, 11)); + rows.AddRange(OneWait(102, 100, T0.AddMinutes(10), 9)); + rows.AddRange(OneWait(102, 100, T0.AddMinutes(20), 9)); + rows.AddRange(OneWait(102, 100, T0.AddMinutes(30), 6)); + rows.AddRange(OneWait(103, 100, T0, 4)); + rows.AddRange(OneWait(102, 101, T0.AddMinutes(1), 7)); + + var sql = "WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "waits", ValuesSource(rows, 1), new[] { "server_id" }) + + " SELECT COUNT(*), MAX(wait_time_ms) FROM waits"; + + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand(sql, connection); + await using var reader = await command.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + + var expected = Collapse(rows); + Assert.Equal(expected.Count, Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture)); + Assert.Equal(6, expected.Count); + Assert.Equal(expected.Max(w => w.WaitTimeMs), Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); + } +} From 7a7ac3deb54cd627a3f1ea07ea8d3c0c70841fcf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:37:52 -0400 Subject: [PATCH 09/42] Web UI audit lane F: strict integer query params, changed-only restart banner, job history and plan corrections repeats (#5644) --- .../DarlingMcpPlanCorrectionToolsTests.cs | 45 ++++++ .../Darling.Tests/DarlingWebEndpointsTests.cs | 79 ++++++++++ Darling/Darling.Tests/JobHistoryPageTests.cs | 7 +- .../PlanCorrectionCollapseLiveTests.cs | 144 ++++++++++++++++++ .../ServerTabGridColumnPinTests.cs | 2 +- .../DarlingWebEndpoints.cs | 49 +++++- .../Mcp/DarlingMcpPlanCorrectionTools.cs | 36 ++++- .../Mcp/DarlingPlanCorrectionReader.cs | 100 +++++++++++- .../wwwroot/js/pages/job-history.js | 5 +- .../wwwroot/js/pages/server-tabs.js | 7 +- Lite.Tests/BaselineDiscontinuityTests.cs | 24 ++- .../BaselineDiscontinuity.cs | 3 +- PerformanceMonitor.Collectors/ServerEpoch.cs | 34 ++++- 13 files changed, 513 insertions(+), 22 deletions(-) create mode 100644 Darling/Darling.Tests/PlanCorrectionCollapseLiveTests.cs diff --git a/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs b/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs index e446039abe..2e90667af9 100644 --- a/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs @@ -7,6 +7,7 @@ */ using System; +using System.Collections.Generic; using System.ComponentModel; using System.Linq; using System.Reflection; @@ -93,6 +94,49 @@ the window's count. */ Assert.DoesNotContain("LIMIT 200", sql, StringComparison.Ordinal); } + /// #5644 W21: the web's collapsed read keeps ONE row per recommendation (its newest capture), counts the captures + /// and reports the oldest, under the same window, scope, cap and database filter as the per-capture read. + [Fact] + public void PlanCorrectionsCollapsedSql_OneRowPerRecommendation_NewestCapture_WithFirstSeenAndCount() + { + var sql = DarlingPlanCorrectionReader.PlanCorrectionsCollapsedSql; + + SqlTextPin.AssertExpresses("PARTITION BY database_name, recommendation_name ORDER BY collection_time DESC", sql, + "the newest capture of each recommendation is no longer the one row kept"); + SqlTextPin.AssertExpresses("MIN(collection_time) OVER (PARTITION BY database_name, recommendation_name) AS first_seen", sql, + "the recommendation's first capture is no longer reported"); + SqlTextPin.AssertExpresses("COUNT(*) OVER (PARTITION BY database_name, recommendation_name) AS capture_count", sql, + "the capture count is no longer reported"); + SqlTextPin.AssertExpresses("WHERE newest_rank = 1", sql, "every capture, not one per recommendation, comes back"); + SqlTextPin.AssertExpresses("recommendation_name IS NOT NULL", sql, "enablement-only rows come back as phantom recommendations"); + SqlTextPin.AssertExpresses("WHERE server_id = $1", sql, "the read is not scoped to the requested server"); + SqlTextPin.AssertExpresses("collection_time >= $2", sql, "the window's start no longer bounds the read"); + SqlTextPin.AssertExpresses("collection_time <= $3", sql, "the window's end no longer bounds the read"); + SqlTextPin.AssertExpresses("($5::text[] IS NULL OR database_name = ANY($5))", sql, "the database filter no longer narrows the read"); + SqlTextPin.AssertExpresses("ORDER BY collection_time DESC, score DESC", sql, "the newest recommendations no longer come first"); + SqlTextPin.AssertExpresses("LIMIT $4", sql, "the row cap is no longer the caller's limit"); + } + + /// #5644 W21: the web-only keys are added to the serialized rows, and the MCP tool's own call never reaches them. + [Fact] + public void WithSeenKeys_AddsFirstSeenLastSeenAndCount_ToEachRow() + { + var newest = new DateTime(2026, 10, 12, 8, 5, 0); + var oldest = new DateTime(2026, 10, 11, 8, 5, 0); + var page = new List + { + new(newest, "db", null, "Active", null, null, 80, null, 7, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, oldest, 17), + }; + + var array = DarlingMcpPlanCorrectionTools.WithSeenKeys(new object[] { new { database_name = "db", query_id = 7 } }, page); + + var row = array[0]!.AsObject(); + Assert.Equal("db", (string?)row["database_name"]); + Assert.Equal(newest.ToString("o"), (string?)row["last_seen"]); + Assert.Equal(oldest.ToString("o"), (string?)row["first_seen"]); + Assert.Equal(17L, (long?)row["capture_count"]); + } + [Fact] public void AutomaticTuningSql_NewestCaptureOnly_OneRowPerDatabase() { @@ -114,6 +158,7 @@ public void AutomaticTuningSql_NewestCaptureOnly_OneRowPerDatabase() [Theory] [InlineData(nameof(DarlingPlanCorrectionReader.PlanCorrectionsSql))] + [InlineData(nameof(DarlingPlanCorrectionReader.PlanCorrectionsCollapsedSql))] [InlineData(nameof(DarlingPlanCorrectionReader.AutomaticTuningSql))] public void Reads_ArePostgresDialect_NoTsqlIsms(string sqlName) { diff --git a/Darling/Darling.Tests/DarlingWebEndpointsTests.cs b/Darling/Darling.Tests/DarlingWebEndpointsTests.cs index 342ee956af..c02c1c2f28 100644 --- a/Darling/Darling.Tests/DarlingWebEndpointsTests.cs +++ b/Darling/Darling.Tests/DarlingWebEndpointsTests.cs @@ -464,6 +464,85 @@ public void ClassifyToolResponse_TheResolversRegistryReadFault_IsAServerError() public void ParseInt_DefaultsOnMissOrGarbage(string? raw, int expected) => Assert.Equal(expected, DarlingWebEndpoints.ParseInt(raw, 10)); + /// + /// #5644 W22: a present, non-integer integer knob is refused (the 400 envelope), not read as the default, so + /// ?hours=abc and ?hours=1.5 no longer answer 200 for a different window than the one asked for. + /// Absent and empty knobs still take the default; a whole number (even out of range) is the tool's to judge. + /// + [Theory] + [InlineData("hours", "abc")] + [InlineData("hours", "1.5")] + [InlineData("hours", "1e2")] + [InlineData("hours_back", "24h")] + [InlineData("limit", "ten")] + [InlineData("top", "5.0")] + [InlineData("days_back", "x")] + [InlineData("bucket_minutes", "five")] + [InlineData("worst_count", "-")] + public void NonIntegerParamRefusal_RefusesAPresentNonInteger(string key, string raw) + { + var refusal = DarlingWebEndpoints.NonIntegerParamRefusal(k => k == key ? raw : null); + + Assert.NotNull(refusal); + Assert.True(McpHelpers.IsRefusalEnvelope(refusal)); + var envelope = JsonNode.Parse(refusal)!; + Assert.Equal("invalid", (string?)envelope["status"]); + Assert.Equal(key, (string?)envelope["hints"]!["parameter"]); + Assert.Equal($"Invalid {key} value '{raw}'. Must be a whole number.", (string?)envelope["message"]); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("24")] + [InlineData("-5")] // readable: the tool owns the range refusal ("Must be a positive integer") + [InlineData("500")] + [InlineData("+7")] + public void NonIntegerParamRefusal_LeavesAbsentEmptyAndWholeNumbersToTheTool(string? raw) + { + foreach (var key in DarlingWebEndpoints.IntegerQueryKeys) + { + Assert.Null(DarlingWebEndpoints.NonIntegerParamRefusal(k => k == key ? raw : null)); + } + } + + [Fact] + public void NonIntegerParamRefusal_ThroughTheQueryString_ReadsTheFirstNonEmptyValue() + { + var context = new Microsoft.AspNetCore.Http.DefaultHttpContext(); + context.Request.QueryString = new Microsoft.AspNetCore.Http.QueryString("?server_name=SQL1&hours=abc"); + Assert.NotNull(DarlingWebEndpoints.NonIntegerParamRefusal(k => DarlingWebEndpoints.First(context, k))); + + context.Request.QueryString = new Microsoft.AspNetCore.Http.QueryString("?hours=&hours=12"); + Assert.Null(DarlingWebEndpoints.NonIntegerParamRefusal(k => DarlingWebEndpoints.First(context, k))); + } + + /// + /// #5644 W22 source pin: the /api/read dispatch loop and /api/fleet both call the check BEFORE any tool or reader + /// runs, and every key the dispatch reads through QueryInt/Rows/Hours is in the list. + /// + [Fact] + public void NonIntegerParamRefusal_RunsBeforeEveryReadAndCoversEveryIntegerKnob() + { + var source = RepoFile.ReadRepoFile("Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs"); + + Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(source, @"NonIntegerParamRefusal\(context\) is \{ \}").Count); + Assert.True(source.IndexOf("NonIntegerParamRefusal(context) is { } nonIntegerRefusal", StringComparison.Ordinal) + < source.IndexOf("result = await handler(context, postgres, analysis);", StringComparison.Ordinal)); + + var keys = new HashSet(StringComparer.Ordinal); + foreach (System.Text.RegularExpressions.Match m in System.Text.RegularExpressions.Regex.Matches( + source, @"(?:QueryInt\((?:c|context), ""|Rows\(c, "")([a-z_]+)""")) + { + keys.Add(m.Groups[1].Value); + } + + keys.Add("hours"); + keys.Add("hours_back"); + Assert.NotEmpty(keys); + Assert.All(keys, k => Assert.Contains(k, DarlingWebEndpoints.IntegerQueryKeys)); + } + [Theory] [InlineData("true", false, true)] [InlineData("false", true, false)] diff --git a/Darling/Darling.Tests/JobHistoryPageTests.cs b/Darling/Darling.Tests/JobHistoryPageTests.cs index 9ad0072e86..e9ac89101e 100644 --- a/Darling/Darling.Tests/JobHistoryPageTests.cs +++ b/Darling/Darling.Tests/JobHistoryPageTests.cs @@ -73,7 +73,7 @@ public void TheColumnKeysAreTheOnesTheToolEmits() { var source = ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingMcpJobTools.cs"); var keys = Regex.Matches(Page(), "\\{ key: \"([a-z_]+)\"").Select(m => m.Groups[1].Value).ToList(); - Assert.Equal(10, keys.Count); + Assert.Equal(9, keys.Count); foreach (var key in keys) Assert.Matches("(?m)^\\s+" + Regex.Escape(key) + " = ", source.ReplaceLineEndings("\n")); } @@ -149,8 +149,9 @@ public void TheFirstBuildReadsTheFleetWithTheDefaultsAndTheDesktopColumns() Assert.Equal("100", q.GetProperty("limit").GetString()); Assert.False(q.TryGetProperty("server", out _)); Assert.True(call.GetProperty("signal").GetBoolean()); - Assert.Equal("Run Time,Server,Job,Category,Step,Status,Duration,Retries,Last Success,Message", string.Join(",", Strings(run.GetProperty("columns")))); - Assert.Equal("Run Time,Server,Job,Category,Step,Status,Duration,Retries,Last Success,Message", string.Join(",", Strings(first.GetProperty("headers")))); + /* #5644 W21: no Last Success column; it was the job's current last success repeated on every run row. */ + Assert.Equal("Run Time,Server,Job,Category,Step,Status,Duration,Retries,Message", string.Join(",", Strings(run.GetProperty("columns")))); + Assert.Equal("Run Time,Server,Job,Category,Step,Status,Duration,Retries,Message", string.Join(",", Strings(first.GetProperty("headers")))); Assert.Equal(2, first.GetProperty("bodyRows").GetInt32()); Assert.Equal(",srv-a,srv-b", string.Join(",", Strings(first.GetProperty("servers")))); Assert.Equal(",Backup,Maintenance", string.Join(",", Strings(first.GetProperty("categories")))); diff --git a/Darling/Darling.Tests/PlanCorrectionCollapseLiveTests.cs b/Darling/Darling.Tests/PlanCorrectionCollapseLiveTests.cs new file mode 100644 index 0000000000..5f88b3f3b0 --- /dev/null +++ b/Darling/Darling.Tests/PlanCorrectionCollapseLiveTests.cs @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #5644 W21: the web's Plan Corrections read collapses the collector's per-cycle re-captures to one row per +/// recommendation. Three recommendations captured five times each: the per-capture read returns 15 rows, the +/// collapsed read returns 3, each at its newest capture with its first capture and its capture count. The cap then +/// bounds DISTINCT recommendations. +/// +/// Plants fifteen rows under a per-run server id and deletes them in a finally through +/// , creating and dropping no relations. +/// +[Collection("live-postgres")] +public sealed class PlanCorrectionCollapseLiveTests +{ + private const string ServerName = "darling-5644-plan-correction-collapse"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + + private const int Recommendations = 3; + private const int Captures = 5; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task RepeatedCaptures_CollapseToOneRowPerRecommendation_WithFirstSeenAndCount() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live collapse test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(cs!); + var bodySucceeded = false; + + try + { + /* Whole seconds, so the planted and returned instants compare for exact equality. */ + var newest = DateTime.SpecifyKind( + new DateTime(DateTime.UtcNow.AddMinutes(-5).Ticks / TimeSpan.TicksPerSecond * TimeSpan.TicksPerSecond), DateTimeKind.Unspecified); + var oldest = newest.AddMinutes(-5 * (Captures - 1)); + + for (var recommendation = 1; recommendation <= Recommendations; recommendation++) + { + for (var capture = 0; capture < Captures; capture++) + { + await PlantAsync(connection, oldest.AddMinutes(5 * capture), recommendation, capture, ct); + } + } + + var start = newest.AddHours(-1); + var end = newest.AddHours(1); + + var perCapture = await DarlingPlanCorrectionReader.GetPlanCorrectionsAsync( + postgres, ServerId, start, end, cap: 50, cancellationToken: ct); + Assert.Equal(Recommendations * Captures, perCapture.Count); + Assert.All(perCapture, r => Assert.Null(r.CaptureCount)); + + var collapsed = await DarlingPlanCorrectionReader.GetCollapsedPlanCorrectionsAsync( + postgres, ServerId, start, end, cap: 50, cancellationToken: ct); + + Assert.Equal(Recommendations, collapsed.Count); + Assert.All(collapsed, r => + { + Assert.Equal(newest, r.CollectionTime); + Assert.Equal(oldest, r.FirstSeen); + Assert.Equal(Captures, r.CaptureCount); + /* The newest capture's own fields, not the oldest's: the state advanced with each capture. */ + Assert.Equal("State" + (Captures - 1), r.RecommendationState); + }); + Assert.Equal(new long?[] { 1, 2, 3 }, collapsed.Select(r => r.QueryId).OrderBy(q => q).ToArray()); + + /* The cap bounds distinct recommendations: a cap of 2 returns 2 of the 3, not 2 captures of one. */ + var capped = await DarlingPlanCorrectionReader.GetCollapsedPlanCorrectionsAsync( + postgres, ServerId, start, end, cap: 2, cancellationToken: ct); + Assert.Equal(2, capped.Count); + Assert.Equal(2, capped.Select(r => r.QueryId).Distinct().Count()); + + /* A window that holds only the last two captures counts those two, and first_seen is the oldest in the window. */ + var narrow = await DarlingPlanCorrectionReader.GetCollapsedPlanCorrectionsAsync( + postgres, ServerId, newest.AddMinutes(-5), end, cap: 50, cancellationToken: ct); + Assert.Equal(Recommendations, narrow.Count); + Assert.All(narrow, r => + { + Assert.Equal(2, r.CaptureCount); + Assert.Equal(newest.AddMinutes(-5), r.FirstSeen); + }); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + private static async Task PlantAsync(NpgsqlConnection connection, DateTime collectionTime, int recommendation, int capture, CancellationToken ct) + { + using var command = new NpgsqlCommand( + """ + INSERT INTO plan_correction + (collection_id, collection_time, server_id, server_name, database_name, recommendation_name, + recommendation_state, query_id, score) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + """, connection); + command.Parameters.AddWithValue((long)(recommendation * 100 + capture)); + command.Parameters.AddWithValue(collectionTime); + command.Parameters.AddWithValue(ServerId); + command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue("collapse_probe_db"); + command.Parameters.AddWithValue("PlanRegression_" + recommendation); + command.Parameters.AddWithValue("State" + capture); + command.Parameters.AddWithValue((long)recommendation); + command.Parameters.AddWithValue(70 + recommendation); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var cleanup = new NpgsqlCommand($"DELETE FROM plan_correction WHERE server_id = {ServerId};", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/ServerTabGridColumnPinTests.cs b/Darling/Darling.Tests/ServerTabGridColumnPinTests.cs index 64ff87ede3..38f9bd0ea8 100644 --- a/Darling/Darling.Tests/ServerTabGridColumnPinTests.cs +++ b/Darling/Darling.Tests/ServerTabGridColumnPinTests.cs @@ -37,7 +37,7 @@ private static void EveryKeyIsReturnedBy(string constant, string toolFile, param var tool = Tool(toolFile); var cols = Columns(constant); foreach (var (key, _) in cols) - Assert.True(Regex.IsMatch(tool, @"\b" + key + @"\s*="), $"{constant}: '{key}' is not a field of {toolFile}"); + Assert.True(Regex.IsMatch(tool, @"\b" + key + @"(\s*=|""\]\s*=)"), $"{constant}: '{key}' is not a field of {toolFile}"); foreach (var key in expected) Assert.Contains(cols, c => c.Key == key); Assert.Equal(cols.Count, cols.Select(c => c.Key).Distinct().Count()); diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs index 448e6b7640..27a34bd5e8 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs @@ -342,6 +342,12 @@ makes it the only one that can answer when the store IS the problem — so it re /* The pre-banded fleet roll-up (also surfaced as the get_fleet_overview MCP tool). */ app.MapGet("/api/fleet", async (HttpContext context) => { + /* #5644 W22: a present, non-integer window or count is refused like an out-of-range one. */ + if (NonIntegerParamRefusal(context) is { } fleetRefusal) + { + return ToHttpResult(fleetRefusal, DarlingWebFailureLog.RouteOf(context.Request), logger, 0, context.RequestAborted); + } + var hours = Math.Clamp(QueryInt(context, "hours_back", "hours", DefaultFleetHours), 1, 168); var worstCount = Math.Max(0, QueryInt(context, "worst_count", null, DarlingFleetReader.DefaultWorstCount)); var now = DateTime.UtcNow; @@ -390,6 +396,14 @@ makes it the only one that can answer when the store IS the problem — so it re readScope.Scope.CaptureStatements = true; try { + /* #5644 W22: ?hours=abc / ?hours=1.5 (or any other integer knob) was read as the default and answered + 200 for a different window than the one asked for, while ?hours=500 got a 400. Refused up front now, + before any tool runs, in the same envelope the range refusals use. */ + if (NonIntegerParamRefusal(context) is { } nonIntegerRefusal) + { + return ToHttpResult(nonIntegerRefusal, DarlingWebFailureLog.RouteOf(context.Request), logger, stopwatch.ElapsedMilliseconds, context.RequestAborted); + } + /* A newest-first capped read is judged against the window it read: with no anchor sent, the end is taken once here and handed to both the read and the note check. A request with its own anchor is left as sent. */ @@ -5584,7 +5598,7 @@ budget cut does not silently shrink what the viewer renders. */ web viewer has always shown the whole query text. The row pins its OWN default to true so #4198's MCP-side budget cut does not silently truncate what the viewer renders. */ ["get_plan_corrections"] = (c, pg, an) => DatabaseNames(c) is { } databases - ? DarlingMcpPlanCorrectionTools.GetPlanCorrections(pg, Server(c), Hours(c, 24), Rows(c, "limit", 50), AsOf(c), QueryBool(c, "full_text", true), databases, null, c.RequestAborted) + ? DarlingMcpPlanCorrectionTools.GetPlanCorrections(pg, Server(c), Hours(c, 24), Rows(c, "limit", 50), AsOf(c), QueryBool(c, "full_text", true), databases, null, c.RequestAborted, collapse: true) : DatabaseNamesRefusal(c), /* ── config (current + history) ── */ @@ -6422,6 +6436,39 @@ private static Task UnparseableParam(string key, string? expected = null Task.FromResult(McpHelpers.Refusal(key, $"Invalid value for parameter '{key}'. {expected ?? "Expected a number."}")); + /// + /// Every query-string knob the dispatch reads as a whole number through / / + /// (#5644 W22). reads an unreadable one as the default, so a window or row + /// count the caller got wrong was answered, with a 200, for a different one than the one asked for. + /// + internal static readonly string[] IntegerQueryKeys = + [ + "hours", "hours_back", "days_back", "trend_hours_back", "baseline_hours_back", "limit", "top", + "bucket_minutes", "min_dop", "worst_count" + ]; + + /// + /// The 400 envelope for the first knob that is present but not a whole number + /// (?hours=abc, ?hours=1.5), else null. An absent or empty knob still takes the tool's default, and a number out of + /// range is left to the tool, which owns that refusal. Pure over the lookup, so a test pins it without a server. + /// + internal static string? NonIntegerParamRefusal(Func lookup) + { + foreach (var key in IntegerQueryKeys) + { + var raw = lookup(key); + if (!string.IsNullOrEmpty(raw) && !int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out _)) + { + return McpHelpers.Refusal(key, $"Invalid {key} value '{McpHelpers.Truncate(raw, 40)}'. Must be a whole number."); + } + } + + return null; + } + + private static string? NonIntegerParamRefusal(HttpContext context) => + NonIntegerParamRefusal(key => First(context, key)); + private static int QueryInt(HttpContext context, string key, string? aliasKey, int def) => ParseInt(First(context, key) ?? (aliasKey is null ? null : First(context, aliasKey)), def); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCorrectionTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCorrectionTools.cs index 532bd7372e..c95fd73e94 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCorrectionTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCorrectionTools.cs @@ -7,9 +7,11 @@ */ using System; +using System.Collections.Generic; using System.ComponentModel; using System.Linq; using System.Text.Json; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -43,6 +45,28 @@ public sealed class DarlingMcpPlanCorrectionTools /// private const int QueryTextPreviewLength = 150; + /// + /// #5644 W21, the web read only: each collapsed recommendation row gains first_seen (its oldest capture in the + /// window), last_seen (its newest, the same instant as collection_time) and capture_count. Added to + /// the serialized row rather than to the anonymous shape above, so the MCP tool's payload keys do not move. + /// + internal static JsonArray WithSeenKeys(IEnumerable projected, List page) + { + var array = new JsonArray(); + var index = 0; + foreach (var row in projected) + { + var node = JsonSerializer.SerializeToNode(row, McpHelpers.JsonOptions)!.AsObject(); + var source = page[index++]; + node["first_seen"] = (source.FirstSeen ?? source.CollectionTime).ToString("o"); + node["last_seen"] = source.CollectionTime.ToString("o"); + node["capture_count"] = source.CaptureCount ?? 1; + array.Add(node); + } + + return array; + } + [McpServerTool(Name = "get_plan_corrections"), Description( "Gets SQL Server automatic plan correction (APC) activity: FORCE_LAST_GOOD_PLAN recommendations/actions over the window ending at as_of, newest capture first, plus each database's automatic-tuning enablement. Rows recur per capture, not per distinct recommendation. THE PAGE IS BOUNDED BY limit, NOT hours_back: truncated means more rows existed; oldest/newest_returned_collection_time bound the page. automatic_tuning ignores the window: the latest snapshot; each row's as_of says when. All timestamps are UTC. No rows and no automatic_tuning: empty (not_collected checked first). <> Gets SQL Server automatic plan correction (APC) activity: the engine's FORCE_LAST_GOOD_PLAN recommendations and actions over the window, NEWEST CAPTURE FIRST, plus each database's current automatic-tuning enablement state. Use when a query's plan changed suddenly - APC forcing or unforcing a plan is a first-class explanation - or to check whether automatic tuning is on and actually working (desired vs actual state). Rows come from sys.dm_db_tuning_recommendations captured on a schedule, and the collector RE-CAPTURES every open recommendation on every cycle, so the same recommendation appears once per capture and a few open recommendations fill a page fast. THE PAGE IS BOUNDED BY limit, NOT BY hours_back: recommendations_returned is how many rows you got, truncated says the window held more than limit, and oldest_returned_collection_time / newest_returned_collection_time bound the page - under newest-first ordering the oldest stamp IS how far back this read reached, and on a server with open recommendations a week-long request at the default limit reaches back hours, not days. Raise limit or narrow hours_back when truncated is true. automatic_tuning is a latest-snapshot read that ignores the window entirely; its as_of stamps say when. A recommendation's state moves through Active/Verifying/Success/Reverted as the engine acts. query_text is a preview by default, capped at 150 characters with query_text_truncated marking whichever rows were actually cut - pass full_text=true for the whole regressed statement on every row instead, the same opt-in get_store_query_stats uses. Every timestamp here is UTC, including valid_since / last_refresh / execute_action_initiated_time / revert_action_initiated_time - sys.dm_db_tuning_recommendations reports those four in UTC and they are stored and returned unconverted - so they order correctly against collection_time and against get_query_store_regressions.")] public static Task GetPlanCorrections( @@ -73,7 +97,8 @@ internal static async Task GetPlanCorrections( bool full_text, DatabaseFilter databaseFilter, ILogger? logger, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool collapse = false) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name, cancellationToken); if (error != null) return error; @@ -90,8 +115,11 @@ internal static async Task GetPlanCorrections( /* #3541 A3: the caller's limit + 1 as the fetch, the extra row as the observed truncation signal. The reader's LIMIT 200 over per-cycle re-captures gave every window the same ~16-hour reach, and `total_recommendations` published that page as the window's count. */ - var rows = await DarlingPlanCorrectionReader.GetPlanCorrectionsAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1, databaseFilter, cancellationToken); + var rows = collapse + ? await DarlingPlanCorrectionReader.GetCollapsedPlanCorrectionsAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1, databaseFilter, cancellationToken) + : await DarlingPlanCorrectionReader.GetPlanCorrectionsAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1, databaseFilter, cancellationToken); var truncated = rows.Count > limit; var page = truncated ? rows.Take(limit).ToList() : rows; @@ -175,7 +203,7 @@ recommendation rows and only the tuning snapshot answered. */ oldest_returned_collection_time = page.Count == 0 ? null : McpHelpers.FormatEffectiveStart(page.Min(r => r.CollectionTime)), newest_returned_collection_time = page.Count == 0 ? null : McpHelpers.FormatEffectiveStart(page.Max(r => r.CollectionTime)), order = "collection_time_desc", - recommendations, + recommendations = collapse ? WithSeenKeys(recommendations, page) : (object)recommendations, }; var json = JsonSerializer.Serialize(payload, McpHelpers.JsonOptions); return notice.IsUnavailable ? DarlingMcpWindowNotice.WithoutKeys(json) : json; diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs index 1b944a9e47..a01a2d9162 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs @@ -88,6 +88,84 @@ FROM plan_correction AND recommendation_name IS NOT NULL AND ($5::text[] IS NULL OR database_name = ANY($5)) ORDER BY collection_time DESC, score DESC +LIMIT $4"; + + /// + /// #5644 W21: collapsed to ONE row per recommendation (database + the engine's + /// recommendation_name). The collector re-captures every open recommendation on every cycle, so the per-capture + /// read above fills a 50-row page with three recommendations seen seventeen times each. This read returns each + /// recommendation's NEWEST capture (so collection_time is its last-seen time and the state is its latest), with + /// first_seen (its oldest capture in the window) and capture_count (captures in the window) as the two + /// trailing columns. The cap bounds DISTINCT recommendations, newest last-seen first. The same $1-$5 parameters. + /// Used by the web read only; the MCP tool keeps the per-capture rows its contract documents. + /// + public const string PlanCorrectionsCollapsedSql = @" +WITH captured AS ( + SELECT + collection_time, + database_name, + query_text, + recommendation_state, + recommendation_state_reason, + recommendation_reason, + score, + estimated_gain_seconds, + query_id, + regressed_plan_id, + last_good_plan_id, + last_good_plan_forcing_type, + last_good_plan_is_forced, + last_good_plan_force_failure_reason, + regressed_plan_execution_count, + regressed_plan_cpu_time_average_ms, + last_good_plan_execution_count, + last_good_plan_cpu_time_average_ms, + valid_since, + last_refresh, + execute_action_initiated_by, + execute_action_initiated_time, + revert_action_initiated_by, + revert_action_initiated_time, + ROW_NUMBER() OVER (PARTITION BY database_name, recommendation_name ORDER BY collection_time DESC) AS newest_rank, + MIN(collection_time) OVER (PARTITION BY database_name, recommendation_name) AS first_seen, + COUNT(*) OVER (PARTITION BY database_name, recommendation_name) AS capture_count + FROM plan_correction + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + AND recommendation_name IS NOT NULL + AND ($5::text[] IS NULL OR database_name = ANY($5)) +) +SELECT + collection_time, + database_name, + query_text, + recommendation_state, + recommendation_state_reason, + recommendation_reason, + score, + estimated_gain_seconds, + query_id, + regressed_plan_id, + last_good_plan_id, + last_good_plan_forcing_type, + last_good_plan_is_forced, + last_good_plan_force_failure_reason, + regressed_plan_execution_count, + regressed_plan_cpu_time_average_ms, + last_good_plan_execution_count, + last_good_plan_cpu_time_average_ms, + valid_since, + last_refresh, + execute_action_initiated_by, + execute_action_initiated_time, + revert_action_initiated_by, + revert_action_initiated_time, + first_seen, + capture_count +FROM captured +WHERE newest_rank = 1 +ORDER BY collection_time DESC, score DESC LIMIT $4"; /// @@ -135,7 +213,9 @@ public sealed record PlanCorrectionRow( string? ExecuteActionInitiatedBy, DateTime? ExecuteActionInitiatedTimeUtc, string? RevertActionInitiatedBy, - DateTime? RevertActionInitiatedTimeUtc); + DateTime? RevertActionInitiatedTimeUtc, + DateTime? FirstSeen = null, + long? CaptureCount = null); /// One database's FORCE_LAST_GOOD_PLAN enablement state at the newest capture. public sealed record AutomaticTuningRow( @@ -148,11 +228,21 @@ public sealed record AutomaticTuningRow( /// The newest recommendation rows over the window. Callers detecting /// truncation pass limit + 1 and read the extra row as the signal. (#5244) narrows /// to the chosen databases before the cap (, the default: every database). - public static async Task> GetPlanCorrectionsAsync( + public static Task> GetPlanCorrectionsAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, DatabaseFilter databases = default, CancellationToken cancellationToken = default) + => ReadPlanCorrectionsAsync(postgres, serverId, startUtc, endUtc, cap, databases, collapse: false, cancellationToken); + + /// collapsed to one row per recommendation (#5644 W21, ): + /// each row is the recommendation's newest capture with FirstSeen and CaptureCount set, and the cap counts recommendations. + public static Task> GetCollapsedPlanCorrectionsAsync( NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, DatabaseFilter databases = default, CancellationToken cancellationToken = default) + => ReadPlanCorrectionsAsync(postgres, serverId, startUtc, endUtc, cap, databases, collapse: true, cancellationToken); + + private static async Task> ReadPlanCorrectionsAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, DatabaseFilter databases, bool collapse, CancellationToken cancellationToken) { var rows = new List(); - await using var command = postgres.CreateCommand(PlanCorrectionsSql); + await using var command = postgres.CreateCommand(collapse ? PlanCorrectionsCollapsedSql : PlanCorrectionsSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; command.Parameters.AddWithValue(serverId); command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); @@ -187,7 +277,9 @@ public static async Task> GetPlanCorrectionsAsync( reader.IsDBNull(20) ? null : reader.GetString(20), reader.IsDBNull(21) ? null : reader.GetDateTime(21), reader.IsDBNull(22) ? null : reader.GetString(22), - reader.IsDBNull(23) ? null : reader.GetDateTime(23))); + reader.IsDBNull(23) ? null : reader.GetDateTime(23), + collapse && !reader.IsDBNull(24) ? reader.GetDateTime(24) : null, + collapse && !reader.IsDBNull(25) ? reader.GetInt64(25) : null)); } return rows; diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/job-history.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/job-history.js index f5120cf229..a2face744c 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/job-history.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/job-history.js @@ -42,7 +42,9 @@ export const STATUSES = [ /** The row limits on offer (the read takes at most 1000). */ export const LIMITS = [100, 250, 500, 1000]; -/** The desktop grid's columns, in its order: Run Time, Server, Job, Category, Step, Status, Duration, Retries, Last Success, Message. */ +/** The grid's columns, in the desktop grid's order: Run Time, Server, Job, Category, Step, Status, Duration, Retries, Message. + * #5644 W21: the desktop grid's Last Success column is not here. The read's `last_success` is the JOB's current last + * success, the same on every run row of a job, so an older run showed a time later than its own run. */ export const JOB_HISTORY_COLUMNS = [ { key: "run_time", label: "Run Time", format: "time" }, { key: "server", label: "Server" }, @@ -52,7 +54,6 @@ export const JOB_HISTORY_COLUMNS = [ { key: "status", label: "Status" }, { key: "duration_formatted", label: "Duration", sortValue: (r) => r.duration_seconds }, { key: "retries", label: "Retries", format: "int" }, - { key: "last_success", label: "Last Success", format: "time" }, { key: "message", label: "Message", wrap: true, render: messageCell }, ]; diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js index 97cb90a3a9..628ad6a78b 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js @@ -3589,8 +3589,12 @@ const LONG_QUERY_COLUMNS = [ returns no implementation script or action flags, and the page does not rebuild them. */ const PLAN_CORRECTION_GROUPS = { groups: ["Plans", "Plan metrics", "Lifecycle"], defaultGroups: [] }; +/* #5644 W21: the web read collapses the collector's per-cycle re-captures to one row per recommendation, so the old + "Collected" column is the recommendation's LAST capture, with its first capture and how many captures the window held beside it. */ const PLAN_CORRECTION_COLUMNS = [ - { key: "collection_time", label: "Collected", format: "time" }, + { key: "first_seen", label: "First seen", format: "time" }, + { key: "collection_time", label: "Last seen", format: "time" }, + { key: "capture_count", label: "Captures", format: "int" }, { key: "query_text", label: "Query", valueList: false, render: (r) => codeDisclosure(r.query_text) }, { key: "database_name", label: "Database" }, { key: "recommendation_state", label: "State" }, @@ -4548,7 +4552,6 @@ const JOB_HISTORY_COLUMNS = [ { key: "status", label: "Status" }, { key: "duration_formatted", label: "Duration", sortValue: (r) => r.duration_seconds }, { key: "retries", label: "Retries", format: "int" }, - { key: "last_success", label: "Last Success", format: "time" }, { key: "message", label: "Message", wrap: true, render: jobMessageCell }, ]; diff --git a/Lite.Tests/BaselineDiscontinuityTests.cs b/Lite.Tests/BaselineDiscontinuityTests.cs index c9be69622d..68026d3264 100644 --- a/Lite.Tests/BaselineDiscontinuityTests.cs +++ b/Lite.Tests/BaselineDiscontinuityTests.cs @@ -112,10 +112,32 @@ public void Compose_IdentityMarker_WithItsPair_IsARestartNamingBothStartTimes() Assert.Equal(T0, one.At); Assert.Equal(BaselineDiscontinuities.RestartReason, one.Reason); Assert.Contains("2026-09-01 08:00:00 → 2026-09-20 03:11:40", one.Detail, StringComparison.Ordinal); - Assert.Contains("server name NODE1 → NODE1", one.Detail, StringComparison.Ordinal); + /* #5644 W17: the name did not move, so it is not listed as a change. */ + Assert.DoesNotContain("server name", one.Detail, StringComparison.Ordinal); Assert.EndsWith("observed by wait_stats", one.Detail, StringComparison.Ordinal); } + /// #5644 W17: only the sides that changed are named. A rename shows the name pair alone; a failover shows both; + /// a name that differs by case only is not a change (the epoch test compares names case-insensitively). + [Fact] + public void Compose_NamesOnlyTheSidesThatChanged() + { + var oldStart = new DateTime(2026, 10, 8, 0, 10, 55); + var newStart = oldStart.AddHours(24); + var rows = new[] { Marker(T0, "wait_stats", "identity_epoch_changes=1") }; + + var restart = Assert.Single(BaselineDiscontinuities.Compose(rows, Pair("wait_stats", T0, new(oldStart, "AG1"), new(newStart, "ag1")))).Detail; + Assert.Contains("start time 2026-10-08 00:10:55 → 2026-10-09 00:10:55", restart, StringComparison.Ordinal); + Assert.DoesNotContain("server name", restart, StringComparison.Ordinal); + + var rename = Assert.Single(BaselineDiscontinuities.Compose(rows, Pair("wait_stats", T0, new(oldStart, "OLD"), new(oldStart, "NEW")))).Detail; + Assert.Contains("server name OLD → NEW", rename, StringComparison.Ordinal); + Assert.DoesNotContain("start time", rename, StringComparison.Ordinal); + + var failover = Assert.Single(BaselineDiscontinuities.Compose(rows, Pair("wait_stats", T0, new(oldStart, "OLD"), new(newStart, "NEW")))).Detail; + Assert.Contains("start time 2026-10-08 00:10:55 → 2026-10-09 00:10:55, server name OLD → NEW", failover, StringComparison.Ordinal); + } + [Fact] public void Compose_BothHalvesMoved_IsAFailover() { diff --git a/PerformanceMonitor.Collectors/BaselineDiscontinuity.cs b/PerformanceMonitor.Collectors/BaselineDiscontinuity.cs index 2e87bfb9a8..c5b150f0ac 100644 --- a/PerformanceMonitor.Collectors/BaselineDiscontinuity.cs +++ b/PerformanceMonitor.Collectors/BaselineDiscontinuity.cs @@ -333,8 +333,7 @@ private static Resolved Resolve(Observation o, Dictionary late { var reason = DeriveIdentityReason(replaced, current); return new Resolved(o.At, o.Collector, o.Label, reason, - "instance identity changed — start time " + Describe(replaced.StartTime) + " → " + Describe(current.StartTime) - + ", server name " + Describe(replaced.Name) + " → " + Describe(current.Name) + "instance identity changed — " + ServerEpoch.DescribeChange(replaced, current) + "; every delta baseline for this server was forgotten and re-established on the next pass", Specific: reason != IdentityReason); } diff --git a/PerformanceMonitor.Collectors/ServerEpoch.cs b/PerformanceMonitor.Collectors/ServerEpoch.cs index a9f03efa38..287204b208 100644 --- a/PerformanceMonitor.Collectors/ServerEpoch.cs +++ b/PerformanceMonitor.Collectors/ServerEpoch.cs @@ -317,8 +317,7 @@ public static bool ObserveInstance(CollectorContext context, Stamp current) { context.Deltas.ClearServer( context.ServerId, - "identity epoch changed — start time " + Describe(replaced.StartTime) + " → " + Describe(current.StartTime) - + ", server name " + Describe(replaced.Name) + " → " + Describe(current.Name) + "identity epoch changed — " + DescribeChange(replaced, current) + ": every delta baseline and pass window for this server was forgotten, so this pass re-baselines " + "and the next reports the first interval measured on the instance the connection now reaches"); /* What was observed, not the merge with the prior: the prior's components describe the OLD @@ -468,6 +467,37 @@ private static void Persist(CollectorContext context, string stateKey, Stamp? pr } } + /// + /// The sides of an identity change that actually moved, for a banner or a log line (#5644 W17): "start time A → B" + /// and/or "server name X → Y", joined by a comma. A side that did not change is left out, because listing + /// "server name X → X" reads as a rename that never happened. "Moved" is exactly 's + /// own test (known on both sides and different; names case-insensitive). When neither side moved by that + /// test (a defensive case: the caller only asks about an epoch), both pairs are printed as before. + /// + public static string DescribeChange(Stamp replaced, Stamp current) + { + var parts = new List(2); + if (replaced.StartTime.HasValue && current.StartTime.HasValue + && replaced.StartTime.Value != current.StartTime.Value) + { + parts.Add("start time " + Describe(replaced.StartTime) + " → " + Describe(current.StartTime)); + } + + if (replaced.Name is not null && current.Name is not null + && !string.Equals(replaced.Name, current.Name, StringComparison.OrdinalIgnoreCase)) + { + parts.Add("server name " + Describe(replaced.Name) + " → " + Describe(current.Name)); + } + + if (parts.Count == 0) + { + parts.Add("start time " + Describe(replaced.StartTime) + " → " + Describe(current.StartTime)); + parts.Add("server name " + Describe(replaced.Name) + " → " + Describe(current.Name)); + } + + return string.Join(", ", parts); + } + private static string Describe(DateTime? value) => value.HasValue ? value.Value.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture) : "unknown"; From 69bd3206d43b386c84685762c8c3ad87dd665acf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:39:18 -0400 Subject: [PATCH 10/42] Default Trace: stop re-storing the newest event and keep the Commit row of each DDL; readers drop stored copies (#5644 W6) --- ...DarlingDefaultTraceReaderServerClockTests.cs | 17 +++++++++++++++++ .../Mcp/DarlingDefaultTraceReader.cs | 6 +++++- ...efaultTraceEventsCollectorDefinitionTests.cs | 16 +++++++++++++++- Lite/Services/LocalDataService.SystemEvents.cs | 5 ++++- .../DefaultTraceEventsCollector.cs | 7 ++++++- 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs b/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs index ca5690a4c2..95797418da 100644 --- a/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs +++ b/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs @@ -64,6 +64,23 @@ public async Task AnEventStoredAtServerLocalTime_OnEachSideOfAChange_LandsAtTheR Assert.Equal(Naive(2026, 11, 2, 13), byLabel["after-fall"]); } + [Fact] + public async Task TheSameEventStoredSeveralTimes_ComesBackOnce_AndADifferentEventSurvives() + { + /* #5644 W6: nine stored copies of one Server Memory Change row, and a Begin/Commit pair, are one event each. */ + var table = NewTable(); + for (var i = 0; i < 9; i++) + { + AddEvent(table, Naive(2026, 10, 12, 10, 49), "same"); + } + + AddEvent(table, Naive(2026, 10, 12, 10, 49), "other"); + + var rows = await ReadAsync(table, Eastern(), Naive(2026, 1, 1, 0), Naive(2026, 12, 31, 0)); + + Assert.Equal(new[] { "same", "other" }, rows.Select(r => r.DatabaseName).ToArray()); + } + [Fact] public async Task AnEventInASkippedOrRepeatedLocalHour_DoesNotThrow() { diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDefaultTraceReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDefaultTraceReader.cs index 06bae30a7e..c549a56fc4 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDefaultTraceReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDefaultTraceReader.cs @@ -200,6 +200,10 @@ internal static async Task> ReadEventRowsAsync( } /* OrderByDescending is a stable sort (List.Sort is not), so ties keep the reader's order. */ - return rows.OrderByDescending(static r => r.EventTimeUtc).ToList(); + /* #5644 W6: the store holds rows that are the same event twice (an older build kept both the Begin and the + Commit of every DDL, and re-stored an event whose datetime StartTime did not round-trip through the + watermark parameter). The record's value equality makes Distinct drop the copies, keeping the first of each in + reader order, so they stop filling the page limit. */ + return rows.Distinct().OrderByDescending(static r => r.EventTimeUtc).ToList(); } } diff --git a/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs b/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs index f5db62579e..560bdd31f5 100644 --- a/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs +++ b/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs @@ -84,13 +84,27 @@ public void BuildQuery_ReadsDefaultTraceViaFnTraceGettable_ServerSide() Assert.Contains("ELSE t.max_files", text, StringComparison.Ordinal); Assert.Contains("t.is_default = 1", text, StringComparison.Ordinal); Assert.Contains("t.status = 1", text, StringComparison.Ordinal); + /* #5644 W6: the bound is CONVERTed to datetime so it compares like for like with ft.StartTime. A bare datetime2 + parameter holding 10:49:13.643 sits BELOW the trace's own 10:49:13.6433333 (datetime ticks in 1/300 s), + so `>` held the newest stored event true on every poll and stored it again until a newer event came. */ Assert.Contains( - "ft.StartTime > COALESCE(@cutoff_time, DATEADD(HOUR, -24, SYSDATETIME()))", + "ft.StartTime > CONVERT(datetime, COALESCE(@cutoff_time, DATEADD(HOUR, -24, SYSDATETIME())))", text, StringComparison.Ordinal); Assert.Contains("SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED", text, StringComparison.Ordinal); Assert.Contains("OPTION(RECOMPILE)", text, StringComparison.Ordinal); } + [Fact] + public void ObjectDdl_KeepsTheCommitRowOnly_NotTheBeginAndCommitPair() + { + /* #5644 W6: SQL Server writes Object:Created/Altered/Deleted twice, EventSubClass 0 (Begin) and 1 (Commit). */ + var text = DefaultTraceEventsCollector.Instance.BuildQuery(MakeContext()).Text; + var ddl = text.IndexOf("te.name IN (N'Object:Created', N'Object:Altered', N'Object:Deleted')", StringComparison.Ordinal); + var subclass = text.IndexOf("AND ISNULL(ft.EventSubClass, 1) = 1", StringComparison.Ordinal); + + Assert.True(ddl > 0 && subclass > ddl, "the Commit-only predicate must follow the Object DDL names, inside the same arm"); + } + [Fact] public void BuildQuery_RolloverPathStrip_FallsBackToBasePath_WhenNoUnderscore() { diff --git a/Lite/Services/LocalDataService.SystemEvents.cs b/Lite/Services/LocalDataService.SystemEvents.cs index b08b8e1f90..584cb0e18b 100644 --- a/Lite/Services/LocalDataService.SystemEvents.cs +++ b/Lite/Services/LocalDataService.SystemEvents.cs @@ -838,8 +838,11 @@ the same clock and held to the exact window below (#4766). Bind db filter params var endTime = clock.ToServerLocal(windowEndUtc).AddHours(1); var dbClause = BuildDbInClause(databaseNames, "database_name", 4, out var dbValues); + /* #5644 W6: DISTINCT drops the stored copies of one event (the Begin and Commit rows of a DDL that an older + build kept, and an event re-stored by the datetime watermark round trip): they match on every column + selected here, and only the event_sequence the collector stores tells them apart. */ command.CommandText = @" -SELECT +SELECT DISTINCT event_time, event_name, database_name, diff --git a/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs b/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs index 6d6f0e9911..e3c29a63fe 100644 --- a/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs +++ b/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs @@ -269,7 +269,7 @@ JOIN sys.trace_events AS te ON ft.EventClass = te.trace_event_id WHERE t.is_default = 1 AND t.status = 1 -AND ft.StartTime > {1} +AND ft.StartTime > CONVERT(datetime, {1}) AND ( /*The sp_configure line (msg 15457, an ErrorLog write): kept regardless of the session's database @@ -307,6 +307,11 @@ OR te.name LIKE N'%Log File Auto Shrink%' AND ISNULL(ft.DatabaseID, 0) <> 2 AND ISNULL(ft.ObjectName, N'') NOT LIKE N'[_]WA[_]%' AND te.name IN (N'Object:Created', N'Object:Altered', N'Object:Deleted') + /*#5644 W6: SQL Server writes each DDL twice, EventSubClass 0 (Begin) and 1 (Commit), identical + but for EventSequence. Keep the Commit: it is the one that took effect. A rolled-back DDL + leaves only its Begin, which the trace shows but nothing changed. NULL stays in case a build + writes no subclass.*/ + AND ISNULL(ft.EventSubClass, 1) = 1 ) OR /*Security audit events (audit-change, DBCC, alter-trace)*/ From 3bdd4ffbfb45b3bc1a478c6b4e945a3effc93c8d Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:27:01 -0400 Subject: [PATCH 11/42] G1a: update pins the lane E merge made stale; test plainText and localizeIsoText together (#5644) --- .../ViewerGridPayloadColumnOrderPinTests.cs | 9 +++++---- .../Darling.Tests/WebChartAtTimeBehaviourTests.cs | 12 +++++++----- .../Darling.Tests/WebClickthroughStaleStateTests.cs | 2 +- .../WebCollectionHealthRunAtColumnTests.cs | 2 +- .../Darling.Tests/WebDataStartNoteEventReadsTests.cs | 6 ++++-- Darling/Darling.Tests/WebDataStartNoteTests.cs | 12 ++++++++---- .../Darling.Tests/WebPartialWindowBehaviourTests.cs | 3 +++ .../Darling.Tests/WebQueryStoreGridGroupsTests.cs | 6 +++--- .../WebWaitQueriesLinkBehaviourTests.cs | 2 +- Darling/Darling.Tests/WebWordingFixesTests.cs | 2 +- Darling/Darling.Tests/web-partial-window-harness.mjs | 4 ++++ Darling/Darling.Tests/web-qs-grids-harness.mjs | 2 +- .../wwwroot/js/charts.js | 2 +- 13 files changed, 40 insertions(+), 24 deletions(-) diff --git a/Darling/Darling.Tests/ViewerGridPayloadColumnOrderPinTests.cs b/Darling/Darling.Tests/ViewerGridPayloadColumnOrderPinTests.cs index 4d688f4b8d..da92995b9d 100644 --- a/Darling/Darling.Tests/ViewerGridPayloadColumnOrderPinTests.cs +++ b/Darling/Darling.Tests/ViewerGridPayloadColumnOrderPinTests.cs @@ -182,16 +182,17 @@ public void PayloadColumnsFollowTheAnchor(string relativePath, string gridName) [InlineData("TOP_QUERY_COLUMNS", "database_name", "query_text", 10)] [InlineData("QUERY_STORE_COLUMNS", "database_name", "query_text", 8)] [InlineData("LONG_QUERY_COLUMNS", "event_time", "statement", 8)] - [InlineData("PLAN_CORRECTION_COLUMNS", "collection_time", "query_text", 8)] + [InlineData("PLAN_CORRECTION_COLUMNS", "capture_count", "query_text", 8, 2)] [InlineData("BLOCKING_COLUMNS", "event_time", "blocked_sql_text", 9)] [InlineData("DEADLOCK_COLUMNS", "deadlock_time", "victim_sql_text", 5)] - public void EveryWebGridWithQueryText_PutsItRightOfTheAnchor(string array, string anchor, string text, int minKeys) + public void EveryWebGridWithQueryText_PutsItRightOfTheAnchor(string array, string anchor, string text, int minKeys, int anchorAt = 0) { // Array order IS the rendered column order — the page hands the array straight to the table renderer. var keys = JsColumnKeys(ReadRepoFile(ServerPageJs), array); Assert.True(keys.Count >= minKeys, $"only {keys.Count} {array} keys parsed — the scan is broken."); - Assert.Equal(anchor, keys[0]); - Assert.Equal(text, keys[1]); + // anchorAt is 0 except Plan Corrections, whose First seen and Last seen stamps lead (#5644 W21) and Captures is the anchor. + Assert.Equal(anchor, keys[anchorAt]); + Assert.Equal(text, keys[anchorAt + 1]); // Without this, re-adding a second text column at the BACK would leave the pin green — the same hole // EveryPinnedPayloadColumnIsUniqueInItsGrid closes on the XAML side. diff --git a/Darling/Darling.Tests/WebChartAtTimeBehaviourTests.cs b/Darling/Darling.Tests/WebChartAtTimeBehaviourTests.cs index 40004b0ae1..f56f9e658c 100644 --- a/Darling/Darling.Tests/WebChartAtTimeBehaviourTests.cs +++ b/Darling/Darling.Tests/WebChartAtTimeBehaviourTests.cs @@ -176,10 +176,11 @@ string Tab(string id) Assert.Contains("panel: \"Blocking\"", charts); Assert.Matches(@"table\(\s*""Blocking"",", blocking); Assert.Contains("panel: \"Deadlocks\"", charts); - // Both the chart and the grid are titled "Deadlocks", and the scroll takes the later one, so the grid must be the later. - var chart = blocking.IndexOf("line(\"Deadlocks\",", StringComparison.Ordinal); + // The Blocking tab's chart is "Deadlock trend" (#5644 W16), so the scroll to "Deadlocks" finds the grid; it still sits below its chart. + var chart = blocking.IndexOf("line(\"Deadlock trend\",", StringComparison.Ordinal); var grid = Regex.Match(blocking, @"table\(\s*""Deadlocks"","); - Assert.True(chart >= 0 && grid.Success && grid.Index > chart, "the Deadlocks grid must come after the Deadlocks chart on the Blocking tab"); + Assert.DoesNotContain("line(\"Deadlocks\",", blocking, StringComparison.Ordinal); + Assert.True(chart >= 0 && grid.Success && grid.Index > chart, "the Deadlocks grid must come after the Deadlock trend chart on the Blocking tab"); } [Fact] @@ -352,8 +353,9 @@ public void TheBlockingAndDeadlockCharts_NameTheirOwnItem_AndNoOtherChartDoes() // deadlock charts "Show Deadlocks at This Time". Every other server-tab line chart keeps the Active Queries default. var tabs = File.ReadAllText(PathTo("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js", "pages", "server-tabs.js")); - // line() panels: Blocking Events and Deadlocks sit on both the Overview tab and the Blocking tab. - foreach (var (title, item, count) in new[] { ("Blocking Events", "blocking", 2), ("Deadlocks", "deadlocks", 2), ("Lock Waits", "blocking", 1) }) + // line() panels: Blocking Events sits on both the Overview tab and the Blocking tab; the deadlock chart is "Deadlocks" on Overview and + // "Deadlock trend" on the Blocking tab (#5644 W16, so the Deadlocks grid below it is the only panel with that title there). + foreach (var (title, item, count) in new[] { ("Blocking Events", "blocking", 2), ("Deadlocks", "deadlocks", 1), ("Deadlock trend", "deadlocks", 1), ("Lock Waits", "blocking", 1) }) { var calls = Regex.Matches(tabs, @"line\(""" + title + @""","); Assert.Equal(count, calls.Count); diff --git a/Darling/Darling.Tests/WebClickthroughStaleStateTests.cs b/Darling/Darling.Tests/WebClickthroughStaleStateTests.cs index 9ac7e5e54a..2cf822254c 100644 --- a/Darling/Darling.Tests/WebClickthroughStaleStateTests.cs +++ b/Darling/Darling.Tests/WebClickthroughStaleStateTests.cs @@ -87,7 +87,7 @@ public void AServerPastTheOfflineMark_ReadsNotAvailableWithItsAge_NotLiveLooking public void AnOnlineServer_KeepsItsLiveValues() { var online = Texts(Run("cards").GetProperty("online")); - Assert.Equal("CPU0%SQL 0%", online[0]); + Assert.Equal("Total CPU0%SQL 0%", online[0]); Assert.StartsWith("Threads77 free", online[1], StringComparison.Ordinal); Assert.StartsWith("Memoryok", online[2], StringComparison.Ordinal); } diff --git a/Darling/Darling.Tests/WebCollectionHealthRunAtColumnTests.cs b/Darling/Darling.Tests/WebCollectionHealthRunAtColumnTests.cs index 63dad300e0..ef615c3214 100644 --- a/Darling/Darling.Tests/WebCollectionHealthRunAtColumnTests.cs +++ b/Darling/Darling.Tests/WebCollectionHealthRunAtColumnTests.cs @@ -64,7 +64,7 @@ that lists every collector carries the run time too. */ Assert.Contains("{ key: \"run_at\", label: \"Run at\", hideWhenEmpty: true },", columns, StringComparison.Ordinal); Assert.Contains("{ key: \"next_run_utc\", label: \"Next run\", format: \"time\", hideWhenEmpty: true },", columns, StringComparison.Ordinal); - Assert.Contains("{ key: \"run_time_note\", label: \"Run time\", wrap: true, hideWhenEmpty: true },", columns, StringComparison.Ordinal); + Assert.Contains("{ key: \"run_time_note\", label: \"Run time\", clamp: true, hideWhenEmpty: true },", columns, StringComparison.Ordinal); Assert.True(columns.IndexOf("key: \"last_success\"", StringComparison.Ordinal) < columns.IndexOf("key: \"run_at\"", StringComparison.Ordinal)); } diff --git a/Darling/Darling.Tests/WebDataStartNoteEventReadsTests.cs b/Darling/Darling.Tests/WebDataStartNoteEventReadsTests.cs index 5bf6012a04..a530e7af14 100644 --- a/Darling/Darling.Tests/WebDataStartNoteEventReadsTests.cs +++ b/Darling/Darling.Tests/WebDataStartNoteEventReadsTests.cs @@ -213,7 +213,7 @@ await Assert.ThrowsAnyAsync( Assert.Equal(unavailable, await WebDataStartNote.AddAsync(store, read, "sql01", 48, WindowEnd, unavailable, null, Cancelled)); } - /// The system_health CPU line shares its read with the Entries table, and a chart draws no note: the page's note helper + /// The system_health CPU line shares its read with the Entries table, and a chart draws no data-start note: the page note helper /// returns null for any viz other than a table or a stat, so the listed read adds no strip above the line (#4966). [Fact] public void TheSystemHealthCpuLine_IsAChart_AndTheNoteHelperDrawsNothingForACharts() @@ -224,7 +224,9 @@ public void TheSystemHealthCpuLine_IsAChart_AndTheNoteHelperDrawsNothingForAChar Assert.Contains("viz: \"line\"", tabs.AsSpan(line, 200).ToString(), StringComparison.Ordinal); var util = ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js", "util.js"); - Assert.Contains("(desc.viz !== \"table\" && desc.viz !== \"stat\")) return null;", util, StringComparison.Ordinal); + Assert.Contains("if (desc.viz !== \"table\" && desc.viz !== \"stat\" && !chart) return null;", util, StringComparison.Ordinal); + // ...and a chart draws only the row-capped note that names the oldest row shown (#5644 W12), which the data-start note does not. + Assert.Contains("if (typeof source.oldest_shown_utc !== \"string\") return null;", util, StringComparison.Ordinal); } /// The four reads' premises, in the tools' source: the payload fields the rules read. diff --git a/Darling/Darling.Tests/WebDataStartNoteTests.cs b/Darling/Darling.Tests/WebDataStartNoteTests.cs index b4ae501994..42db38c900 100644 --- a/Darling/Darling.Tests/WebDataStartNoteTests.cs +++ b/Darling/Darling.Tests/WebDataStartNoteTests.cs @@ -354,8 +354,10 @@ public void AToolsOwnTruncationNote_OnAPanelOutsideTheList_NamesNoInstant_AndThe } var util = ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js", "util.js").ReplaceLineEndings("\n"); - Assert.Contains("if (!desc || desc.windowNote === false || (desc.viz !== \"table\" && desc.viz !== \"stat\")) return null;", util, StringComparison.Ordinal); - Assert.Contains("if (!source || source.window_truncated !== true) return null;", util, StringComparison.Ordinal); + Assert.Contains("if (!desc || desc.windowNote === false) return null;", util, StringComparison.Ordinal); + Assert.Contains("if (desc.viz !== \"table\" && desc.viz !== \"stat\" && !chart) return null;", util, StringComparison.Ordinal); + // A chart draws only the row-capped note (#5644 W12), and only when the note names the oldest row shown. + Assert.Contains("if (typeof source.oldest_shown_utc !== \"string\") return null;", util, StringComparison.Ordinal); Assert.Contains("return sent;\n}", util, StringComparison.Ordinal); } @@ -388,8 +390,10 @@ public void TheHeatmapAndRegressionsPanels_DrawTheToolsOwnTruncationNote_AsSent( Assert.Equal(baselineTail, note.EndsWith("baseline_end.", StringComparison.Ordinal)); var util = ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js", "util.js").ReplaceLineEndings("\n"); - Assert.Contains("if (!desc || desc.windowNote === false || (desc.viz !== \"table\" && desc.viz !== \"stat\")) return null;", util, StringComparison.Ordinal); - Assert.Contains("if (!source || source.window_truncated !== true) return null;", util, StringComparison.Ordinal); + Assert.Contains("if (!desc || desc.windowNote === false) return null;", util, StringComparison.Ordinal); + Assert.Contains("if (desc.viz !== \"table\" && desc.viz !== \"stat\" && !chart) return null;", util, StringComparison.Ordinal); + // A chart draws only the row-capped note (#5644 W12), and only when the note names the oldest row shown. + Assert.Contains("if (typeof source.oldest_shown_utc !== \"string\") return null;", util, StringComparison.Ordinal); Assert.Contains("return sent;\n}", util, StringComparison.Ordinal); } diff --git a/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs b/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs index 387c6d9291..fc58143cda 100644 --- a/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs +++ b/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs @@ -114,6 +114,9 @@ public void AnIsoInstantInASentence_PrintsInTheGridsLocalTimeFormat() Assert.Equal("stored at " + a + " and not collected since " + b + ".", S(r, "sentence")); Assert.Equal("last event " + a, S(r, "emptyStripText")); Assert.Equal("since " + b, S(r, "noticeStripText")); + // G1a: plainText (lane C, W8) and localizeIsoText (lane E, W20) both run in the strips. + Assert.Equal("TRUNCATED: since " + b + ": 9 indexes match and 5 were returned. Rows are ordered by total wait time descending.", S(r, "bothEmpty")); + Assert.Equal(S(r, "bothEmpty"), S(r, "bothNotice")); Assert.Equal("nothing to change 10/9/2026, 10:36:32 AM", S(r, "noStamp")); Assert.Equal("null", S(r, "nonString")); Assert.DoesNotContain("T", a); diff --git a/Darling/Darling.Tests/WebQueryStoreGridGroupsTests.cs b/Darling/Darling.Tests/WebQueryStoreGridGroupsTests.cs index 7f14defbc6..2c0d5ed119 100644 --- a/Darling/Darling.Tests/WebQueryStoreGridGroupsTests.cs +++ b/Darling/Darling.Tests/WebQueryStoreGridGroupsTests.cs @@ -90,7 +90,7 @@ public void EachGrid_StartsOnItsCoreColumns_WithATogglePerGroup() Assert.Equal(new[] { "Time", "Statement", "EventType", "Duration", "CPU", "Result", "Database", "Object", "QueryHash" }, Arr(r, "completions", "heads")); Assert.Equal(new[] { "Columns:", "Plans", "Plan metrics", "Lifecycle" }, Arr(r, "corrections", "toggles")); - Assert.Equal(new[] { "Collected", "Query", "Database", "State", "StateReason", "Reason", "Score", "Est.gain(s)", "QueryID" }, + Assert.Equal(new[] { "Firstseen", "Lastseen", "Captures", "Query", "Database", "State", "StateReason", "Reason", "Score", "Est.gain(s)", "QueryID" }, Arr(r, "corrections", "heads")); } @@ -126,9 +126,9 @@ public void AUtcInstant_PrintsInTheBrowsersLocalTime_InEveryTimestampColumn() var r = Run("localTime"); var expected = r.GetProperty("expected").GetString()!; Assert.Equal("1/15/2026, 12:30:00 PM", expected); - // Collected, Valid Since, Last Refresh and Executed At (Lifecycle is on in this scenario). + // First seen, Last seen, Valid Since, Last Refresh and Executed At (Lifecycle is on in this scenario; #5644 W21 put two stamps and Captures first). var cells = Arr(r, "cells"); - foreach (var i in new[] { 0, 9, 10, 12 }) Assert.Equal(expected, cells[i]); + foreach (var i in new[] { 0, 1, 11, 12, 14 }) Assert.Equal(expected, cells[i]); Assert.Equal(expected, Arr(r, "regressionCells").Last()); Assert.Equal(expected, Arr(r, "completionCells").First()); } diff --git a/Darling/Darling.Tests/WebWaitQueriesLinkBehaviourTests.cs b/Darling/Darling.Tests/WebWaitQueriesLinkBehaviourTests.cs index 5f16601c17..ebf31abaa4 100644 --- a/Darling/Darling.Tests/WebWaitQueriesLinkBehaviourTests.cs +++ b/Darling/Darling.Tests/WebWaitQueriesLinkBehaviourTests.cs @@ -163,7 +163,7 @@ public void TheFilteredGrid_ReadsWaitTypeAtLimit500_AndTheStripClearsIt() Assert.Equal("#/server/A/waits", r.GetProperty("hashAfterShowAll").GetString()); // A filtered miss names the wait: the server's envelope does, and so does the panel's own text. - Assert.Equal(new[] { "No active-query snapshots with wait_type 'WRITELOG' were collected in this window." }, Strings(r.GetProperty("emptyEnvelopeText"))); + Assert.Equal(new[] { "No active-query snapshots with wait type 'WRITELOG' were collected in this window." }, Strings(r.GetProperty("emptyEnvelopeText"))); Assert.Equal(new[] { "No active queries waiting on WRITELOG were captured in this window." }, Strings(r.GetProperty("emptyRowsText"))); } diff --git a/Darling/Darling.Tests/WebWordingFixesTests.cs b/Darling/Darling.Tests/WebWordingFixesTests.cs index bdc3c3f843..ac834060fd 100644 --- a/Darling/Darling.Tests/WebWordingFixesTests.cs +++ b/Darling/Darling.Tests/WebWordingFixesTests.cs @@ -42,7 +42,7 @@ public void TheNoticeStrips_AndTheTrendNotes_RunThroughPlainText() var tabs = ServerTabs(); /* W8: a payload's own note (aggregate_note, a truncation note) is written for an MCP caller. */ - Assert.Contains("[plainText(message)]", util, StringComparison.Ordinal); + Assert.Contains("[localizeIsoText(plainText(message))]", util, StringComparison.Ordinal); Assert.Contains("text: plainText(text)", util, StringComparison.Ordinal); Assert.DoesNotContain("class: \"mp-metric-note\", text: res.data.aggregate_note", tabs, StringComparison.Ordinal); Assert.DoesNotContain("class: \"mp-metric-note\", text: trend.data.aggregate_note", tabs, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/web-partial-window-harness.mjs b/Darling/Darling.Tests/web-partial-window-harness.mjs index fa91114298..cb232796a5 100644 --- a/Darling/Darling.Tests/web-partial-window-harness.mjs +++ b/Darling/Darling.Tests/web-partial-window-harness.mjs @@ -93,5 +93,9 @@ out.noStamp = localizeIsoText("nothing to change 10/9/2026, 10:36:32 AM"); out.nonString = String(localizeIsoText(null)); out.emptyStripText = emptyStrip("last event " + iso).textContent; out.noticeStripText = noticeStrip("since " + isoZ).textContent; +/* G1a: a caller-wording sentence that also carries an instant gets BOTH treatments (plainText rewrites the wording, then the instant goes local) */ +const both = "since " + isoZ + ": 9 indexes match and 5 were returned. Rows are ordered by total wait time descending; raise limit to see more."; +out.bothEmpty = emptyStrip("TRUNCATED: " + both).textContent; +out.bothNotice = noticeStrip("TRUNCATED: " + both).textContent; console.log(JSON.stringify(out)); diff --git a/Darling/Darling.Tests/web-qs-grids-harness.mjs b/Darling/Darling.Tests/web-qs-grids-harness.mjs index 22739553b4..0fb995d202 100644 --- a/Darling/Darling.Tests/web-qs-grids-harness.mjs +++ b/Darling/Darling.Tests/web-qs-grids-harness.mjs @@ -55,7 +55,7 @@ try { const UTC_STAMP = "2026-01-15T17:30:00.0000000"; const regressions = [{ severity: "High", database_name: "d1", query_text: "select 1", query_id: 7, additional_duration_ms: 5, baseline_cpu_ms: 11, recent_cpu_ms: 12, baseline_reads: 13, recent_reads: 14, baseline_exec_count: 15, recent_exec_count: 16, baseline_plan_count: 1, recent_plan_count: 2, last_execution_time: UTC_STAMP }]; const completions = [{ event_time: UTC_STAMP, statement: "select 2", event_type: "rpc", duration_ms: 9, logical_reads: 21, physical_reads: 22, writes: 23, row_count: 24, session_id: 55, client_app_name: "AppX", server_principal_name: "loginx", database_name: "d1" }]; -const recommendations = [{ collection_time: UTC_STAMP, query_text: "select 3", database_name: "d1", recommendation_state: "Active", score: 90, query_id: 7, regressed_plan_id: 101, last_good_plan_id: 100, last_good_plan_forcing_type: "AUTO", regressed_plan_execution_count: 31, valid_since: UTC_STAMP, last_refresh: UTC_STAMP, execute_action_initiated_by: "System", execute_action_initiated_time: UTC_STAMP }]; +const recommendations = [{ first_seen: UTC_STAMP, collection_time: UTC_STAMP, capture_count: 3, query_text: "select 3", database_name: "d1", recommendation_state: "Active", score: 90, query_id: 7, regressed_plan_id: 101, last_good_plan_id: 100, last_good_plan_forcing_type: "AUTO", regressed_plan_execution_count: 31, valid_since: UTC_STAMP, last_refresh: UTC_STAMP, execute_action_initiated_by: "System", execute_action_initiated_time: UTC_STAMP }]; answers = { get_query_store_regressions: { regressions }, get_long_query_completions: { completions }, diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js index 75ff7393f2..890eef8a54 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js @@ -1367,7 +1367,7 @@ export async function openServerTabAt(server, tMs, item, { beforeRoute = null, s if (typeof beforeRoute === "function") beforeRoute(); /* Bring the item's own grid into view once the router has built the tab. The router's hashchange listener was added first (app.js), and for a server already on screen renderServer paints the tab synchronously, so the new - panels are in place when this one runs. A chart and a grid share the title "Deadlocks"; the grid is the later one. + panels are in place when this one runs. The Blocking tab's chart is "Deadlock trend" (#5644 W16), so the grid is the one "Deadlocks" panel there. The listener is once-only, and added after the range is taken, so a refused range leaves none behind. */ if (typeof window.addEventListener === "function") { window.addEventListener( From 1b7c51f0a778341d7b79113962ce9b8b583c945b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:27:46 -0400 Subject: [PATCH 12/42] Pin the CPU ring-buffer dedup with its tolerance; run the DuckDB blocking-wait SQL against seeded reports (#5644 W1, W3) --- .../CollectorTimestampFrameTests.cs | 7 +- Lite.Tests/BlockingWaitDuckDbTests.cs | 185 ++++++++++++++++++ 2 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 Lite.Tests/BlockingWaitDuckDbTests.cs diff --git a/Darling/Darling.Tests/CollectorTimestampFrameTests.cs b/Darling/Darling.Tests/CollectorTimestampFrameTests.cs index cdd31bdc5e..696a225d0e 100644 --- a/Darling/Darling.Tests/CollectorTimestampFrameTests.cs +++ b/Darling/Darling.Tests/CollectorTimestampFrameTests.cs @@ -153,7 +153,12 @@ public void CpuUtilization_DedupsInTheWatermarksFrame_NotOnTheLocalStampAlone() var code = QueryTextOf("CpuUtilizationCollector.cs"); Assert.Contains("var stampInWatermarkFrame = context.WatermarkFromUtcColumn ? sampleTimeUtc : sampleTime;", code, StringComparison.Ordinal); - Assert.Contains("&& stampInWatermarkFrame <= context.Watermark.Value)", code, StringComparison.Ordinal); + /* #5644 W3: the watermark bound carries the ring-buffer resample tolerance (the record is re-read a few + milliseconds later); the compared column is still the one in the watermark's frame. The behaviour (8 ms + drift dropped, the next record 60 s later kept) is Lite.Tests CpuUtilizationCollectorDefinitionTests + ReadAsync_RingBuffer_AlreadyStoredRecordWithDriftedStamp_IsNotStoredAgain. */ + Assert.Contains("&& stampInWatermarkFrame <= WithRingBufferTolerance(context.Watermark.Value))", code, StringComparison.Ordinal); + Assert.DoesNotContain("stampInWatermarkFrame <= context.Watermark.Value", code, StringComparison.Ordinal); Assert.DoesNotContain("sampleTime <= context.Watermark.Value", code, StringComparison.Ordinal); Assert.Equal("sample_time_utc", PerformanceMonitor.Collectors.CpuUtilizationCollector.Instance.UtcWatermarkColumn); Assert.Equal("sample_time", PerformanceMonitor.Collectors.CpuUtilizationCollector.Instance.WatermarkColumn); diff --git a/Lite.Tests/BlockingWaitDuckDbTests.cs b/Lite.Tests/BlockingWaitDuckDbTests.cs new file mode 100644 index 0000000000..0815e90641 --- /dev/null +++ b/Lite.Tests/BlockingWaitDuckDbTests.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using DuckDB.NET.Data; +using PerformanceMonitor.Analysis; +using PerformanceMonitor.Common; +using PerformanceMonitorLite.Analysis; +using PerformanceMonitorLite.Database; +using PerformanceMonitorLite.Services; +using Xunit; + +namespace PerformanceMonitorLite.Tests; + +/// +/// #5644 W1: Lite's DuckDB form of the blocking-wait rule ( with +/// epoch_ms) run for real against seeded blocked-process reports, and checked against the C# rule +/// () over the very same rows. Two readers use the SQL: the analysis +/// BLOCKING_EVENTS fact and the overview card's last-hour count (which Lite's MCP summary reads). +/// One wait is one blocked session, blocker and wait start; SQL Server writes a report for a still-blocked +/// session every threshold seconds, so a one-minute wait is about eleven report rows. +/// +[Collection(SeparatelyMonitoredProviderCollection.Name)] +public class BlockingWaitDuckDbTests : IClassFixture, IDisposable +{ + private const int ServerId = -5644_11; + + private readonly DuckDbInitializer _duckDb; + private readonly DateTime _windowEnd = DateTime.UtcNow; + private readonly DateTime _windowStart; + private DuckDBConnection? _conn; + private long _nextId = -1; + private readonly List _seeded = new(); + + private sealed record Report(int Blocked, int Blocker, DateTime EventTime, long WaitMs); + + public BlockingWaitDuckDbTests(SharedDuckDbFixture fixture) + { + fixture.ResetData(); + _duckDb = fixture.DuckDb; + _windowStart = _windowEnd.AddHours(-4); + } + + public void Dispose() => _conn?.Dispose(); + + private async Task ExecAsync(string sql, params object?[] args) + { + using var readLock = _duckDb.AcquireReadLock(); + _conn ??= _duckDb.CreateConnection(); + if (_conn.State != System.Data.ConnectionState.Open) await _conn.OpenAsync(); + using var cmd = _conn.CreateCommand(); + cmd.CommandText = sql; + foreach (var arg in args) + cmd.Parameters.Add(new DuckDBParameter { Value = arg ?? DBNull.Value }); + await cmd.ExecuteNonQueryAsync(); + } + + /* Wait-stats deltas give the fact collector its observed time: four hours, a collection every 15 minutes. */ + private async Task SeedObservedWindowAsync() + { + for (var i = 0; i <= 16; i++) + await ExecAsync( + @"INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, + wait_time_ms, signal_wait_time_ms, waiting_tasks_count, + delta_wait_time_ms, delta_signal_wait_time_ms, delta_waiting_tasks) + VALUES ($1, $2, $3, 'TestServer', 'CXPACKET', 1000, 0, 1, $4, 0, $4)", + _nextId--, _windowStart.AddMinutes(15 * i), ServerId, i == 0 ? 0L : 1000L); + } + + /// + /// One wait as SQL Server reports it: rows, one per threshold interval, the k-th + /// at start + (k + 1) * interval carrying wait_time_ms = (k + 1) * interval. + /// moves each report's wait time off the exact value, alternately up and down, so consecutive derived starts differ by twice that. + /// + private async Task SeedWaitAsync( + int blocked, int blocker, DateTime start, int reports, int intervalSeconds = 5, int jitterMs = 0, bool dmv = false) + { + for (var k = 0; k < reports; k++) + { + var waitMs = ((k + 1) * intervalSeconds * 1000L) + (k % 2 == 0 ? jitterMs : -jitterMs); + var eventTime = start.AddSeconds((k + 1) * intervalSeconds); + _seeded.Add(new Report(blocked, blocker, eventTime, waitMs)); + if (dmv) + await ExecAsync( + "INSERT INTO dmv_blocking_snapshots (collection_id, collection_time, event_time, server_id, server_name, database_name, monitor_loop, wait_time_ms, blocked_spid, blocking_spid) VALUES ($1,$2,$2,$3,'TestServer','db',1,$4,$5,$6)", + _nextId--, eventTime, ServerId, waitMs, blocked, blocker); + else + await ExecAsync( + "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, event_time, server_id, server_name, database_name, wait_time_ms, blocked_spid, blocking_spid) VALUES ($1,$2,$2,$3,'TestServer','db',$4,$5,$6)", + _nextId--, eventTime, ServerId, waitMs, blocked, blocker); + } + } + + /* The C# rule over exactly the rows that were seeded. */ + private int CSharpWaitCount() => BlockingWaits.Collapse( + _seeded, _ => "xe", r => r.Blocked, r => r.Blocker, r => (DateTime?)r.EventTime, r => r.WaitMs).Count; + + private async Task BlockingFactAsync() + { + var context = new AnalysisContext + { + ServerId = ServerId, + ServerName = "TestServer", + TimeRangeStart = _windowStart, + TimeRangeEnd = _windowEnd, + }; + var facts = await new DuckDbFactCollector(_duckDb).CollectFactsAsync(context); + return facts.First(f => f.Key == "BLOCKING_EVENTS"); + } + + private async Task OverviewCountAsync() => + (await new LocalDataService(_duckDb).GetServerSummaryAsync(ServerId, "TestServer", registeredAtUtc: null))!.BlockingCount; + + /* Seed times sit 5-50 minutes back, inside both the analysis window and the overview's last hour. */ + private DateTime Ago(int minutes) => _windowEnd.AddMinutes(-minutes); + + private async Task AssertAllThreeAgreeAsync(int expectedWaits) + { + await SeedObservedWindowAsync(); + Assert.Equal(expectedWaits, CSharpWaitCount()); + Assert.Equal(expectedWaits, (int)(await BlockingFactAsync()).Metadata["event_count"]); + Assert.Equal(expectedWaits, await OverviewCountAsync()); + } + + [Fact] + public async Task SeveralReportsOfOneWait_CountOnce() + { + await SeedWaitAsync(52, 51, Ago(30), reports: 11); + + await AssertAllThreeAgreeAsync(1); + + /* The longest wait time is the one kept: the 11th report, 55 s. */ + Assert.Equal(55_000.0, (await BlockingFactAsync()).Metadata["max_wait_time_ms"]); + } + + [Fact] + public async Task TheAuditsElevenNineNineAndSixReports_AreFourWaits() + { + await SeedWaitAsync(52, 50, Ago(45), reports: 11); + await SeedWaitAsync(53, 50, Ago(40), reports: 9); + await SeedWaitAsync(54, 50, Ago(35), reports: 9); + await SeedWaitAsync(52, 50, Ago(20), reports: 6); /* the same pair again, a later wait */ + + await AssertAllThreeAgreeAsync(4); + } + + [Fact] + public async Task ReportsWhoseDerivedStartsJitterWithinTwoSeconds_AreOneWait() + { + await SeedWaitAsync(52, 51, Ago(30), reports: 9, jitterMs: 900); + + await AssertAllThreeAgreeAsync(1); + } + + [Fact] + public async Task ReportsWhoseDerivedStartsDriftMoreThanTwoSeconds_AreTwoWaits() + { + /* Same pair, the second wait starting 6 s after the first (the threshold is at least 5 s). */ + await SeedWaitAsync(52, 51, Ago(30), reports: 4); + await SeedWaitAsync(52, 51, Ago(30).AddSeconds(6), reports: 4); + + await AssertAllThreeAgreeAsync(2); + } + + [Fact] + public async Task OneBlockerWithTwoBlockedSessions_IsTwoWaits_AndTwoBlockersOneBlocked_IsTwoToo() + { + await SeedWaitAsync(52, 51, Ago(30), reports: 6); + await SeedWaitAsync(53, 51, Ago(30), reports: 6); + await SeedWaitAsync(52, 60, Ago(30), reports: 6); + + await AssertAllThreeAgreeAsync(3); + } + + [Fact] + public async Task TheDmvSnapshotFallback_CountsWaitsToo() + { + /* No blocked-process reports at all: the overview falls back to the always-on DMV snapshots. */ + await SeedWaitAsync(52, 51, Ago(30), reports: 11, dmv: true); + await SeedWaitAsync(53, 51, Ago(20), reports: 6, dmv: true); + + Assert.Equal(2, CSharpWaitCount()); + Assert.Equal(2, await OverviewCountAsync()); + } +} From 342f058015f17209ddcf7e0da75b59a8f0afcccb Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:33:47 -0400 Subject: [PATCH 13/42] Lite tests: seeded blocked-process rows are separate waits (#5644 W1) --- Lite.Tests/AzureMasterAnalysisScopeTests.cs | 4 ++-- Lite.Tests/AzureMasterOverviewCardScopeTests.cs | 8 ++++---- Lite.Tests/BlockingDeadlockContextBuilderTests.cs | 6 ++++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Lite.Tests/AzureMasterAnalysisScopeTests.cs b/Lite.Tests/AzureMasterAnalysisScopeTests.cs index a549048707..d48a4860a7 100644 --- a/Lite.Tests/AzureMasterAnalysisScopeTests.cs +++ b/Lite.Tests/AzureMasterAnalysisScopeTests.cs @@ -69,8 +69,8 @@ private async Task SeedBprAsync(int count, string? database, int offsetMinutes = { for (var i = 0; i < count; i++) await ExecAsync( - "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, event_time, server_id, server_name, database_name, wait_time_ms) VALUES ($1,$2,$2,$3,'TestServer',$4,1000)", - _nextId--, WindowStart.AddMinutes(30 + offsetMinutes + (i * 7)), ServerId, database); + "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, event_time, server_id, server_name, database_name, wait_time_ms, blocked_spid, blocking_spid) VALUES ($1,$2,$2,$3,'TestServer',$4,1000,$5,50)", + _nextId--, WindowStart.AddMinutes(30 + offsetMinutes + (i * 7)), ServerId, database, 1000 - (int)_nextId); /* a blocked session of its own per row: seeded rows are separate waits (#5644 W1) */ } private async Task SeedDmvAsync(int count, string? database) diff --git a/Lite.Tests/AzureMasterOverviewCardScopeTests.cs b/Lite.Tests/AzureMasterOverviewCardScopeTests.cs index 6b6ad445af..719160ef3f 100644 --- a/Lite.Tests/AzureMasterOverviewCardScopeTests.cs +++ b/Lite.Tests/AzureMasterOverviewCardScopeTests.cs @@ -53,8 +53,8 @@ private async Task SeedBprAsync(int count, string? database) { for (var i = 0; i < count; i++) await ExecAsync( - "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, event_time, server_id, server_name, database_name, wait_time_ms) VALUES ($1,$2,$2,$3,'TestServer',$4,1000)", - _nextId--, DateTime.UtcNow.AddMinutes(-10 - i), ServerId, database); + "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, event_time, server_id, server_name, database_name, wait_time_ms, blocked_spid, blocking_spid) VALUES ($1,$2,$2,$3,'TestServer',$4,1000,$5,50)", + _nextId--, DateTime.UtcNow.AddMinutes(-10 - i), ServerId, database, 1000 - (int)_nextId); /* a blocked session of its own per row: seeded rows are separate waits (#5644 W1) */ } private static string Graph(params string[] databases) => @@ -74,8 +74,8 @@ private async Task SeedDmvAsync(int count, string? database) { for (var i = 0; i < count; i++) await ExecAsync( - "INSERT INTO dmv_blocking_snapshots (collection_id, collection_time, event_time, server_id, server_name, database_name, monitor_loop, wait_time_ms) VALUES ($1,$2,$2,$3,'TestServer',$4,1,1000)", - _nextId--, DateTime.UtcNow.AddMinutes(-10 - i), ServerId, database); + "INSERT INTO dmv_blocking_snapshots (collection_id, collection_time, event_time, server_id, server_name, database_name, monitor_loop, wait_time_ms, blocked_spid, blocking_spid) VALUES ($1,$2,$2,$3,'TestServer',$4,1,1000,$5,50)", + _nextId--, DateTime.UtcNow.AddMinutes(-10 - i), ServerId, database, 1000 - (int)_nextId); } private async Task SeedDeadlockRowAsync(string rowDatabase, params string[] graphDatabases) => diff --git a/Lite.Tests/BlockingDeadlockContextBuilderTests.cs b/Lite.Tests/BlockingDeadlockContextBuilderTests.cs index c33cf5c754..966fbea2f2 100644 --- a/Lite.Tests/BlockingDeadlockContextBuilderTests.cs +++ b/Lite.Tests/BlockingDeadlockContextBuilderTests.cs @@ -39,11 +39,17 @@ public class BlockingDeadlockContextBuilderTests /* ---------------- blocking builder ---------------- */ + /* #5644 W1: reports of one blocked session, blocker and wait start are ONE wait, so a row that stands for a + separate wait of its own gets a blocked session of its own by default. */ + private static int s_nextBlockedSpid = 100; + private static BlockedProcessAlertRow Blocked( string db = "StackOverflow", string obj = "", string blockedSql = "", string blockingSql = "", long waitMs = 5000, string lockMode = "", string xml = "", DateTime? eventTime = null) => new() { EventTime = eventTime ?? new DateTime(2026, 6, 19, 12, 0, 0), + BlockedSpid = System.Threading.Interlocked.Increment(ref s_nextBlockedSpid), + BlockingSpid = 50, DatabaseName = db, ContentiousObject = obj, BlockedSqlText = blockedSql, From ede1df903fe0244b6dd52d6aab493f544a5f900c Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:37:15 -0400 Subject: [PATCH 14/42] Fix the doubled comma in the Version Store collector's on-prem query (#5644 W5) --- PerformanceMonitor.Collectors/PvsStatsCollector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PerformanceMonitor.Collectors/PvsStatsCollector.cs b/PerformanceMonitor.Collectors/PvsStatsCollector.cs index 0b551c9f16..962cb32541 100644 --- a/PerformanceMonitor.Collectors/PvsStatsCollector.cs +++ b/PerformanceMonitor.Collectors/PvsStatsCollector.cs @@ -205,7 +205,7 @@ public readonly record struct Row( online_index_version_store_size_mb = CONVERT(decimal(19,2), pvss.online_index_version_store_size_kb / 1024.0), database_data_size_mb = - CONVERT(decimal(19,2), df.total_db_size_kb / 1024.0),, + CONVERT(decimal(19,2), df.total_db_size_kb / 1024.0), current_aborted_transaction_count = pvss.current_aborted_transaction_count, oldest_active_transaction_id = From 87fec7a381c0c11c59ad2c041921201b9a17065e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:37:57 -0400 Subject: [PATCH 15/42] G1a: relabel plan cache (all cache stores); name the server clock in the restart banner start times (#5644) --- Darling/Darling.Tests/WebAuditLaneDLabelTests.cs | 2 +- .../Darling.Tests/WebPartialWindowBehaviourTests.cs | 2 ++ Darling/Darling.Tests/web-partial-window-harness.mjs | 3 +++ .../wwwroot/js/pages/server-tabs.js | 2 +- Lite.Tests/BaselineDiscontinuityTests.cs | 8 ++++++-- PerformanceMonitor.Collectors/ServerEpoch.cs | 10 ++++++++-- 6 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs b/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs index ad51d2d750..1daec931d3 100644 --- a/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs +++ b/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs @@ -56,7 +56,7 @@ public void W19_EachMisleadingLabelNamesItsValue() var tabs = Js("pages", "server-tabs.js"); Assert.Contains("{ key: \"hyperthread_ratio\", label: \"Logical CPUs per socket\"", tabs, StringComparison.Ordinal); Assert.DoesNotContain("label: \"HT ratio\"", tabs, StringComparison.Ordinal); - Assert.Contains("{ key: \"plan_cache_mb\", label: \"Plan cache (memory clerk)\", format: \"mb\" },", tabs, StringComparison.Ordinal); + Assert.Contains("{ key: \"plan_cache_mb\", label: \"Plan cache (all cache stores)\", format: \"mb\" },", tabs, StringComparison.Ordinal); Assert.Contains("{ key: \"summary.total_size_mb\", label: \"Cached plans size\", format: \"mb\" },", tabs, StringComparison.Ordinal); Assert.Contains("{ key: \"pct_of_sweep_budget_per_run\", label: \"% of run budget\"", tabs, StringComparison.Ordinal); /* fmtMs prints seconds above a second ("32.9 s"), so the header carries no unit. */ diff --git a/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs b/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs index fc58143cda..fefb61d98e 100644 --- a/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs +++ b/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs @@ -117,6 +117,8 @@ public void AnIsoInstantInASentence_PrintsInTheGridsLocalTimeFormat() // G1a: plainText (lane C, W8) and localizeIsoText (lane E, W20) both run in the strips. Assert.Equal("TRUNCATED: since " + b + ": 9 indexes match and 5 were returned. Rows are ordered by total wait time descending.", S(r, "bothEmpty")); Assert.Equal(S(r, "bothEmpty"), S(r, "bothNotice")); + // The restart banner's start-time pair is the server's own clock (not UTC): the strip leaves it as written, never shifted. + Assert.Equal(S(r, "bannerText"), S(r, "bannerNotice")); Assert.Equal("nothing to change 10/9/2026, 10:36:32 AM", S(r, "noStamp")); Assert.Equal("null", S(r, "nonString")); Assert.DoesNotContain("T", a); diff --git a/Darling/Darling.Tests/web-partial-window-harness.mjs b/Darling/Darling.Tests/web-partial-window-harness.mjs index cb232796a5..0dc4490424 100644 --- a/Darling/Darling.Tests/web-partial-window-harness.mjs +++ b/Darling/Darling.Tests/web-partial-window-harness.mjs @@ -97,5 +97,8 @@ out.noticeStripText = noticeStrip("since " + isoZ).textContent; const both = "since " + isoZ + ": 9 indexes match and 5 were returned. Rows are ordered by total wait time descending; raise limit to see more."; out.bothEmpty = emptyStrip("TRUNCATED: " + both).textContent; out.bothNotice = noticeStrip("TRUNCATED: " + both).textContent; +/* G1a: the restart banner names the server clock for a SQL Server start-time pair (the pair is not UTC, so no "Z" and no shift) */ +out.bannerText = "baseline discontinuity at 10/12/2026, 8:15:00 AM (restart) — instance identity changed — start time 2026-10-09 14:00:00 → 2026-10-12 08:15:00 (server clock); every delta baseline for this server was forgotten"; +out.bannerNotice = noticeStrip(out.bannerText).textContent; console.log(JSON.stringify(out)); diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js index 25632c74c3..19e3739094 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js @@ -3034,7 +3034,7 @@ const MEMORY_STATS = [ { key: "total_server_memory_mb", label: "Total server", format: "mb" }, { key: "target_server_memory_mb", label: "Target server", format: "mb" }, { key: "buffer_pool_mb", label: "Buffer pool", format: "mb" }, - { key: "plan_cache_mb", label: "Plan cache (memory clerk)", format: "mb" }, + { key: "plan_cache_mb", label: "Plan cache (all cache stores)", format: "mb" }, { key: "system_memory_state", label: "System state", format: "text", small: true, nullKey: "system_memory_state_note" }, { key: "sql_memory_model", label: "Memory model", format: "text", small: true }, ]; diff --git a/Lite.Tests/BaselineDiscontinuityTests.cs b/Lite.Tests/BaselineDiscontinuityTests.cs index 68026d3264..dc2bdd3a1e 100644 --- a/Lite.Tests/BaselineDiscontinuityTests.cs +++ b/Lite.Tests/BaselineDiscontinuityTests.cs @@ -127,7 +127,7 @@ public void Compose_NamesOnlyTheSidesThatChanged() var rows = new[] { Marker(T0, "wait_stats", "identity_epoch_changes=1") }; var restart = Assert.Single(BaselineDiscontinuities.Compose(rows, Pair("wait_stats", T0, new(oldStart, "AG1"), new(newStart, "ag1")))).Detail; - Assert.Contains("start time 2026-10-08 00:10:55 → 2026-10-09 00:10:55", restart, StringComparison.Ordinal); + Assert.Contains("start time 2026-10-08 00:10:55 → 2026-10-09 00:10:55 (server clock)", restart, StringComparison.Ordinal); Assert.DoesNotContain("server name", restart, StringComparison.Ordinal); var rename = Assert.Single(BaselineDiscontinuities.Compose(rows, Pair("wait_stats", T0, new(oldStart, "OLD"), new(oldStart, "NEW")))).Detail; @@ -135,7 +135,11 @@ public void Compose_NamesOnlyTheSidesThatChanged() Assert.DoesNotContain("start time", rename, StringComparison.Ordinal); var failover = Assert.Single(BaselineDiscontinuities.Compose(rows, Pair("wait_stats", T0, new(oldStart, "OLD"), new(newStart, "NEW")))).Detail; - Assert.Contains("start time 2026-10-08 00:10:55 → 2026-10-09 00:10:55, server name OLD → NEW", failover, StringComparison.Ordinal); + Assert.Contains("start time 2026-10-08 00:10:55 → 2026-10-09 00:10:55 (server clock), server name OLD → NEW", failover, StringComparison.Ordinal); + // #5644 W20: the pair is the target's local clock, so the banner names it and never writes an ISO instant with a "Z" + // (the web would shift a "Z" stamp into the browser's zone and show a start time that never happened). + Assert.DoesNotMatch(@"\d{4}-\d\d-\d\dT|\dZ", restart); + Assert.DoesNotMatch(@"\d{4}-\d\d-\d\dT|\dZ", failover); } [Fact] diff --git a/PerformanceMonitor.Collectors/ServerEpoch.cs b/PerformanceMonitor.Collectors/ServerEpoch.cs index 287204b208..25c023c86f 100644 --- a/PerformanceMonitor.Collectors/ServerEpoch.cs +++ b/PerformanceMonitor.Collectors/ServerEpoch.cs @@ -467,6 +467,12 @@ private static void Persist(CollectorContext context, string stateKey, Stamp? pr } } + /// Said after a SQL Server start-time pair (#5644 W20 leftover). sqlserver_start_time is the target's LOCAL clock + /// and the host does not know its offset, so the pair is not UTC and must not be written as an instant with a "Z": the web's + /// localizeIsoText would shift it by the server's UTC offset and show a start time that never happened. The banner names the clock + /// instead, so a reader does not take the pair for the browser's zone or for UTC. + internal const string ServerClockNote = " (server clock)"; + /// /// The sides of an identity change that actually moved, for a banner or a log line (#5644 W17): "start time A → B" /// and/or "server name X → Y", joined by a comma. A side that did not change is left out, because listing @@ -480,7 +486,7 @@ public static string DescribeChange(Stamp replaced, Stamp current) if (replaced.StartTime.HasValue && current.StartTime.HasValue && replaced.StartTime.Value != current.StartTime.Value) { - parts.Add("start time " + Describe(replaced.StartTime) + " → " + Describe(current.StartTime)); + parts.Add("start time " + Describe(replaced.StartTime) + " → " + Describe(current.StartTime) + ServerClockNote); } if (replaced.Name is not null && current.Name is not null @@ -491,7 +497,7 @@ public static string DescribeChange(Stamp replaced, Stamp current) if (parts.Count == 0) { - parts.Add("start time " + Describe(replaced.StartTime) + " → " + Describe(current.StartTime)); + parts.Add("start time " + Describe(replaced.StartTime) + " → " + Describe(current.StartTime) + ServerClockNote); parts.Add("server name " + Describe(replaced.Name) + " → " + Describe(current.Name)); } From c36ef944d9e30ee180e74eb6e1fc1a413fbf9a4e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:48:45 -0400 Subject: [PATCH 16/42] Guard the Version Store collector's query text against syntax errors: lexical check always, live SET PARSEONLY when DARLING_TEST_SQL is set (#5644 W5) --- Lite.Tests/PvsStatsCollectorSyntaxTests.cs | 141 +++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 Lite.Tests/PvsStatsCollectorSyntaxTests.cs diff --git a/Lite.Tests/PvsStatsCollectorSyntaxTests.cs b/Lite.Tests/PvsStatsCollectorSyntaxTests.cs new file mode 100644 index 0000000000..4f5583d941 --- /dev/null +++ b/Lite.Tests/PvsStatsCollectorSyntaxTests.cs @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Lite.Tests.Helpers; +using Microsoft.Data.SqlClient; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// #5644 W5: the on-prem Version Store query shipped with ...df.total_db_size_kb / 1024.0),, and every +/// on-prem run failed with "Incorrect syntax near ','". The definition tests only used Contains, so +/// nothing parsed the text. Two guards: a lexical check that always runs (empty select-list elements in any +/// variant the collector builds), and a live SET PARSEONLY ON parse against a real server when +/// DARLING_TEST_SQL is set (the same variable the Darling live E2E uses). +/// +public sealed class PvsStatsCollectorSyntaxTests +{ + private static readonly RecordingCollectorDeltaCalculator s_deltas = new(); + + public static IEnumerable Variants() + { + yield return new object[] { "onprem-2019", 15, false, false }; + yield return new object[] { "onprem-2022", 16, false, false }; + yield return new object[] { "onprem-2025", 17, false, false }; + yield return new object[] { "managed-instance", 16, false, true }; + yield return new object[] { "azure-sql-db", 16, true, false }; + } + + private static CollectorQuery BuildQuery(int major, bool azureSqlDb, bool managedInstance) + { + var context = new CollectorContext + { + ServerId = 42, + ServerName = "test-server", + CollectionTime = new DateTime(2026, 8, 1, 12, 0, 0, DateTimeKind.Utc), + Deltas = s_deltas, + Target = new CollectorTargetInfo + { + SqlMajorVersion = major, + IsAzureSqlDb = azureSqlDb, + IsAzureManagedInstance = managedInstance, + }, + /* A non-empty exclusion list so the on-prem exclusion clause is part of the text too. */ + ExcludedDatabases = new[] { "skipme" }, + }; + return PvsStatsCollector.Instance.BuildQuery(context); + } + + /// Drops comments and string literals so the lexical checks see only code. + private static string StripCommentsAndStrings(string sql) + { + string noBlock = Regex.Replace(sql, @"/\*.*?\*/", " ", RegexOptions.Singleline); + string noLine = Regex.Replace(noBlock, @"--[^\r\n]*", " "); + return Regex.Replace(noLine, @"'(?:[^']|'')*'", "''"); + } + + [Theory] + [MemberData(nameof(Variants))] + public void Query_HasNoEmptyListElement(string variant, int major, bool azureSqlDb, bool managedInstance) + { + string code = StripCommentsAndStrings(BuildQuery(major, azureSqlDb, managedInstance).Text); + + Assert.False(Regex.IsMatch(code, @",\s*,"), $"{variant}: doubled comma in the query text"); + Assert.False(Regex.IsMatch(code, @"\(\s*,"), $"{variant}: comma straight after an opening parenthesis"); + Assert.False(Regex.IsMatch(code, @",\s*\)"), $"{variant}: comma straight before a closing parenthesis"); + Assert.False( + Regex.IsMatch(code, @",\s*(FROM|WHERE|GROUP\s+BY|ORDER\s+BY|OPTION|UNION)\b", RegexOptions.IgnoreCase), + $"{variant}: trailing comma before a clause keyword"); + Assert.False( + Regex.IsMatch(code, @"\bSELECT\s*,", RegexOptions.IgnoreCase), + $"{variant}: comma straight after SELECT"); + } + + [Theory] + [MemberData(nameof(Variants))] + public async Task Query_ParsesOnALiveServer(string variant, int major, bool azureSqlDb, bool managedInstance) + { + string? host = Environment.GetEnvironmentVariable("DARLING_TEST_SQL"); + Assert.SkipWhen(string.IsNullOrEmpty(host), "Set DARLING_TEST_SQL (and _USER/_PASSWORD for sql auth) to parse the query on a live server."); + + var builder = new SqlConnectionStringBuilder + { + DataSource = host!, + InitialCatalog = "master", + TrustServerCertificate = true, + Encrypt = SqlConnectionEncryptOption.Optional, + }; + string? user = Environment.GetEnvironmentVariable("DARLING_TEST_SQL_USER"); + if (string.IsNullOrEmpty(user)) + { + builder.IntegratedSecurity = true; + } + else + { + builder.UserID = user; + builder.Password = Environment.GetEnvironmentVariable("DARLING_TEST_SQL_PASSWORD") ?? ""; + } + + var query = BuildQuery(major, azureSqlDb, managedInstance); + /* The collector binds its exclusion parameters at run time; declare them so the parse resolves the names. */ + string declares = string.Concat(query.Parameters.Select(p => $"DECLARE {p.Name} nvarchar(256); ")); + string text = declares + query.Text; + var ct = TestContext.Current.CancellationToken; + await using var connection = new SqlConnection(builder.ConnectionString); + await connection.OpenAsync(ct); + + /* PARSEONLY checks syntax only and runs nothing, so it is safe on a server that holds real data and it + works on versions that lack the DMVs (the parse of 2022-only columns does not need them to exist). */ + await using (var on = new SqlCommand("SET PARSEONLY ON;", connection)) + { + await on.ExecuteNonQueryAsync(ct); + } + + try + { + await using var parse = new SqlCommand(text, connection); + await parse.ExecuteNonQueryAsync(ct); + } + catch (SqlException ex) + { + Assert.Fail($"{variant}: the query does not parse: {ex.Message}"); + } + finally + { + await using var off = new SqlCommand("SET PARSEONLY OFF;", connection); + await off.ExecuteNonQueryAsync(ct); + } + } +} From 5382d828be3c0151bf7221485b1fae32be49912b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:54:52 -0400 Subject: [PATCH 17/42] Choose XE or DMV blocking waits before the excluded-database filter; keep reused plan-correction names per query (#5644 W1, W21) --- .../Darling.Tests/BlockingWaitCountTests.cs | 78 +++++++++++++++++++ .../DarlingMcpPlanCorrectionToolsTests.cs | 12 +-- .../PlanCorrectionCollapseLiveTests.cs | 77 ++++++++++++++++++ .../Mcp/DarlingPlanCorrectionReader.cs | 11 +-- .../BlockingDeadlockContextBuilderTests.cs | 19 +++-- Lite.Tests/LiteAlertForwardingTests.cs | 48 ++++++++++++ .../AlertContextBuilders.cs | 15 +++- PerformanceMonitor.Alerting/AlertEngine.cs | 8 +- 8 files changed, 243 insertions(+), 25 deletions(-) diff --git a/Darling/Darling.Tests/BlockingWaitCountTests.cs b/Darling/Darling.Tests/BlockingWaitCountTests.cs index 1024eba566..ebab4f63c0 100644 --- a/Darling/Darling.Tests/BlockingWaitCountTests.cs +++ b/Darling/Darling.Tests/BlockingWaitCountTests.cs @@ -179,6 +179,84 @@ public async Task Engine_FiresOnTheWaitCount_AndTheValueMatchesTheCard() Assert.Contains("55", waitRange, StringComparison.Ordinal); } + /* ---- #5644 review (W1/W2): the XE-or-DMV choice is made BEFORE the excluded-database filter ---- */ + + private static List InDb(List rows, string db) + { + foreach (var r in rows) r.DatabaseName = db; + return rows; + } + + private static readonly string[] ExcludedOnly = { "excl" }; + + [Fact] + public void XeReportsOnlyInAnExcludedDatabase_AreNotReplacedByDmvWaitsElsewhere() + { + var rows = InDb(OneWait(102, 100, T0, 5), "excl"); + rows.AddRange(InDb(OneWait(202, 200, T0.AddMinutes(5), 3, source: BlockedProcessAlertRow.DmvSnapshotSource), "other")); + + Assert.Empty(AlertContextBuilders.BlockingWaitRows(rows, ExcludedOnly)); + Assert.Null(AlertContextBuilders.BuildBlockingContext("srv", rows, ExcludedOnly)); + } + + [Fact] + public void XeReportsInANonExcludedDatabase_AreTheSource_AndDmvWaitsAreIgnored() + { + var rows = InDb(OneWait(102, 100, T0, 5), "other"); + rows.AddRange(InDb(OneWait(202, 200, T0.AddMinutes(5), 3, source: BlockedProcessAlertRow.DmvSnapshotSource), "other")); + rows.AddRange(InDb(OneWait(302, 300, T0.AddMinutes(9), 3), "excl")); + + var counted = AlertContextBuilders.BlockingWaitRows(rows, ExcludedOnly); + + var wait = Assert.Single(counted); + Assert.Equal(BlockedProcessAlertRow.XeReportSource, wait.Source); + Assert.Equal(102, wait.BlockedSpid); + } + + [Fact] + public void NoXeReportsAtAll_CountsTheDmvWaits_OutsideTheExcludedDatabases() + { + var rows = InDb(OneWait(102, 100, T0, 1, source: BlockedProcessAlertRow.DmvSnapshotSource), "other"); + rows.AddRange(InDb(OneWait(202, 200, T0.AddMinutes(5), 1, source: BlockedProcessAlertRow.DmvSnapshotSource), "excl")); + + var wait = Assert.Single(AlertContextBuilders.BlockingWaitRows(rows, ExcludedOnly)); + Assert.Equal(BlockedProcessAlertRow.DmvSnapshotSource, wait.Source); + Assert.Equal(102, wait.BlockedSpid); + } + + [Fact] + public async Task Engine_DoesNotFire_WhenTheOnlyXeReportsAreExcluded_AndDmvWaitsSitElsewhere() + { + var h = new AlertEngineTests.Harness(); + h.Settings.BlockingEnabled = true; + h.Settings.BlockingCountThreshold = 1; + h.Settings.ExcludedDatabasesList.Add("excl"); + var engine = h.Build(); + + h.Adapter.Blocking.AddRange(InDb(OneWait(102, 100, T0, 5), "excl")); + h.Adapter.Blocking.AddRange(InDb(OneWait(202, 200, T0.AddMinutes(5), 3, source: BlockedProcessAlertRow.DmvSnapshotSource), "other")); + await engine.EvaluateServerAsync(AlertEngineTests.Harness.Snapshot()); + + Assert.Empty(h.Deliverer.Outcomes); + } + + [Fact] + public async Task Engine_Fires_OnXeWaits_WhenXeIsInANonExcludedDatabase() + { + var h = new AlertEngineTests.Harness(); + h.Settings.BlockingEnabled = true; + h.Settings.BlockingCountThreshold = 1; + h.Settings.ExcludedDatabasesList.Add("excl"); + var engine = h.Build(); + + h.Adapter.Blocking.AddRange(InDb(OneWait(102, 100, T0, 5), "other")); + h.Adapter.Blocking.AddRange(InDb(OneWait(202, 200, T0.AddMinutes(5), 3, source: BlockedProcessAlertRow.DmvSnapshotSource), "other")); + await engine.EvaluateServerAsync(AlertEngineTests.Harness.Snapshot()); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("1", fired.CurrentValue); + } + /* ---- the SQL form of the same rule (PostgreSQL): the fleet tile, the analysis fact and the summaries run it ---- */ private static string ValuesSource(IReadOnlyList rows, int server) => diff --git a/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs b/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs index 2e90667af9..4461f2b9ee 100644 --- a/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs @@ -101,12 +101,12 @@ public void PlanCorrectionsCollapsedSql_OneRowPerRecommendation_NewestCapture_Wi { var sql = DarlingPlanCorrectionReader.PlanCorrectionsCollapsedSql; - SqlTextPin.AssertExpresses("PARTITION BY database_name, recommendation_name ORDER BY collection_time DESC", sql, - "the newest capture of each recommendation is no longer the one row kept"); - SqlTextPin.AssertExpresses("MIN(collection_time) OVER (PARTITION BY database_name, recommendation_name) AS first_seen", sql, - "the recommendation's first capture is no longer reported"); - SqlTextPin.AssertExpresses("COUNT(*) OVER (PARTITION BY database_name, recommendation_name) AS capture_count", sql, - "the capture count is no longer reported"); + SqlTextPin.AssertExpresses("PARTITION BY database_name, recommendation_name, query_id ORDER BY collection_time DESC, score DESC NULLS LAST, regressed_plan_id DESC NULLS LAST, last_refresh DESC NULLS LAST", sql, + "the newest capture of each recommendation is no longer the one row kept, or a collection_time tie no longer resolves deterministically"); + SqlTextPin.AssertExpresses("MIN(collection_time) OVER (PARTITION BY database_name, recommendation_name, query_id) AS first_seen", sql, + "the recommendation's first capture is no longer reported, or two queries sharing a recommendation name collapse into one row"); + SqlTextPin.AssertExpresses("COUNT(*) OVER (PARTITION BY database_name, recommendation_name, query_id) AS capture_count", sql, + "the capture count is no longer reported, or two queries sharing a recommendation name collapse into one row"); SqlTextPin.AssertExpresses("WHERE newest_rank = 1", sql, "every capture, not one per recommendation, comes back"); SqlTextPin.AssertExpresses("recommendation_name IS NOT NULL", sql, "enablement-only rows come back as phantom recommendations"); SqlTextPin.AssertExpresses("WHERE server_id = $1", sql, "the read is not scoped to the requested server"); diff --git a/Darling/Darling.Tests/PlanCorrectionCollapseLiveTests.cs b/Darling/Darling.Tests/PlanCorrectionCollapseLiveTests.cs index 5f88b3f3b0..4bcfc2c88f 100644 --- a/Darling/Darling.Tests/PlanCorrectionCollapseLiveTests.cs +++ b/Darling/Darling.Tests/PlanCorrectionCollapseLiveTests.cs @@ -115,6 +115,83 @@ await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) = } } + /// + /// #5644 W21 review: the engine can reuse a recommendation name (PR_1) for a different query after a restart. + /// Two recommendations that share a name but not a query stay two rows, each with its own capture count, and two + /// captures of one recommendation at the identical instant resolve to the higher-scored row on every run. + /// + [Fact] + public async Task ReusedRecommendationName_ForADifferentQuery_StaysASeparateRow_AndATieIsDeterministic() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live collapse test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(cs!); + var bodySucceeded = false; + + try + { + var newest = DateTime.SpecifyKind( + new DateTime(DateTime.UtcNow.AddMinutes(-5).Ticks / TimeSpan.TicksPerSecond * TimeSpan.TicksPerSecond), DateTimeKind.Unspecified); + + /* Query 11 under PR_1 three times; the same name for query 22 twice, the later two inserted at one instant + with different scores (the lower score is inserted LAST so insertion order cannot decide the winner). */ + await PlantRawAsync(connection, 1, newest.AddMinutes(-10), "PR_1", 11, "A", 50, ct); + await PlantRawAsync(connection, 2, newest.AddMinutes(-5), "PR_1", 11, "A", 50, ct); + await PlantRawAsync(connection, 3, newest, "PR_1", 11, "A", 50, ct); + await PlantRawAsync(connection, 4, newest.AddMinutes(-5), "PR_1", 22, "B", 60, ct); + await PlantRawAsync(connection, 5, newest, "PR_1", 22, "B-high", 90, ct); + await PlantRawAsync(connection, 6, newest, "PR_1", 22, "B-low", 10, ct); + + var collapsed = await DarlingPlanCorrectionReader.GetCollapsedPlanCorrectionsAsync( + postgres, ServerId, newest.AddHours(-1), newest.AddHours(1), cap: 50, cancellationToken: ct); + + Assert.Equal(2, collapsed.Count); + var first = Assert.Single(collapsed, r => r.QueryId == 11); + Assert.Equal(3, first.CaptureCount); + Assert.Equal(newest.AddMinutes(-10), first.FirstSeen); + var second = Assert.Single(collapsed, r => r.QueryId == 22); + Assert.Equal(3, second.CaptureCount); + Assert.Equal(newest.AddMinutes(-5), second.FirstSeen); + Assert.Equal("B-high", second.RecommendationState); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + private static async Task PlantRawAsync( + NpgsqlConnection connection, long collectionId, DateTime collectionTime, string recommendationName, long queryId, string state, int score, CancellationToken ct) + { + using var command = new NpgsqlCommand( + """ + INSERT INTO plan_correction + (collection_id, collection_time, server_id, server_name, database_name, recommendation_name, + recommendation_state, query_id, score) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + """, connection); + command.Parameters.AddWithValue(900 + collectionId); + command.Parameters.AddWithValue(collectionTime); + command.Parameters.AddWithValue(ServerId); + command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue("collapse_probe_db"); + command.Parameters.AddWithValue(recommendationName); + command.Parameters.AddWithValue(state); + command.Parameters.AddWithValue(queryId); + command.Parameters.AddWithValue(score); + await command.ExecuteNonQueryAsync(ct); + } + private static async Task PlantAsync(NpgsqlConnection connection, DateTime collectionTime, int recommendation, int capture, CancellationToken ct) { using var command = new NpgsqlCommand( diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs index a01a2d9162..c811d295bd 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs @@ -92,7 +92,8 @@ AND recommendation_name IS NOT NULL /// /// #5644 W21: collapsed to ONE row per recommendation (database + the engine's - /// recommendation_name). The collector re-captures every open recommendation on every cycle, so the per-capture + /// recommendation_name + the query_id it targets, because the engine can reuse a name for a different query + /// after a restart). The collector re-captures every open recommendation on every cycle, so the per-capture /// read above fills a 50-row page with three recommendations seen seventeen times each. This read returns each /// recommendation's NEWEST capture (so collection_time is its last-seen time and the state is its latest), with /// first_seen (its oldest capture in the window) and capture_count (captures in the window) as the two @@ -126,9 +127,9 @@ WITH captured AS ( execute_action_initiated_time, revert_action_initiated_by, revert_action_initiated_time, - ROW_NUMBER() OVER (PARTITION BY database_name, recommendation_name ORDER BY collection_time DESC) AS newest_rank, - MIN(collection_time) OVER (PARTITION BY database_name, recommendation_name) AS first_seen, - COUNT(*) OVER (PARTITION BY database_name, recommendation_name) AS capture_count + ROW_NUMBER() OVER (PARTITION BY database_name, recommendation_name, query_id ORDER BY collection_time DESC, score DESC NULLS LAST, regressed_plan_id DESC NULLS LAST, last_refresh DESC NULLS LAST) AS newest_rank, + MIN(collection_time) OVER (PARTITION BY database_name, recommendation_name, query_id) AS first_seen, + COUNT(*) OVER (PARTITION BY database_name, recommendation_name, query_id) AS capture_count FROM plan_correction WHERE server_id = $1 AND collection_time >= $2 @@ -165,7 +166,7 @@ AND recommendation_name IS NOT NULL capture_count FROM captured WHERE newest_rank = 1 -ORDER BY collection_time DESC, score DESC +ORDER BY collection_time DESC, score DESC, database_name, query_id LIMIT $4"; /// diff --git a/Lite.Tests/BlockingDeadlockContextBuilderTests.cs b/Lite.Tests/BlockingDeadlockContextBuilderTests.cs index c33cf5c754..32b90c5e3d 100644 --- a/Lite.Tests/BlockingDeadlockContextBuilderTests.cs +++ b/Lite.Tests/BlockingDeadlockContextBuilderTests.cs @@ -41,8 +41,11 @@ public class BlockingDeadlockContextBuilderTests private static BlockedProcessAlertRow Blocked( string db = "StackOverflow", string obj = "", string blockedSql = "", string blockingSql = "", - long waitMs = 5000, string lockMode = "", string xml = "", DateTime? eventTime = null) => new() + long waitMs = 5000, string lockMode = "", string xml = "", DateTime? eventTime = null, int spid = 0) => new() { + /* #5644 W1: rows with one blocked spid, blocker and start are ONE wait; a test that wants separate + waits gives each its own spid. */ + BlockedSpid = spid, EventTime = eventTime ?? new DateTime(2026, 6, 19, 12, 0, 0), DatabaseName = db, ContentiousObject = obj, @@ -111,7 +114,7 @@ public void BuildBlockingContext_CapsAtTenGroups_WithTrueTotalTrailer() /* 12 distinct chains -> 10 shown + "+2 more" trailer (gotqn's report: never silently drop). */ var rows = new List(); for (int i = 0; i < 12; i++) - rows.Add(Blocked(obj: $"db.dbo.Table{i}", waitMs: 1000 + i)); + rows.Add(Blocked(obj: $"db.dbo.Table{i}", waitMs: 1000 + i, spid: i + 1)); var context = AlertContextBuilders.BuildBlockingContext(Server, rows, NoExclusions); @@ -128,9 +131,9 @@ public void BuildBlockingContext_AttachesFirstReportXml() { var rows = new List { - Blocked(obj: "db.dbo.A"), /* DMV-style row: no XML */ - Blocked(obj: "db.dbo.B", xml: ""), - Blocked(obj: "db.dbo.C", xml: "") + Blocked(obj: "db.dbo.A", spid: 1), /* DMV-style row: no XML */ + Blocked(obj: "db.dbo.B", xml: "", spid: 2), + Blocked(obj: "db.dbo.C", xml: "", spid: 3) }; var context = AlertContextBuilders.BuildBlockingContext(Server, rows, NoExclusions); @@ -164,9 +167,9 @@ public void BuildBlockingContext_EachIncidentCarriesItsOwnReport_AndADmvOnlyChai { var rows = new List { - Blocked(obj: "db.dbo.A", xml: ""), - Blocked(obj: "db.dbo.B", xml: ""), - Blocked(obj: "db.dbo.C") /* DMV-style row: no report */ + Blocked(obj: "db.dbo.A", xml: "", spid: 1), + Blocked(obj: "db.dbo.B", xml: "", spid: 2), + Blocked(obj: "db.dbo.C", spid: 3) /* DMV-style row: no report */ }; var context = AlertContextBuilders.BuildBlockingContext(Server, rows, NoExclusions); diff --git a/Lite.Tests/LiteAlertForwardingTests.cs b/Lite.Tests/LiteAlertForwardingTests.cs index 0931f8a5d8..5d55615ec0 100644 --- a/Lite.Tests/LiteAlertForwardingTests.cs +++ b/Lite.Tests/LiteAlertForwardingTests.cs @@ -567,6 +567,54 @@ public async Task Cpu_SqlOnlyMode_UsesSqlValueAndLabel() 2. Blocking rolling-count fire + watermark persist (old :118-196 + #1091/#1145) ===================================================================================== */ + /* #5644 review (W1/W2): the XE-or-DMV choice is made on the rows BEFORE the excluded-database filter, so XE + reports that sit only in an excluded database do not fall back to DMV waits elsewhere (Lite's overview + makes the same choice on the unfiltered set). Lite runs the shared engine, so this is the Lite twin. */ + [Fact] + public async Task Blocking_XeOnlyInExcludedDatabase_DoesNotFallBackToDmvWaits() + { + DisableAllChecks(); + App.AlertBlockingEnabled = true; + App.AlertExcludedDatabases = new List { "ExcludedDb" }; + var h = new Harness(); + var engine = h.Build(); + var xe = BlockingRow(51); + xe.DatabaseName = "ExcludedDb"; + var dmv = BlockingRow(61); + dmv.Source = BlockedProcessAlertRow.DmvSnapshotSource; + h.Adapter.Blocking.Add(xe); + h.Adapter.Blocking.Add(dmv); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Empty(h.Deliverer.Outcomes); + + /* The XE report now sits in a database that is not excluded: it fires on the XE wait alone. */ + xe.DatabaseName = "StackOverflow"; + h.Now = h.Now.AddMinutes(10); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("1", fired.CurrentValue); + } + + [Fact] + public async Task Blocking_NoXeReports_FiresOnDmvWaits() + { + DisableAllChecks(); + App.AlertBlockingEnabled = true; + var h = new Harness(); + var engine = h.Build(); + var dmv = BlockingRow(61); + dmv.Source = BlockedProcessAlertRow.DmvSnapshotSource; + h.Adapter.Blocking.Add(dmv); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("1", fired.CurrentValue); + } + [Fact] public async Task Blocking_FiresOnce_PersistsWatermark_AndEdgeTriggerHoldsWithinTheHour() { diff --git a/PerformanceMonitor.Alerting/AlertContextBuilders.cs b/PerformanceMonitor.Alerting/AlertContextBuilders.cs index 3592010296..1adaddc81c 100644 --- a/PerformanceMonitor.Alerting/AlertContextBuilders.cs +++ b/PerformanceMonitor.Alerting/AlertContextBuilders.cs @@ -470,7 +470,9 @@ alert on it. */ /// #5644 W1/W2: THE set of blocking events a blocking alert is made of, so the alert's value and its detail /// cannot disagree. Excluded databases drop their rows, the reports of one wait collapse to one row carrying /// the longest wait time (), and the count is XE-preferred as Lite's overview has - /// always been: the blocked-process-report waits when there are any, else the DMV-snapshot waits. The alert + /// always been: the blocked-process-report waits when there are any, else the DMV-snapshot waits. "When there + /// are any" is judged on the rows BEFORE the excluded-database filter, so XE reports that exist only in + /// excluded databases yield an empty set rather than a fall back to DMV waits. The alert /// value is this set's Count; the card's "Blocking chain (xN)" groups are drawn from the same list, so /// the chain counts add up to the value. /// @@ -479,6 +481,8 @@ public static IReadOnlyList BlockingWaitRows( { if (events == null || events.Count == 0) return Array.Empty(); + var hasXeReports = events.Any(r => r.Source == BlockedProcessAlertRow.XeReportSource); + var waits = BlockingWaits.Collapse( FilterBlocking(events, excludedDatabases), r => r.Source ?? "", @@ -487,8 +491,13 @@ public static IReadOnlyList BlockingWaitRows( r => r.EventTime, r => r.WaitTimeMs); - var xeWaits = waits.Where(r => r.Source == BlockedProcessAlertRow.XeReportSource).ToList(); - return xeWaits.Count > 0 ? xeWaits : waits; + /* XE-or-DMV is decided on the UNFILTERED rows, as Lite's overview and the fleet tile do: if blocked-process + reports exist at all, they are the source, and when every one sits in an excluded database the answer + is empty. Deciding after the filter would fall back to the DMV snapshots there and fire an alert the + excluded-database rule means to suppress. */ + return hasXeReports + ? waits.Where(r => r.Source == BlockedProcessAlertRow.XeReportSource).ToList() + : waits; } /* Excluded databases drop their rows; rows with no database always pass. Shared by the render path and diff --git a/PerformanceMonitor.Alerting/AlertEngine.cs b/PerformanceMonitor.Alerting/AlertEngine.cs index 0d8e69924f..b0409a8c3a 100644 --- a/PerformanceMonitor.Alerting/AlertEngine.cs +++ b/PerformanceMonitor.Alerting/AlertEngine.cs @@ -1205,9 +1205,11 @@ private async Task CheckBlockingAsync( carries Lite's overview semantics (LocalDataService.Overview.cs:74-77): the XE blocked-process-report waits when there are any, else the DMV-snapshot waits (AWS RDS / unset blocked-process threshold), and the excluded-database filter (:118-127), whose rows - with no database always pass. The old shape counted every row, then recounted outside the - excluded set only once the raw count reached the threshold; filtering before counting - gives the same fire decision and a watermark that tracks the filtered count throughout. */ + with no database always pass. The XE-or-DMV choice is made on the rows BEFORE the excluded + filter, as before: XE reports that sit only in excluded databases count zero and + do not fall back to DMV waits. The old shape counted every row, then recounted outside + the excluded set only once the raw count reached the threshold; filtering before counting + keeps that fire decision and gives a watermark that tracks the filtered count throughout. */ effectiveBlockingCount = AlertContextBuilders.BlockingWaitRows(blockingRows, excludedDatabases).Count; } catch (OperationCanceledException) when (ct.IsCancellationRequested) From 0e2e946ec6baef539497399fd9a2916929d86cb9 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:00:58 -0400 Subject: [PATCH 18/42] Default Trace: read at-or-after the watermark tick and drop only the stored rows of that tick by EventSequence (#5644 W6) --- ...aultTraceEventsCollectorDefinitionTests.cs | 135 +++++++++++++++++- .../DefaultTraceEventsCollector.cs | 121 +++++++++++++++- 2 files changed, 244 insertions(+), 12 deletions(-) diff --git a/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs b/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs index 560bdd31f5..1ed85336d1 100644 --- a/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs +++ b/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs @@ -86,9 +86,11 @@ public void BuildQuery_ReadsDefaultTraceViaFnTraceGettable_ServerSide() Assert.Contains("t.status = 1", text, StringComparison.Ordinal); /* #5644 W6: the bound is CONVERTed to datetime so it compares like for like with ft.StartTime. A bare datetime2 parameter holding 10:49:13.643 sits BELOW the trace's own 10:49:13.6433333 (datetime ticks in 1/300 s), - so `>` held the newest stored event true on every poll and stored it again until a newer event came. */ + so `>` held the newest stored event true on every poll and stored it again until a newer event came. The bound is + `>=` so an event the trace flushes into the watermark's own tick after a poll is read; ReadAsync drops the stored + rows of that tick by EventSequence (see the WatermarkTick tests). */ Assert.Contains( - "ft.StartTime > CONVERT(datetime, COALESCE(@cutoff_time, DATEADD(HOUR, -24, SYSDATETIME())))", + "ft.StartTime >= CONVERT(datetime, COALESCE(@cutoff_time, DATEADD(HOUR, -24, SYSDATETIME())))", text, StringComparison.Ordinal); Assert.Contains("SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED", text, StringComparison.Ordinal); Assert.Contains("OPTION(RECOMPILE)", text, StringComparison.Ordinal); @@ -200,7 +202,7 @@ public void BuildQuery_KeepsTheReconfigureLine_OutsideEveryDatabasePredicate() /* The keep is the FIRST arm of the OR that follows the StartTime bound, and every database predicate sits AFTER it — inside the other arm — so none of them can reach the 15457 row. */ - var startTime = text.IndexOf("AND ft.StartTime > ", StringComparison.Ordinal); + var startTime = text.IndexOf("AND ft.StartTime >= ", StringComparison.Ordinal); var keepAt = text.IndexOf(keep, StringComparison.Ordinal); var systemDbs = text.IndexOf("ISNULL(ft.DatabaseID, 0) NOT IN (1, 3, 4)", StringComparison.Ordinal); var agDbs = text.IndexOf("ISNULL(ft.DatabaseID, 0) < 32761", StringComparison.Ordinal); @@ -384,7 +386,9 @@ public async Task ReadAsync_WritePayload_PinsTwentyOneColumnOrder_AndToleratesNu Assert.Empty(s_deltas.Calls); /* A reader carrying only the payload (no trailing path set) records nothing rather than throwing. */ - Assert.Empty(context.PendingState); + Assert.False(context.PendingState.ContainsKey(DefaultTraceEventsCollector.LastTraceFilePathStateKey)); + /* ... but it does record the newest tick's EventSequence for the next cycle's at-or-after read (#5644 W6). */ + Assert.Equal(TickState(eventTime, "900001"), context.PendingState[DefaultTraceEventsCollector.WatermarkTickSequencesStateKey]); } /* ---- #1962: read the CURRENT rollover file in steady state, fall back when the trace moved ---- */ @@ -395,11 +399,12 @@ or fail on checkout settings rather than on the SQL. */ private static string Lf(string text) => text.Replace("\r\n", "\n", StringComparison.Ordinal); [Fact] - public void StateKeys_DeclaresTheLastSeenTracePath_AndTheDeclaringSetIsPinned() + public void StateKeys_DeclaresTheLastSeenTracePathAndTheWatermarkTickSequences_AndTheDeclaringSetIsPinned() { Assert.Equal( - new[] { "last_trace_file_path" }, + new[] { "last_trace_file_path", "watermark_tick_event_sequences" }, DefaultTraceEventsCollector.Instance.StateKeys.ToArray()); + Assert.Equal("watermark_tick_event_sequences", DefaultTraceEventsCollector.WatermarkTickSequencesStateKey); Assert.Equal("last_trace_file_path", DefaultTraceEventsCollector.LastTraceFilePathStateKey); /* Almost every collector's dedup IS derivable from its rows (a MAX() over the target table), so it @@ -513,6 +518,124 @@ would never notice those rollovers — leaving the collector in the expensive fa Assert.DoesNotContain("trace_path", string.Join(",", DefaultTraceEventsCollector.Instance.PayloadColumns.Select(c => c.Name)), StringComparison.Ordinal); } + /* #5644 W6: the read is at-or-after the watermark tick (a strict `>` lost an event the trace flushed into the + watermark's own 3.33 ms tick after the poll), so the rows already stored for that tick come back and are + dropped here by EventSequence. */ + private static readonly DateTime s_tick = new(2026, 7, 9, 10, 49, 13, 643, DateTimeKind.Unspecified); + + private static string TickState(DateTime tick, string sequences) + => tick.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture) + ":" + sequences; + + private static object[] TraceRow(DateTime eventTime, long sequence) + { + var row = Enumerable.Repeat((object)DBNull.Value, 21).ToArray(); + row[0] = eventTime; + row[18] = sequence; + return row; + } + + private static CollectorContext MakeTickContext(DateTime? watermark, string? tickState) + { + var context = MakeContext(watermark: watermark); + if (tickState is null) + { + return context; + } + + return new CollectorContext + { + State = new Dictionary(StringComparer.Ordinal) + { + [DefaultTraceEventsCollector.WatermarkTickSequencesStateKey] = tickState, + }, + ServerId = context.ServerId, + ServerName = context.ServerName, + CollectionTime = context.CollectionTime, + Deltas = context.Deltas, + Target = context.Target, + Watermark = watermark, + ExcludedDatabases = context.ExcludedDatabases, + CollectSchemaChangeEvents = context.CollectSchemaChangeEvents, + }; + } + + [Fact] + public async Task ReadAsync_WatermarkTick_DropsTheStoredReRead_KeepsANewEventInTheSameTick_AndTheNextTick() + { + var context = MakeTickContext(s_tick, TickState(s_tick, "900,901")); + using var reader = new FakeCollectorDataReader( + TraceRow(s_tick.AddMilliseconds(10), 905), /* next tick: kept */ + TraceRow(s_tick, 902), /* same tick, flushed after the last poll: kept */ + TraceRow(s_tick, 901), /* stored re-read: dropped */ + TraceRow(s_tick, 900)); /* stored re-read: dropped */ + + var rows = await DefaultTraceEventsCollector.Instance.ReadAsync(reader, context, CancellationToken.None); + + Assert.Equal(new long?[] { 905, 902 }, rows.Select(r => r.EventSequence).ToArray()); + /* The next cycle's watermark is the newer tick, so the state moves to it. */ + Assert.Equal( + TickState(s_tick.AddMilliseconds(10), "905"), + context.PendingState[DefaultTraceEventsCollector.WatermarkTickSequencesStateKey]); + } + + [Fact] + public async Task ReadAsync_WatermarkTick_WhenNothingNewerArrives_KeepsEveryStoredSequenceAndAddsTheNewOne() + { + var context = MakeTickContext(s_tick, TickState(s_tick, "900,901")); + using var reader = new FakeCollectorDataReader(TraceRow(s_tick, 902), TraceRow(s_tick, 900)); + + var rows = await DefaultTraceEventsCollector.Instance.ReadAsync(reader, context, CancellationToken.None); + + Assert.Equal(new long?[] { 902 }, rows.Select(r => r.EventSequence).ToArray()); + Assert.Equal( + TickState(s_tick, "900,901,902"), + context.PendingState[DefaultTraceEventsCollector.WatermarkTickSequencesStateKey]); + } + + [Theory] + [InlineData(null)] /* nothing recorded: a first run, an upgrade, a restarted host */ + [InlineData("not a state")] /* malformed */ + [InlineData("1:900,x")] /* malformed sequence list */ + [InlineData("1:900")] /* recorded for a different tick than the watermark */ + public async Task ReadAsync_WatermarkTick_WithNoUsableState_DropsNothing(string? state) + { + var context = MakeTickContext(s_tick, state); + using var reader = new FakeCollectorDataReader(TraceRow(s_tick, 900), TraceRow(s_tick, 901)); + + var rows = await DefaultTraceEventsCollector.Instance.ReadAsync(reader, context, CancellationToken.None); + + Assert.Equal(2, rows.Count); + } + + [Fact] + public async Task ReadAsync_WatermarkTick_WithNoWatermark_KeepsEverythingAndRecordsTheNewestTick() + { + var context = MakeTickContext(null, TickState(s_tick, "900")); + using var reader = new FakeCollectorDataReader(TraceRow(s_tick, 900), TraceRow(s_tick.AddMilliseconds(-10), 899)); + + var rows = await DefaultTraceEventsCollector.Instance.ReadAsync(reader, context, CancellationToken.None); + + Assert.Equal(2, rows.Count); + Assert.Equal(TickState(s_tick, "900"), context.PendingState[DefaultTraceEventsCollector.WatermarkTickSequencesStateKey]); + } + + [Fact] + public async Task ReadAsync_WatermarkTick_KeepsARowWithNoSequence_AndLeavesTheStateAloneOnAnEmptyRead() + { + var state = TickState(s_tick, "900"); + var noSequence = TraceRow(s_tick, 0); + noSequence[18] = DBNull.Value; + var context = MakeTickContext(s_tick, state); + using var withRow = new FakeCollectorDataReader(noSequence); + var rows = await DefaultTraceEventsCollector.Instance.ReadAsync(withRow, context, CancellationToken.None); + Assert.Single(rows); + + var emptyContext = MakeTickContext(s_tick, state); + using var empty = new FakeCollectorDataReader(); + Assert.Empty(await DefaultTraceEventsCollector.Instance.ReadAsync(empty, emptyContext, CancellationToken.None)); + Assert.False(emptyContext.PendingState.ContainsKey(DefaultTraceEventsCollector.WatermarkTickSequencesStateKey)); + } + [Fact] public async Task ReadAsync_RecordsThePathItRead_EvenWhenTheCycleCollectedNothing() { diff --git a/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs b/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs index e3c29a63fe..75a432e855 100644 --- a/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs +++ b/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.Data.Common; using System.Globalization; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -48,8 +49,10 @@ namespace PerformanceMonitor.Collectors; /// /// STATEFUL dedup (must-handle): mirrors the deadlock / blocked-process-report / /// system_health collectors — an event_time watermark (the newest already-collected StartTime from -/// the host store) filters ft.StartTime > @cutoff_time, so a rollover file re-read across cycles -/// never re-ingests an event. Unlike those XE ring-buffer collectors (whose 10-minute first-run fallback +/// the host store) bounds the read at ft.StartTime >= @cutoff_time (at-or-after, #5644 W6: an event in the +/// watermark's own 3.33 ms tick that the trace flushes after a poll must not be lost), and the rows already +/// stored for that one tick are dropped by EventSequence (see ), +/// so a rollover file re-read across cycles never re-ingests an event. Unlike those XE ring-buffer collectors (whose 10-minute first-run fallback /// fits their tiny recent-only buffers), the FIRST run here collects EVERYTHING still in the trace files /// (a far-past cutoff), because the default trace persists hours-to-days of history on disk that would /// otherwise be lost — the same first-run-collects-all choice the Dashboard collector makes. @@ -145,6 +148,20 @@ private DefaultTraceEventsCollector() /// public const string LastTraceFilePathStateKey = "last_trace_file_path"; + /// + /// The key holding the EventSequence of every event this collector + /// read at the newest StartTime tick it has seen, as "<DateTime ticks>:<seq>,<seq>..." + /// (#5644 W6). The read is at-or-after the watermark (ft.StartTime >= cutoff), so it returns the + /// rows already stored for the watermark's own tick as well as any event the trace flushed into that tick + /// after the last poll; drops the first and keeps the second. A strict + /// > bound could not tell them apart and lost the second. State, not a read of the store, so + /// the identity is host-neutral: Lite and Darling both persist it after the cycle (the same path as + /// ). Absent, stale (its tick is not the watermark) or malformed + /// state drops nothing: a possible repeat of one tick's rows, which the readers' DISTINCT collapses, is + /// cheaper than a lost event. + /// + public const string WatermarkTickSequencesStateKey = "watermark_tick_event_sequences"; + public sealed class Row { public DateTime? EventTime { get; set; } @@ -269,7 +286,7 @@ JOIN sys.trace_events AS te ON ft.EventClass = te.trace_event_id WHERE t.is_default = 1 AND t.status = 1 -AND ft.StartTime > CONVERT(datetime, {1}) +AND ft.StartTime >= CONVERT(datetime, {1}) AND ( /*The sp_configure line (msg 15457, an ErrorLog write): kept regardless of the session's database @@ -334,7 +351,8 @@ ft.StartTime DESC public override string PrefixIdColumnName => "default_trace_event_id"; /// - /// Only events newer than the newest already-collected StartTime are fetched, so re-reading the + /// Events at or after the newest already-collected StartTime are fetched and the ones already stored for + /// that tick are dropped by EventSequence (), so re-reading the /// rolling trace files across cycles never re-ingests an event (mirrors the deadlock / system_health /// collectors' event_time watermark). First run has no watermark → collect all on-disk history. /// @@ -348,7 +366,7 @@ ft.StartTime DESC /// before the query is built and persists whatever this cycle observed once the cycle completes. /// The only collector that declares state — every other one's dedup is derivable from its rows. /// - public override IReadOnlyList StateKeys { get; } = new[] { LastTraceFilePathStateKey }; + public override IReadOnlyList StateKeys { get; } = new[] { LastTraceFilePathStateKey, WatermarkTickSequencesStateKey }; /// /// Collects on SQL Server, Azure SQL Managed Instance, and AWS RDS — everywhere the built-in default @@ -501,7 +519,98 @@ leaves the next cycle on the safe fallback. */ context.PendingState[LastTraceFilePathStateKey] = reader.GetString(0); } - return rows; + return DropStoredWatermarkTickRows(rows, context); + } + + /// + /// #5644 W6: the query reads at-or-after the watermark tick, so rows already stored for that tick come + /// back with any new event of the same tick. Drops exactly the stored ones (same tick, EventSequence in + /// the state the previous cycle recorded) and records this cycle's newest-tick sequences for the next. + /// A row with no EventSequence cannot be matched and is kept: a repeat beats a lost event. + /// + private static List DropStoredWatermarkTickRows(List rows, CollectorContext context) + { + context.State.TryGetValue(WatermarkTickSequencesStateKey, out var stored); + var storedSequences = TryParseTickSequences(stored, out var storedTicks, out var parsed) + ? parsed + : new HashSet(); + long? watermarkTicks = context.Watermark?.Ticks; + bool storedIsForWatermark = watermarkTicks.HasValue && storedTicks == watermarkTicks.Value; + + var kept = new List(rows.Count); + long newestTicks = long.MinValue; + foreach (var row in rows) + { + if (row.EventTime is not { } time) + { + kept.Add(row); + continue; + } + + newestTicks = Math.Max(newestTicks, time.Ticks); + bool alreadyStored = storedIsForWatermark + && time.Ticks == watermarkTicks!.Value + && row.EventSequence is { } sequence + && storedSequences.Contains(sequence); + if (!alreadyStored) + { + kept.Add(row); + } + } + + if (newestTicks != long.MinValue) + { + /* Every sequence this cycle read at the newest tick, dropped ones included, plus the earlier ones + when that tick is still the stored one (a rolled-over trace may no longer return them). */ + var newest = new SortedSet(); + foreach (var row in rows) + { + if (row.EventTime is { } t && t.Ticks == newestTicks && row.EventSequence is { } q) + { + newest.Add(q); + } + } + + if (storedIsForWatermark && newestTicks == watermarkTicks!.Value) + { + newest.UnionWith(storedSequences); + } + + context.PendingState[WatermarkTickSequencesStateKey] = + newestTicks.ToString(CultureInfo.InvariantCulture) + ":" + + string.Join(",", newest.Select(static v => v.ToString(CultureInfo.InvariantCulture))); + } + + return kept; + } + + private static bool TryParseTickSequences(string? value, out long ticks, out HashSet sequences) + { + ticks = 0; + sequences = new HashSet(); + if (string.IsNullOrEmpty(value)) + { + return false; + } + + int colon = value.IndexOf(':', StringComparison.Ordinal); + if (colon <= 0 || !long.TryParse(value.AsSpan(0, colon), NumberStyles.None, CultureInfo.InvariantCulture, out ticks)) + { + return false; + } + + foreach (var part in value.Substring(colon + 1).Split(',', StringSplitOptions.RemoveEmptyEntries)) + { + if (!long.TryParse(part, NumberStyles.Integer, CultureInfo.InvariantCulture, out var sequence)) + { + sequences.Clear(); + return false; + } + + sequences.Add(sequence); + } + + return true; } public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) From 01ba2b371e4eb098c0887bd814cded3aee1aa8bd Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:10:00 -0400 Subject: [PATCH 19/42] Blocking waits: epoch_us for DuckDB, NULL times are their own wait with NULLS LAST; viewer default trace drops stored copies; pins follow the wait CTEs (#5644) --- .../Darling.Tests/BlockingWaitCountTests.cs | 79 +++++++++++++++++++ .../ViewerFleetAzureMasterScopeLiveTests.cs | 8 +- .../Darling.Tests/ViewerFleetRollupTests.cs | 7 +- .../ViewerServerClockEventReadsTests.cs | 23 ++++++ Darling/Darling.Tests/ViewerW2aTests.cs | 4 +- .../ViewerDataService.SystemEvents.cs | 41 +++++++--- Lite.Tests/BlockingWaitDuckDbTests.cs | 58 ++++++++++++++ PerformanceMonitor.Common/BlockingWaits.cs | 21 ++--- 8 files changed, 217 insertions(+), 24 deletions(-) diff --git a/Darling/Darling.Tests/BlockingWaitCountTests.cs b/Darling/Darling.Tests/BlockingWaitCountTests.cs index 1024eba566..e854677430 100644 --- a/Darling/Darling.Tests/BlockingWaitCountTests.cs +++ b/Darling/Darling.Tests/BlockingWaitCountTests.cs @@ -217,4 +217,83 @@ public async Task PostgresSql_AgreesWithTheCSharpRule_OnTheSameReports() Assert.Equal(6, expected.Count); Assert.Equal(expected.Max(w => w.WaitTimeMs), Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); } + + /* ── #5644 wave-1 review: sub-millisecond starts and NULL times, in C# and in the store's SQL ── */ + + internal sealed record NullableReport(int Blocked, int Blocker, DateTime? Time, long? Wait); + + /// + /// The edge cases both stores must agree with the C# rule on. Event times are offsets from T0 in 100 ns + /// ticks so a 2000.4 ms difference survives (a DateTime keeps it; PostgreSQL and DuckDB timestamps keep + /// microseconds). Expected waits: pair 1/2 two (derived starts 2000.4 ms apart split), pair 3/4 one (exactly + /// 2000.0 ms merges), pair 5/6 four (one dated report, then a NULL event time, a NULL wait time and both NULL, + /// each a wait of its own), pair 7/8 one (a tie on wait time keeps the LATER report). + /// + internal static List EdgeCaseReports() + { + DateTime At(long ticks) => T0.AddTicks(ticks); + const long Second = TimeSpan.TicksPerSecond; + return new List + { + new(1, 2, At(10 * Second), 10000), new(1, 2, At(12 * Second + 4000), 10000), + new(3, 4, At(10 * Second), 10000), new(3, 4, At(12 * Second), 10000), + new(5, 6, At(10 * Second), 10000), new(5, 6, null, 5000), new(5, 6, At(20 * Second), null), new(5, 6, null, null), + new(7, 8, At(10 * Second), 10000), new(7, 8, At(10 * Second + 4000000), 10000), + }; + } + + internal static List CollapseEdgeCases(IEnumerable rows) => + BlockingWaits.Collapse(rows, _ => "xe", r => r.Blocked, r => r.Blocker, r => r.Time, r => r.Wait); + + internal static string EdgeKey(int blocked, int blocker, DateTime? time, long? wait) => + string.Format(CultureInfo.InvariantCulture, "{0}|{1}|{2:yyyy-MM-dd HH:mm:ss.ffffff}|{3}", blocked, blocker, time, wait); + + internal static string EdgeValuesSource(IEnumerable rows) => + "SELECT v.blocked_spid, v.blocking_spid, v.event_time, v.wait_time_ms FROM (VALUES " + + string.Join(",", rows.Select(r => string.Format( + CultureInfo.InvariantCulture, "({0},{1},CAST({2} AS TIMESTAMP),CAST({3} AS BIGINT))", + r.Blocked, r.Blocker, + r.Time is null ? "NULL" : $"'{r.Time:yyyy-MM-dd HH:mm:ss.ffffff}'", + r.Wait is null ? "NULL" : r.Wait.Value.ToString(CultureInfo.InvariantCulture)))) + + ") AS v(blocked_spid, blocking_spid, event_time, wait_time_ms)"; + + [Fact] + public void EdgeCases_SubMillisecondStartsAndNullTimes_GiveTheDocumentedWaits() + { + var waits = CollapseEdgeCases(EdgeCaseReports()); + + Assert.Equal(2, waits.Count(w => w.Blocked == 1)); + Assert.Single(waits, w => w.Blocked == 3); + Assert.Equal(4, waits.Count(w => w.Blocked == 5)); + var tie = Assert.Single(waits, w => w.Blocked == 7); + Assert.Equal(T0.AddTicks((10 * TimeSpan.TicksPerSecond) + 4000000), tie.Time); + } + + [Fact] + public async Task PostgresSql_AgreesWithTheCSharpRule_OnSubMillisecondStartsAndNullTimes() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrWhiteSpace(connectionString), "Set DARLING_TEST_PG to run the live wait-count SQL test."); + + var sql = "WITH " + BlockingWaits.WaitsCteSql(BlockingWaits.SqlDialect.Postgres, "waits", EdgeValuesSource(EdgeCaseReports())) + + " SELECT blocked_spid, blocking_spid, event_time, wait_time_ms FROM waits"; + var fromSql = new List(); + await using (var connection = new NpgsqlConnection(connectionString)) + { + await connection.OpenAsync(); + await using var command = new NpgsqlCommand(sql, connection); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + fromSql.Add(EdgeKey( + reader.GetInt32(0), reader.GetInt32(1), + reader.IsDBNull(2) ? null : reader.GetDateTime(2), + reader.IsDBNull(3) ? null : reader.GetInt64(3))); + } + } + + var fromCSharp = CollapseEdgeCases(EdgeCaseReports()).Select(w => EdgeKey(w.Blocked, w.Blocker, w.Time, w.Wait)); + Assert.Equal(fromCSharp.OrderBy(k => k, StringComparer.Ordinal), fromSql.OrderBy(k => k, StringComparer.Ordinal)); + Assert.Equal(8, fromSql.Count); + } } diff --git a/Darling/Darling.Tests/ViewerFleetAzureMasterScopeLiveTests.cs b/Darling/Darling.Tests/ViewerFleetAzureMasterScopeLiveTests.cs index ce1d4a2e3e..4d643ecd7b 100644 --- a/Darling/Darling.Tests/ViewerFleetAzureMasterScopeLiveTests.cs +++ b/Darling/Darling.Tests/ViewerFleetAzureMasterScopeLiveTests.cs @@ -372,8 +372,12 @@ public async Task TheRegistryIsReadOnlyForMasters_AndOncePerFleetRefresh() private static Task BprAsync(NpgsqlConnection connection, int serverId, string? db, DateTime at, System.Threading.CancellationToken ct) => DarlingMcpTestData.ExecAsync(connection, ct, - "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, wait_time_ms, database_name) VALUES ($1,$2,$3,'s',$2,5000,$4)", - CollectionIdGenerator.Next(), at, serverId, (object?)db ?? DBNull.Value); + "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, wait_time_ms, database_name, blocked_spid, blocking_spid) VALUES ($1,$2,$3,'s',$2,5000,$4,$5,50)", + CollectionIdGenerator.Next(), at, serverId, (object?)db ?? DBNull.Value, System.Threading.Interlocked.Increment(ref s_nextBlockedSpid)); + + /* #5644 W1: reports of one blocked session, blocker and wait start are ONE wait, so each seeded report gets a + blocked session of its own and counts as a wait of its own. */ + private static int s_nextBlockedSpid = 100; private static Task DeadlockAsync(NpgsqlConnection connection, int serverId, string db, DateTime at, System.Threading.CancellationToken ct) => DarlingMcpTestData.ExecAsync(connection, ct, diff --git a/Darling/Darling.Tests/ViewerFleetRollupTests.cs b/Darling/Darling.Tests/ViewerFleetRollupTests.cs index 5e79421329..f870807da8 100644 --- a/Darling/Darling.Tests/ViewerFleetRollupTests.cs +++ b/Darling/Darling.Tests/ViewerFleetRollupTests.cs @@ -77,8 +77,11 @@ public void FleetTotalsSql_AddsThePostgresCounterDifferences_NeverTheRawColumn() series), and the single window over the whole fleet's rows (27 s for 30 days on a 50-server store) stays gone. */ Assert.Contains("deadlocks - LAG(deadlocks) OVER (PARTITION BY database_name ORDER BY collection_time)", sql, StringComparison.Ordinal); - Assert.DoesNotContain("PARTITION BY server_id", sql, StringComparison.Ordinal); - Assert.Equal(1, CountOccurrences(sql, "OVER (")); + /* #5644 W1: the blocking-wait CTEs (BlockingWaits.WaitsCteSql) carry their own windows, partitioned by + server_id; the deadlock read below them is judged on its own. */ + var deadlockRead = sql[sql.LastIndexOf("r.wait_rank = 1", StringComparison.Ordinal)..]; + Assert.DoesNotContain("PARTITION BY server_id", deadlockRead, StringComparison.Ordinal); + Assert.Equal(1, CountOccurrences(deadlockRead, "OVER (")); /* A pair whose counter never takes two values has only zero or NULL differences and adds nothing to the total, so it is dropped before the ordered pass (this read needs no "a counter in every row" half: it returns the total, not the number of differences). */ diff --git a/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs b/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs index 049cdd8c19..141766a659 100644 --- a/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs +++ b/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs @@ -96,6 +96,29 @@ public async Task ADefaultTraceEventStoredAtServerLocalTime_OnEachSideOfAChange_ Assert.Equal(Naive(2026, 11, 2, 13), byLabel["after-fall"]); } + /// + /// #5644 W6: the store holds copies of one event (a DDL's Begin and Commit rows from an older build, an event + /// re-stored by the datetime watermark round trip) that match on every column the viewer reads. The grid shows + /// each once, in reader order; an event that differs in any column stays. + /// + [Fact] + public async Task StoredCopiesOfOneDefaultTraceEvent_ComeBackOnce_AndNearCopiesStay() + { + var table = NewTable(); + AddEvent(table, Naive(2026, 3, 9, 8), "same-event"); + AddEvent(table, Naive(2026, 3, 9, 8), "same-event"); + AddEvent(table, Naive(2026, 3, 9, 8), "same-event"); + AddEvent(table, Naive(2026, 3, 9, 8, 1), "same-event"); /* a minute later: another event */ + AddEvent(table, Naive(2026, 3, 9, 8), "other-database"); /* same instant, another database */ + + var rows = await ReadAsync(table, Eastern(), Naive(2026, 1, 1, 0), Naive(2026, 12, 31, 0)); + + Assert.Equal(3, rows.Count); + Assert.Single(rows, r => r.DatabaseName == "same-event" && r.EventTimeUtc == Naive(2026, 3, 9, 12)); + Assert.Single(rows, r => r.DatabaseName == "same-event" && r.EventTimeUtc == Naive(2026, 3, 9, 12, 1)); + Assert.Single(rows, r => r.DatabaseName == "other-database"); + } + [Fact] public async Task ADefaultTraceEventInASkippedOrRepeatedLocalHour_DoesNotThrow() { diff --git a/Darling/Darling.Tests/ViewerW2aTests.cs b/Darling/Darling.Tests/ViewerW2aTests.cs index 1136a4da73..0bef1cc0e3 100644 --- a/Darling/Darling.Tests/ViewerW2aTests.cs +++ b/Darling/Darling.Tests/ViewerW2aTests.cs @@ -99,7 +99,9 @@ public void SummaryBlockingSql_CountAndMaxWait_FromBothSources_OverTheWindow() Assert.Contains("server_id = $1", sql, StringComparison.Ordinal); /* #3895: every WINDOWED read carries the partition-column floor, and the two "ever" reads none — a floor would change what "newest event ever" answers. */ - Assert.Equal(4, System.Text.RegularExpressions.Regex.Matches(sql, @"event_time >= \$2 AND collection_time >= \$3\)").Count); + /* #5644 W1: the two windowed reads are now the two wait CTEs' sources (count and worst wait both come off + the CTEs), each closed by the CTE's subquery parenthesis. */ + Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(sql, @"event_time >= \$2 AND collection_time >= \$3\)").Count); Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(sql, @"\(SELECT MAX\(event_time\)\s+FROM \w+\s+WHERE server_id = \$1\)").Count); } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs index 31dcd7ab50..efe6b3f15d 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs @@ -695,6 +695,7 @@ internal static async Task> ReadDefaultTraceEventsAsy DbDataReader reader, ServerClock clock, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken) { var rows = new List(); + var seen = new HashSet<(DateTime?, string?, string?, string?, string?, string?, string?, int?, long?, long?, int?, int?, string?)>(); while (await reader.ReadAsync(cancellationToken)) { @@ -708,21 +709,41 @@ internal static async Task> ReadDefaultTraceEventsAsy if (!SystemEventSignificance.IsSignificantDefaultTraceEvent(eventName, severity)) continue; + var databaseName = reader.IsDBNull(2) ? null : reader.GetString(2); + var objectName = reader.IsDBNull(3) ? null : reader.GetString(3); + var loginName = reader.IsDBNull(4) ? null : reader.GetString(4); + var hostName = reader.IsDBNull(5) ? null : reader.GetString(5); + var applicationName = reader.IsDBNull(6) ? null : reader.GetString(6); + var spid = reader.IsDBNull(7) ? (int?)null : reader.GetInt32(7); + var durationUs = reader.IsDBNull(8) ? (long?)null : reader.GetInt64(8); + var integerData = reader.IsDBNull(9) ? (long?)null : reader.GetInt64(9); + var errorNumber = reader.IsDBNull(11) ? (int?)null : reader.GetInt32(11); + var textData = reader.IsDBNull(12) ? null : reader.GetString(12); + + /* #5644 W6: the store holds rows that are the same event twice (an older build kept both the Begin and + the Commit of every DDL, and re-stored an event whose datetime StartTime did not round-trip through + the watermark parameter). They match on every column read here and only the event_sequence the + collector stores tells them apart, so the first of each in reader order is kept, as the MCP reader + and Lite's do, and the copies stop filling the grid and the page limit. */ + if (!seen.Add((eventTimeUtc, eventName, databaseName, objectName, loginName, hostName, applicationName, + spid, durationUs, integerData, severity, errorNumber, textData))) + continue; + rows.Add(new DefaultTraceEventRow( eventTimeUtc, SystemEventSignificance.ClassifyDefaultTraceEvent(eventName), eventName, - reader.IsDBNull(2) ? null : reader.GetString(2), /* database_name */ - reader.IsDBNull(3) ? null : reader.GetString(3), /* object_name */ - reader.IsDBNull(4) ? null : reader.GetString(4), /* login_name */ - reader.IsDBNull(5) ? null : reader.GetString(5), /* host_name */ - reader.IsDBNull(6) ? null : reader.GetString(6), /* application_name */ - reader.IsDBNull(7) ? (int?)null : reader.GetInt32(7), /* spid */ - reader.IsDBNull(8) ? (long?)null : reader.GetInt64(8), /* duration_us */ - reader.IsDBNull(9) ? (long?)null : reader.GetInt64(9), /* integer_data */ + databaseName, + objectName, + loginName, + hostName, + applicationName, + spid, + durationUs, + integerData, severity, - reader.IsDBNull(11) ? (int?)null : reader.GetInt32(11), /* error_number */ - reader.IsDBNull(12) ? null : reader.GetString(12))); /* text_data */ + errorNumber, + textData)); } /* OrderByDescending is a stable sort (List.Sort is not), so ties keep the reader's order. */ diff --git a/Lite.Tests/BlockingWaitDuckDbTests.cs b/Lite.Tests/BlockingWaitDuckDbTests.cs index 0815e90641..48007ccf98 100644 --- a/Lite.Tests/BlockingWaitDuckDbTests.cs +++ b/Lite.Tests/BlockingWaitDuckDbTests.cs @@ -182,4 +182,62 @@ public async Task TheDmvSnapshotFallback_CountsWaitsToo() Assert.Equal(2, CSharpWaitCount()); Assert.Equal(2, await OverviewCountAsync()); } + + /* ── #5644 wave-1 review: sub-millisecond starts and NULL times ── */ + + private sealed record Edge(int Blocked, int Blocker, DateTime? Time, long? Wait); + + /* The same edge cases Darling.Tests BlockingWaitCountTests.EdgeCaseReports runs on PostgreSQL: pair 1/2 two waits + (derived starts 2000.4 ms apart), 3/4 one (exactly 2000.0 ms), 5/6 four (NULL event time, NULL wait time and + both NULL are waits of their own), 7/8 one (a tie on wait time keeps the later report). */ + private static List EdgeReports() + { + var t0 = new DateTime(2026, 10, 9, 9, 0, 0, DateTimeKind.Unspecified); + DateTime At(long ticks) => t0.AddTicks(ticks); + const long Second = TimeSpan.TicksPerSecond; + return new List + { + new(1, 2, At(10 * Second), 10000), new(1, 2, At((12 * Second) + 4000), 10000), + new(3, 4, At(10 * Second), 10000), new(3, 4, At(12 * Second), 10000), + new(5, 6, At(10 * Second), 10000), new(5, 6, null, 5000), new(5, 6, At(20 * Second), null), new(5, 6, null, null), + new(7, 8, At(10 * Second), 10000), new(7, 8, At((10 * Second) + 4000000), 10000), + }; + } + + private static string EdgeKey(int blocked, int blocker, DateTime? time, long? wait) => + string.Format(System.Globalization.CultureInfo.InvariantCulture, "{0}|{1}|{2:yyyy-MM-dd HH:mm:ss.ffffff}|{3}", blocked, blocker, time, wait); + + [Fact] + public async Task DuckDbSql_AgreesWithTheCSharpRule_OnSubMillisecondStartsAndNullTimes() + { + var rows = EdgeReports(); + var source = "SELECT v.blocked_spid, v.blocking_spid, v.event_time, v.wait_time_ms FROM (VALUES " + + string.Join(",", rows.Select(r => string.Format( + System.Globalization.CultureInfo.InvariantCulture, "({0},{1},CAST({2} AS TIMESTAMP),CAST({3} AS BIGINT))", + r.Blocked, r.Blocker, + r.Time is null ? "NULL" : $"'{r.Time:yyyy-MM-dd HH:mm:ss.ffffff}'", + r.Wait is null ? "NULL" : r.Wait.Value.ToString(System.Globalization.CultureInfo.InvariantCulture)))) + + ") AS v(blocked_spid, blocking_spid, event_time, wait_time_ms)"; + + var fromSql = new List(); + using (var readLock = _duckDb.AcquireReadLock()) + { + _conn ??= _duckDb.CreateConnection(); + if (_conn.State != System.Data.ConnectionState.Open) await _conn.OpenAsync(); + using var cmd = _conn.CreateCommand(); + cmd.CommandText = "WITH " + BlockingWaits.WaitsCteSql(BlockingWaits.SqlDialect.DuckDb, "waits", source) + + " SELECT blocked_spid, blocking_spid, event_time, wait_time_ms FROM waits"; + using var reader = await cmd.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + fromSql.Add(EdgeKey( + Convert.ToInt32(reader.GetValue(0)), Convert.ToInt32(reader.GetValue(1)), + reader.IsDBNull(2) ? null : reader.GetDateTime(2), + reader.IsDBNull(3) ? null : Convert.ToInt64(reader.GetValue(3)))); + } + + var fromCSharp = BlockingWaits.Collapse(rows, _ => "xe", r => r.Blocked, r => r.Blocker, r => r.Time, r => r.Wait) + .Select(w => EdgeKey(w.Blocked, w.Blocker, w.Time, w.Wait)); + Assert.Equal(fromCSharp.OrderBy(k => k, StringComparer.Ordinal), fromSql.OrderBy(k => k, StringComparer.Ordinal)); + Assert.Equal(8, fromSql.Count); + } } diff --git a/PerformanceMonitor.Common/BlockingWaits.cs b/PerformanceMonitor.Common/BlockingWaits.cs index 11a7f07dd1..f85c33fb04 100644 --- a/PerformanceMonitor.Common/BlockingWaits.cs +++ b/PerformanceMonitor.Common/BlockingWaits.cs @@ -47,7 +47,8 @@ public static double WaitStartEpochMs(DateTime eventTime, long waitTimeMs) => /// /// Collapses reports to one row per wait: the report with the longest wait time (the latest one on a tie). /// keeps rows that must never merge apart (the report source, for one). A row with - /// no event time cannot be placed in a wait, so it stays a wait of its own. The result is ordered by the + /// no event time or no wait time has no derived start and cannot be placed in a wait, so it stays a wait of + /// its own (the SQL form does the same). The result is ordered by the /// representative's event time, newest first, as the alert reads are. /// public static List Collapse( @@ -56,20 +57,21 @@ public static List Collapse( Func blockedSpid, Func blockingSpid, Func eventTime, - Func waitTimeMs) + Func waitTimeMs) { var result = new List(); var placeable = new List<(T Row, double Start, DateTime Time, long Wait)>(); foreach (var row in reports) { var time = eventTime(row); - if (time is null) + var wait = waitTimeMs(row); + if (time is null || wait is null) { result.Add(row); continue; } - placeable.Add((row, WaitStartEpochMs(time.Value, waitTimeMs(row)), time.Value, waitTimeMs(row))); + placeable.Add((row, WaitStartEpochMs(time.Value, wait.Value), time.Value, wait.Value)); } foreach (var pair in placeable.GroupBy(p => (partition(p.Row), blockedSpid(p.Row), blockingSpid(p.Row)))) @@ -127,7 +129,7 @@ public static string WaitsCteSql( var partition = string.Join( ", ", (partitionColumns ?? Array.Empty()).Concat(new[] { "blocked_spid", "blocking_spid" })); var epochMs = dialect == SqlDialect.DuckDb - ? "CAST(epoch_ms(s.event_time) AS DOUBLE)" + ? "(CAST(epoch_us(s.event_time) AS DOUBLE) / 1000.0)" : "(date_part('epoch', s.event_time) * 1000.0)"; var tolerance = WaitStartToleranceMs.ToString(System.Globalization.CultureInfo.InvariantCulture); return $@" @@ -138,20 +140,21 @@ public static string WaitsCteSql( {name}_gapped AS ( SELECT m.*, CASE - WHEN LAG(m.wait_start_ms) OVER (PARTITION BY {partition} ORDER BY m.wait_start_ms, m.event_time) IS NULL - OR m.wait_start_ms - LAG(m.wait_start_ms) OVER (PARTITION BY {partition} ORDER BY m.wait_start_ms, m.event_time) > {tolerance} + WHEN m.wait_start_ms IS NULL THEN 1 + WHEN LAG(m.wait_start_ms) OVER (PARTITION BY {partition} ORDER BY m.wait_start_ms NULLS LAST, m.event_time NULLS LAST) IS NULL + OR m.wait_start_ms - LAG(m.wait_start_ms) OVER (PARTITION BY {partition} ORDER BY m.wait_start_ms NULLS LAST, m.event_time NULLS LAST) > {tolerance} THEN 1 ELSE 0 END AS starts_wait FROM {name}_marked AS m ), {name}_numbered AS ( SELECT g.*, - SUM(g.starts_wait) OVER (PARTITION BY {partition} ORDER BY g.wait_start_ms, g.event_time ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS wait_no + SUM(g.starts_wait) OVER (PARTITION BY {partition} ORDER BY g.wait_start_ms NULLS LAST, g.event_time NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS wait_no FROM {name}_gapped AS g ), {name}_ranked AS ( SELECT n.*, - ROW_NUMBER() OVER (PARTITION BY {partition}, n.wait_no ORDER BY n.wait_time_ms DESC, n.event_time DESC) AS wait_rank + ROW_NUMBER() OVER (PARTITION BY {partition}, n.wait_no ORDER BY n.wait_time_ms DESC NULLS LAST, n.event_time DESC NULLS LAST) AS wait_rank FROM {name}_numbered AS n ), {name} AS ( From aa19b70969ca824ed7836056abb2a2203f65f04e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:12:07 -0400 Subject: [PATCH 20/42] Describe the blocking count as waits in the README and the alert setting remark (#5644 W1) --- Darling/README.md | 4 ++-- PerformanceMonitor.Alerting/IAlertEngineSettings.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Darling/README.md b/Darling/README.md index 38b4d27c0c..88d2d2ca57 100644 --- a/Darling/README.md +++ b/Darling/README.md @@ -602,7 +602,7 @@ The shared alert engine's switches and thresholds. Every default mirrors Lite's | `cpuThresholdPercent` | `80` | | | `cpuMode` | `"total"` | `"total"` = SQL + other processes; `"sql"` = SQL process only | | `blockingEnabled` | `true` | | -| `blockingCountThreshold` | `1` | Blocked-process count (rolling window) that trips the alert on a SQL Server target | +| `blockingCountThreshold` | `1` | Blocking waits (rolling window) that trip the alert on a SQL Server target. One wait is one blocked session, blocker and wait start (#5644): the several blocked-process reports SQL Server writes for one long wait count once | | `pgBlockingCountThreshold` | `1` | The PostgreSQL twin (#3444, V122): distinct ROOT blockers in the rolling window that trip `Blocking Detected` on a PostgreSQL target, floor 1. Its own key because the two count different things — engine-recorded blocked-process reports there, roots found in a periodic SAMPLE of `pg_stat_activity` here — so moving one deliberately does not move the other. `blockingEnabled` governs both engines. Inert on a store with no PostgreSQL targets | | `blockingWaitSecondsThreshold` | `0` | Total blocked wait, in seconds, summed across the latest blocking snapshot; `0` = off. A second gate beside the count one, because a count cannot tell one session blocked for an hour from one blocked for a second. Reports as its own "Blocking Wait Time" alert, and unlike the count gate it is level-triggered: it re-fires every cooldown while the wait stays above the threshold and clears when it drops below. Since #3653 the fire sits behind the shared persistence gate as a disjunction: one snapshot at or above **3× the threshold** fires at once, anything between the threshold and that multiple must hold for **3 consecutive collected snapshots** (counted per collection, not per 30 s sweep; a quiet collection between two breaching ones starts a new episode, because the collector writes rows only while something is blocked). The card's `Fired By` field says which arm admitted it (`single_snapshot_3x` / `consecutive_k3`). Measured on one production store class: 97 of 102 episodes were a single snapshot, so a plain consecutive gate would have dropped exactly the severe pile-ups | | `deadlockEnabled` | `true` | | @@ -636,7 +636,7 @@ One alert name means one condition on both engines wherever both engines can hon |---|---|---|---| | **Deadlocks** (alert) | Extended-event graphs, `deadlockCountThreshold` per rolling window | Server-log deadlocks parsed into `pg_deadlocks`, `deadlocks.pg_count_threshold` (V122) | Both. Separate count knobs because the instruments differ (a captured graph vs a parsed log entry) and an operator tuning one should not silently move the other. The SQL Server fire is **graded** since #3653: WARNING by default, CRITICAL when the hour's rate reaches the health band's `deadlock_critical_per_hour` tier below — the count knob decides whether it fires, the band's pair decides the colour, so the alert row and the fleet card agree about the same hour. The PostgreSQL host's fire is graded the same way since #3653 — the same grader, the same one-hour count, the same store tiers — on its own deduped report count (the number the row quotes), not on the counter difference the band row below uses; the read caps at 50 distinct reports, so a Critical tier raised past 50/hr is unreachable on that host. | | **Deadlocks** (health band) | `v_deadlocks` graphs per hour | `pg_stat_database.deadlocks` — the server's own lifetime counter, differenced between consecutive samples per database, clamped at zero across a statistics reset, summed across the cluster's databases | Both, through the **same** `health_bands.deadlock_warn_per_hour` / `deadlock_critical_per_hour` tiers: a deadlock per hour is the same quantity whichever engine recorded it. The PostgreSQL arm bands only when at least one difference was taken in the window (a difference of fewer than two samples is not a zero), so a target nothing has collected from reads Unknown rather than Healthy. The fleet total's `deadlock_coverage.postgres_servers` names how many servers were counted the counter way; `get_pg_deadlocks` has the graphs. Never `SUM(deadlocks)` on the store — the column is a lifetime counter repeated in every one-minute sample. | -| **Blocking** (alert) | Blocked-process reports + DMV snapshots, `blockingCountThreshold` | Distinct root blockers in a periodic sample of `pg_stat_activity`, `blocking.pg_count_threshold` (V122) | Both; separate knobs because the denominators differ (engine-recorded reports vs sampled sightings). The PostgreSQL fire carries an explicit WARNING tier and no CRITICAL tier (#3653): the band row below says why no measured bar exists for sampled sightings, and a fire bar is not a tier. | +| **Blocking** (alert) | Blocking waits counted from blocked-process reports + DMV snapshots (one per blocked session, blocker and wait start), `blockingCountThreshold` | Distinct root blockers in a periodic sample of `pg_stat_activity`, `blocking.pg_count_threshold` (V122) | Both; separate knobs because the denominators differ (engine-recorded reports vs sampled sightings). The PostgreSQL fire carries an explicit WARNING tier and no CRITICAL tier (#3653): the band row below says why no measured bar exists for sampled sightings, and a fire bar is not a tier. | | **Blocking** (health band) | Reports per hour + longest block, the #3596 tiers | **Unknown, by design** | The band's count tiers were measured in *reports per server-hour* on engine-recorded evidence. PostgreSQL blocking is SAMPLED once a minute: a waiter seen in three consecutive captures is one wait observed three times, not three reports, so a sighting count through those tiers would band on a denominator they were never measured against. An honest PostgreSQL band needs its own sampled-shape tiers (share of captures holding a waiter; longest observed wait) measured on that fleet's distribution, which has not been taken. Unknown-with-reason beats a fake Healthy; the alert speaks for PostgreSQL blocking meanwhile. Tiering pending: #3601's `lock_wait` log events (`log_lock_waits` "still waiting" lines, one per wait past `deadlock_timeout`) are the event-grain evidence a report-rate band would read, and the join point for a future PostgreSQL arm. | | **Blocking Wait Time** | Sum of blocked wait across the latest live DMV snapshot, `blockingWaitSecondsThreshold`; since #3653 gated: 3× the bar on one snapshot, or 3 consecutive collections | — | SQL-only. The condition is a *live total* of how long sessions are currently blocked, which the DMV snapshot gives exactly. `pg_blocking` is a per-minute sample of `pg_stat_activity`: summing observed waits across one capture is a lower bound on the same quantity taken at an arbitrary instant, and a threshold in seconds against a once-a-minute sample would fire on the sampling grain as much as on the workload. A sampled twin is possible but needs its own stated denominator; not faked under the SQL Server name. | | **Long-Running Query** | `sys.dm_exec_requests`, five noise opt-outs (sp_server_diagnostics, WAITFOR, backups, misc XE waits, CDC) + `session_id > 50` + `excludedDatabases` + the #3653 opt-out knob (`program_name` prefixes / exact logins, seeded with the SQL Agent job-step prefix and the two `NT AUTHORITY` service logins, applied ahead of the row cap) | Latest `pg_session_states` capture, same threshold and switch | Both. PostgreSQL opt-outs: non-client backends (unconditional — mirrors `session_id > 50`), `VACUUM`/`ANALYZE`/`REINDEX`/`CLUSTER` by command tag (unconditional — SQL Server needs no statement-shape switch because its row carries the text and a mute rule can match it; `pg_session_states` stores no text, so the tag is the only handle), dump/restore utilities by `application_name` on the shared `longRunningQueryExcludeBackups`, the #3653 opt-out knob over `application_name` / `usename` through the same shared predicate builder and ahead of the same row cap (#3743; the SQL Server seeds match nothing here), `excludedDatabases` as the same CTE's third flag ahead of the same cap (#3742). `CREATE` is reported (with its tag) because the tag cannot tell an index build from `CREATE TABLE AS`. The remaining three SQL Server switches name SQL Server internals; `pg_sleep` (WAITFOR's cousin) is invisible without text. The PostgreSQL fire carries an explicit WARNING tier and no CRITICAL tier (#3653): the only bar the alert has is the operator's threshold, and no measured duration distribution exists on either engine to cut a second tier from. | diff --git a/PerformanceMonitor.Alerting/IAlertEngineSettings.cs b/PerformanceMonitor.Alerting/IAlertEngineSettings.cs index 80540f92e3..4339ec50fe 100644 --- a/PerformanceMonitor.Alerting/IAlertEngineSettings.cs +++ b/PerformanceMonitor.Alerting/IAlertEngineSettings.cs @@ -96,7 +96,7 @@ public interface IAlertEngineSettings /// Fire when the selected CPU metric (see ) is at/above this %. int CpuThresholdPercent { get; } - /// Fire when the rolling-window blocked-process-report count reaches this value (count-based; see class remarks). + /// Fire when the rolling-window blocking-WAIT count (one per blocked session, blocker and wait start, not one per blocked-process report; #5644) reaches this value (count-based; see class remarks). int BlockingCountThreshold { get; } /// From 637385cbe045abae028345f623b2f7a0557b82b7 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:23:34 -0400 Subject: [PATCH 21/42] BlockingWaitCountTests joins the live-postgres collection: its two live tests reach the shared store (#5644) --- Darling/Darling.Tests/BlockingWaitCountTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Darling/Darling.Tests/BlockingWaitCountTests.cs b/Darling/Darling.Tests/BlockingWaitCountTests.cs index d40b13835f..5117ff760b 100644 --- a/Darling/Darling.Tests/BlockingWaitCountTests.cs +++ b/Darling/Darling.Tests/BlockingWaitCountTests.cs @@ -24,6 +24,7 @@ namespace Darling.Tests; /// each carrying the wait time so far, with a few milliseconds of jitter between the event timestamp and the wait /// time (the wait start is derived, so it drifts by that much between reports of one wait). /// +[Collection("live-postgres")] public sealed class BlockingWaitCountTests { private static readonly DateTime T0 = new(2026, 10, 9, 9, 0, 0, DateTimeKind.Unspecified); From 9993bf0ba4063e90d11a999d2f02a1c0307b3df1 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:31:15 -0400 Subject: [PATCH 22/42] Lite tests: the Blocked helper sets BlockedSpid once, explicit spid wins else a unique one (#5644) --- Lite.Tests/BlockingDeadlockContextBuilderTests.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Lite.Tests/BlockingDeadlockContextBuilderTests.cs b/Lite.Tests/BlockingDeadlockContextBuilderTests.cs index 5ab30c6561..c3d2fede40 100644 --- a/Lite.Tests/BlockingDeadlockContextBuilderTests.cs +++ b/Lite.Tests/BlockingDeadlockContextBuilderTests.cs @@ -47,11 +47,10 @@ private static BlockedProcessAlertRow Blocked( string db = "StackOverflow", string obj = "", string blockedSql = "", string blockingSql = "", long waitMs = 5000, string lockMode = "", string xml = "", DateTime? eventTime = null, int spid = 0) => new() { - /* #5644 W1: rows with one blocked spid, blocker and start are ONE wait; a test that wants separate - waits gives each its own spid. */ - BlockedSpid = spid, + /* #5644 W1: rows with one blocked spid, blocker and start are ONE wait. A test that names a spid gets it + (rows given the same spid are one wait); otherwise each row gets a unique spid and is its own wait. */ + BlockedSpid = spid != 0 ? spid : System.Threading.Interlocked.Increment(ref s_nextBlockedSpid), EventTime = eventTime ?? new DateTime(2026, 6, 19, 12, 0, 0), - BlockedSpid = System.Threading.Interlocked.Increment(ref s_nextBlockedSpid), BlockingSpid = 50, DatabaseName = db, ContentiousObject = obj, From edd7c3bdb8fc79585634ce892de018bdde725ef7 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:37:52 -0400 Subject: [PATCH 23/42] Web grids localize ISO instants only in opted-in message columns, never in SQL text or details (#5644 W20) --- .../WebClampCellBehaviourTests.cs | 26 +++++++++++++++++++ .../Darling.Tests/web-clamp-cell-harness.mjs | 22 ++++++++++++++++ .../wwwroot/js/pages/server-tabs.js | 16 ++++++------ .../wwwroot/js/panels.js | 6 +++-- .../wwwroot/js/read-fields.js | 8 +++--- 5 files changed, 64 insertions(+), 14 deletions(-) diff --git a/Darling/Darling.Tests/WebClampCellBehaviourTests.cs b/Darling/Darling.Tests/WebClampCellBehaviourTests.cs index 494ae1e03f..f7d422dc70 100644 --- a/Darling/Darling.Tests/WebClampCellBehaviourTests.cs +++ b/Darling/Darling.Tests/WebClampCellBehaviourTests.cs @@ -87,6 +87,32 @@ public void ALongCell_OpensAndClosesOnAClick_WithoutActivatingTheRow() Assert.False(second.GetProperty("open").GetBoolean()); } + [Fact] + public void OnlyAnOptedInMessageColumn_LocalizesAnIsoInstant_SqlTextAndDetailsPrintAsWritten() + { + // #5644 W20: a query_text of WHERE d >= '2026-01-01T00:00:00' showed as a local date in the browser. + var l = Run().GetProperty("localize"); + Assert.True(l.GetProperty("sqlUnchanged").GetBoolean()); + Assert.True(l.GetProperty("detailsUnchanged").GetBoolean()); + var expected = l.GetProperty("expected").GetString(); + Assert.Equal(expected, l.GetProperty("messageLocal").GetString()); + Assert.Equal(expected, l.GetProperty("clampedLocal").GetString()); + Assert.DoesNotContain("T14:37", expected); + } + + [Fact] + public void TheCollectionHealthMessageColumns_AreOptedIn_AndTheCustomViewAndDefaultTraceTextColumnsAreNot() + { + var tabs = Wwwroot("js", "pages", "server-tabs.js"); + var health = tabs[tabs.IndexOf("key: \"run_time_note\"", StringComparison.Ordinal)..tabs.IndexOf("const HEAVIEST_COLUMNS", StringComparison.Ordinal)]; + Assert.Equal(7, Regex.Matches(health, "clamp: true, localizeText: true").Count); + foreach (var f in new[] { "js/read-fields.js", "js/pages/server-tabs.js" }) + { + var src = Wwwroot(f.Split('/')); + Assert.DoesNotMatch(new Regex("key: \"(query_text|statement_text|query_sql_text|sql_text|details)\"[^\n]*localizeText"), src); + } + } + [Fact] public void EnterOpensTheCell_AndOtherKeysDoNot() { diff --git a/Darling/Darling.Tests/web-clamp-cell-harness.mjs b/Darling/Darling.Tests/web-clamp-cell-harness.mjs index b0edee3eba..e8e0bedfa7 100644 --- a/Darling/Darling.Tests/web-clamp-cell-harness.mjs +++ b/Darling/Darling.Tests/web-clamp-cell-harness.mjs @@ -86,4 +86,26 @@ out.notExpandable = { role: fixed.attrs.role || null, text: fixed.textContent == out.empty = { text: clampText(null).textContent, title: clampText(null).attrs.title || null }; out.multiline = clampText("one\ntwo").attrs.role; +/* #5644 W20: only a column marked localizeText rewrites an ISO instant in its text; SQL text and details are data and print as written. */ +const sqlText = "SELECT 1 FROM t WHERE d >= '2026-01-01T00:00:00'"; +const msgText = "not collected since 2026-10-09T14:37:35Z"; +const l = VIZ.table({ rows: [{ query_text: sqlText, details: msgText, message: msgText, collection_time: "2026-10-09T14:37:35Z" }] }, { + rowsKey: "rows", title: "Localize", tools: false, + columns: [ + { key: "query_text", label: "Query", wrap: true }, + { key: "details", label: "Details", wrap: true }, + { key: "message", label: "Message", wrap: true, localizeText: true }, + { key: "message", label: "Clamped message", clamp: true, localizeText: true }, + ], +}); +const lr = l.findAll((n) => n.tag === "tr").filter((n) => n.parent && n.parent.tag === "tbody")[0]; +const ltd = lr.children.map((c) => c.textContent); +out.localize = { + sqlUnchanged: ltd[0] === sqlText, + detailsUnchanged: ltd[1] === msgText, + messageLocal: ltd[2], + clampedLocal: ltd[3], + expected: "not collected since " + new Date("2026-10-09T14:37:35Z").toLocaleString(), +}; + console.log(JSON.stringify(out)); diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js index 19e3739094..6d9d074323 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js @@ -4807,7 +4807,7 @@ const COLLECTION_LOG_COLUMNS = [ { key: "sql_store_ms", label: "Store (in SQL)", format: "ms" }, { key: "store_duration_ms", label: "On Store", format: "ms" }, { key: "rows_collected", label: "Rows", format: "int" }, - { key: "error_message", label: "Error", wrap: true }, + { key: "error_message", label: "Error", wrap: true, localizeText: true }, ]; /* #5227: the collector picked on the Collectors panel, keyed by server at MODULE scope so the 60 s rebuild of the tab @@ -4898,8 +4898,8 @@ const COLLECTOR_COLUMNS = [ time. Each column is left out unless a row fills it, so a server with no run times shows none of them. */ { key: "run_at", label: "Run at", hideWhenEmpty: true }, { key: "next_run_utc", label: "Next run", format: "time", hideWhenEmpty: true }, - { key: "run_time_note", label: "Run time", clamp: true, hideWhenEmpty: true }, - { key: "last_error", label: "Last Error", clamp: true, plain: "named" }, + { key: "run_time_note", label: "Run time", clamp: true, localizeText: true, hideWhenEmpty: true }, + { key: "last_error", label: "Last Error", clamp: true, localizeText: true, plain: "named" }, /* #1837: what a NON-failing run reported (an enumeration that came back with 0 items). Blank for a plainly healthy collector; the same column the two WPF grids carry, so the web view is not the one Collection Health surface that still hides it. note_summary, not the raw last_note: it carries the @@ -4907,24 +4907,24 @@ const COLLECTOR_COLUMNS = [ one, composed server-side from the shared formatter so this table cannot render it a third way. A collector's label=value counts read as its latest NOTED run's ("latest run:" only when every run carried them), never a window total. */ - { key: "note_summary", label: "Note", clamp: true, plain: true }, + { key: "note_summary", label: "Note", clamp: true, localizeText: true, plain: true }, /* #5646: a collector that recovered (its newest 3 runs succeeded after failures) is HEALTHY, and this says so with the time of the last failure. Partial rows carry no note_summary, so the sentence rides in its own field. */ - { key: "recovered_note", label: "Recovered", clamp: true, plain: true, hideWhenEmpty: true }, + { key: "recovered_note", label: "Recovered", clamp: true, localizeText: true, plain: true, hideWhenEmpty: true }, /* #3017: which of the two zero-output readings a collector that spent and stored nothing is — read and found nothing, or could not read. Blank whenever Rows is positive, for the same reason the Note column is blank on a plainly healthy collector. Composed server-side from the shared formatter, so this table cannot render the sentence a second way. */ - { key: "output_finding", label: "Output", clamp: true, plain: true }, + { key: "output_finding", label: "Output", clamp: true, localizeText: true, plain: true }, /* #4620: why a collector that stopped doing what it used to do reads WARNING. The Status column already showed the floor, but no column showed the sentence behind it, so a regressed row read WARNING with every other cell blank. Composed server-side from the shared formatter, like the two columns above. */ - { key: "regression_finding", label: "Regression", clamp: true, plain: true }, + { key: "regression_finding", label: "Regression", clamp: true, localizeText: true, plain: true }, /* #5249: a PostgreSQL collector this server's engine does not collect (a collector this engine kind can never run: an Aurora-only reader on stock PostgreSQL, the sampler on Aurora) has no log row, so the server lists it with status not_collected and the sentence saying why. The status paints with the neutral Unknown band, not an error one: a gate is not a fault. Left out unless a row has a message, so a server with no such collector (and every SQL Server tab) shows no column of dashes. */ - { key: "message", label: "Why not collected", clamp: true, hideWhenEmpty: true, plain: true }, + { key: "message", label: "Why not collected", clamp: true, localizeText: true, hideWhenEmpty: true, plain: true }, ]; const HEAVIEST_COLUMNS = [ diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js index a5ae8a0355..72cc8a7462 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js @@ -1251,8 +1251,10 @@ function cell(row, c) { : c.plain ? plainText(String(raw)) : String(raw); - /* #5644 W20: a column with no format shows any ISO instant in its text (a bare timestamp or one inside a sentence) in local time, like the grids beside it. */ - const text = c.format ? shown : localizeIsoText(shown); + /* #5644 W20: a column marked `localizeText` (a message or note: collection-health notes, collector errors) shows any ISO + instant in its sentence in local time, like the grids beside it. Opt-in: query text, statement text, details and + identifiers are data, and a literal such as '2026-01-01T00:00:00' inside them must show as written. */ + const text = c.localizeText && !c.format ? localizeIsoText(shown) : shown; if (c.clamp) return el("td", { class: cls.join(" ") || null }, [clampText(text)]); return el("td", { class: cls.join(" ") || null, text }); } diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/read-fields.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/read-fields.js index a221785446..9a5f090b3e 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/read-fields.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/read-fields.js @@ -89,9 +89,9 @@ export const READ_FIELDS = { { key: "errors", label: "Errors", format: "int" }, { key: "avg_duration_ms", label: "Avg Dur", format: "ms" }, { key: "last_success", label: "Last Success", format: "time" }, - { key: "note_summary", label: "Note", wrap: true, plain: true }, - { key: "recovered_note", label: "Recovered", wrap: true, plain: true, hideWhenEmpty: true }, - { key: "message", label: "Why not collected", wrap: true, plain: true }, + { key: "note_summary", label: "Note", wrap: true, localizeText: true, plain: true }, + { key: "recovered_note", label: "Recovered", wrap: true, localizeText: true, plain: true, hideWhenEmpty: true }, + { key: "message", label: "Why not collected", wrap: true, localizeText: true, plain: true }, ], }, }, @@ -365,7 +365,7 @@ export const READ_FIELDS = { { key: "status", label: "Status", statusSev: true }, { key: "duration_ms", label: "Total", format: "ms" }, { key: "rows_collected", label: "Rows", format: "int" }, - { key: "error_message", label: "Error", wrap: true }, + { key: "error_message", label: "Error", wrap: true, localizeText: true }, ], }, }, From ff012eb33a3d4201958f9e422460c8fe17962b34 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:52:49 -0400 Subject: [PATCH 24/42] The three Default Trace readers dedup on the same full column set, filename and event_class included (#5644 W6) --- ...rlingDefaultTraceReaderServerClockTests.cs | 21 +++++++++++ .../StoredServerClockRowTextTests.cs | 6 ++-- .../ViewerServerClockEventReadsTests.cs | 26 ++++++++++++-- .../ViewerSystemEventsDataStartTests.cs | 4 ++- .../ViewerDataService.SystemEvents.cs | 10 ++++-- Lite.Tests/DefaultTraceServerClockTests.cs | 36 +++++++++++++++++++ .../Services/LocalDataService.SystemEvents.cs | 8 +++-- 7 files changed, 101 insertions(+), 10 deletions(-) diff --git a/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs b/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs index 95797418da..d192962816 100644 --- a/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs +++ b/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs @@ -81,6 +81,27 @@ public async Task TheSameEventStoredSeveralTimes_ComesBackOnce_AndADifferentEven Assert.Equal(new[] { "same", "other" }, rows.Select(r => r.DatabaseName).ToArray()); } + [Fact] + public async Task TwoAutoGrowsForDifferentFilesInOneTick_AreTwoEvents_AsInTheViewerAndLite() + { + /* #5644 W6: the viewer, Lite and this reader key on the same full column set, filename (9) and event_class (2) + included. */ + var table = new DataTable(); + for (var i = 0; i < 15; i++) + { + table.Columns.Add("c" + i, i == 0 ? typeof(DateTime) : i == 2 ? typeof(int) : typeof(string)); + } + + foreach (var (file, cls) in new[] { ("a.mdf", 92), ("b.mdf", 92), ("b.mdf", 93), ("a.mdf", 92) }) + { + AddRow(table, (0, Naive(2026, 3, 9, 8)), (1, "Data File Auto Grow"), (2, cls), (4, "grow"), (9, file)); + } + + var rows = await ReadAsync(table, Eastern(), Naive(2026, 1, 1, 0), Naive(2026, 12, 31, 0)); + + Assert.Equal(3, rows.Count); + } + [Fact] public async Task AnEventInASkippedOrRepeatedLocalHour_DoesNotThrow() { diff --git a/Darling/Darling.Tests/StoredServerClockRowTextTests.cs b/Darling/Darling.Tests/StoredServerClockRowTextTests.cs index 4d0bef1b4b..5df4f3c0cb 100644 --- a/Darling/Darling.Tests/StoredServerClockRowTextTests.cs +++ b/Darling/Darling.Tests/StoredServerClockRowTextTests.cs @@ -196,6 +196,8 @@ private static DataTable NewDefaultTraceTable() table.Columns.Add("severity", typeof(int)); table.Columns.Add("error_number", typeof(int)); table.Columns.Add("text_data", typeof(string)); + table.Columns.Add("filename", typeof(string)); + table.Columns.Add("event_class", typeof(int)); return table; } @@ -210,11 +212,11 @@ public async Task ADefaultTraceEventReadFromTheStoredWallClock_PrintsNoOffset_In foreach (var label in new[] { "first-pass", "second-pass" }) { table.Rows.Add(Naive(2026, 11, 1, 1, 30), "Data File Auto Grow", label, DBNull.Value, DBNull.Value, DBNull.Value, - DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value); + DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value); } table.Rows.Add(Naive(2026, 11, 1, 2, 30), "Data File Auto Grow", "after", DBNull.Value, DBNull.Value, DBNull.Value, - DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value); + DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value); List rows; using (var reader = table.CreateDataReader()) diff --git a/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs b/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs index 141766a659..04d4b1789c 100644 --- a/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs +++ b/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs @@ -54,20 +54,24 @@ private static DataTable NewTable() table.Columns.Add("severity", typeof(int)); table.Columns.Add("error_number", typeof(int)); table.Columns.Add("text_data", typeof(string)); + table.Columns.Add("filename", typeof(string)); + table.Columns.Add("event_class", typeof(int)); return table; } /// One stored event. rides in database_name so a test can say which /// event came back; the default name is a Default Trace category that is significant as collected. private static void AddEvent(DataTable table, DateTime? local, string label, - string eventName = "Data File Auto Grow", int? severity = null) + string eventName = "Data File Auto Grow", int? severity = null, string? filename = null, int? eventClass = null) { table.Rows.Add( local.HasValue ? (object)local.Value : DBNull.Value, eventName, label, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, 55, 1_500_000L, 256L, severity.HasValue ? (object)severity.Value : DBNull.Value, - DBNull.Value, DBNull.Value); + DBNull.Value, DBNull.Value, + filename is null ? DBNull.Value : filename, + eventClass.HasValue ? (object)eventClass.Value : DBNull.Value); } private static async Task> ReadAsync(DataTable table, ServerClock clock, DateTime startUtc, DateTime endUtc) @@ -119,6 +123,24 @@ public async Task StoredCopiesOfOneDefaultTraceEvent_ComeBackOnce_AndNearCopiesS Assert.Single(rows, r => r.DatabaseName == "other-database"); } + /// + /// #5644 W6: the three Default Trace readers key on the same full column set. Two Data File Auto Grow rows for + /// different files in one tick (or of different classes) are two events here, as in the MCP reader and Lite. + /// + [Fact] + public async Task TwoAutoGrowsForDifferentFilesInOneTick_AreTwoEvents_AndTrueCopiesStayOne() + { + var table = NewTable(); + AddEvent(table, Naive(2026, 3, 9, 8), "grow", filename: "a.mdf", eventClass: 92); + AddEvent(table, Naive(2026, 3, 9, 8), "grow", filename: "b.mdf", eventClass: 92); + AddEvent(table, Naive(2026, 3, 9, 8), "grow", filename: "b.mdf", eventClass: 93); + AddEvent(table, Naive(2026, 3, 9, 8), "grow", filename: "a.mdf", eventClass: 92); /* a true copy */ + + var rows = await ReadAsync(table, Eastern(), Naive(2026, 1, 1, 0), Naive(2026, 12, 31, 0)); + + Assert.Equal(3, rows.Count); + } + [Fact] public async Task ADefaultTraceEventInASkippedOrRepeatedLocalHour_DoesNotThrow() { diff --git a/Darling/Darling.Tests/ViewerSystemEventsDataStartTests.cs b/Darling/Darling.Tests/ViewerSystemEventsDataStartTests.cs index ed0b67efbc..944c21d8d7 100644 --- a/Darling/Darling.Tests/ViewerSystemEventsDataStartTests.cs +++ b/Darling/Darling.Tests/ViewerSystemEventsDataStartTests.cs @@ -157,9 +157,11 @@ private static DateTime Naive(int year, int month, int day, int hour, int minute table.Columns.Add("severity", typeof(int)); table.Columns.Add("error_number", typeof(int)); table.Columns.Add("text_data", typeof(string)); + table.Columns.Add("filename", typeof(string)); + table.Columns.Add("event_class", typeof(int)); foreach (var local in storedLocal) { - table.Rows.Add(local, "Data File Auto Grow", "db", DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value); + table.Rows.Add(local, "Data File Auto Grow", "db", DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value); } using var reader = table.CreateDataReader(); diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs index efe6b3f15d..530a1e35f1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs @@ -647,7 +647,9 @@ ORDER BY sp.collection_time DESC dte.integer_data, dte.severity, dte.error_number, - dte.text_data + dte.text_data, + dte.filename, + dte.event_class FROM default_trace_events AS dte, svr WHERE dte.server_id = $1 AND dte.event_time - make_interval(mins => svr.offset_minutes) >= $2 - interval '1 hour' @@ -695,7 +697,7 @@ internal static async Task> ReadDefaultTraceEventsAsy DbDataReader reader, ServerClock clock, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken) { var rows = new List(); - var seen = new HashSet<(DateTime?, string?, string?, string?, string?, string?, string?, int?, long?, long?, int?, int?, string?)>(); + var seen = new HashSet<(DateTime?, string?, string?, string?, string?, string?, string?, int?, long?, long?, int?, int?, string?, string?, int?)>(); while (await reader.ReadAsync(cancellationToken)) { @@ -719,6 +721,8 @@ internal static async Task> ReadDefaultTraceEventsAsy var integerData = reader.IsDBNull(9) ? (long?)null : reader.GetInt64(9); var errorNumber = reader.IsDBNull(11) ? (int?)null : reader.GetInt32(11); var textData = reader.IsDBNull(12) ? null : reader.GetString(12); + var filename = reader.IsDBNull(13) ? null : reader.GetString(13); + var eventClass = reader.IsDBNull(14) ? (int?)null : reader.GetInt32(14); /* #5644 W6: the store holds rows that are the same event twice (an older build kept both the Begin and the Commit of every DDL, and re-stored an event whose datetime StartTime did not round-trip through @@ -726,7 +730,7 @@ internal static async Task> ReadDefaultTraceEventsAsy collector stores tells them apart, so the first of each in reader order is kept, as the MCP reader and Lite's do, and the copies stop filling the grid and the page limit. */ if (!seen.Add((eventTimeUtc, eventName, databaseName, objectName, loginName, hostName, applicationName, - spid, durationUs, integerData, severity, errorNumber, textData))) + spid, durationUs, integerData, severity, errorNumber, textData, filename, eventClass))) continue; rows.Add(new DefaultTraceEventRow( diff --git a/Lite.Tests/DefaultTraceServerClockTests.cs b/Lite.Tests/DefaultTraceServerClockTests.cs index 64f2d64424..cbc49df646 100644 --- a/Lite.Tests/DefaultTraceServerClockTests.cs +++ b/Lite.Tests/DefaultTraceServerClockTests.cs @@ -78,6 +78,42 @@ INSERT INTO default_trace_events await cmd.ExecuteNonQueryAsync(); } + private async Task SeedAutoGrowAsync(DateTime eventTimeServerLocal, string filename, int eventClass) + { + using var readLock = _duckDb.AcquireReadLock(); + var conn = await SeedConnectionAsync(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = @" +INSERT INTO default_trace_events + (default_trace_event_id, collection_time, server_id, server_name, event_time, event_name, event_class, + database_name, filename, duration_us, integer_data, severity, error_number, text_data) +VALUES ($1, $2, $3, 'TestSrv', $4, 'Data File Auto Grow', $5, 'grow', $6, 1500000, 256, NULL, NULL, NULL)"; + cmd.Parameters.Add(new DuckDBParameter { Value = _nextId++ }); + cmd.Parameters.Add(new DuckDBParameter { Value = DateTime.UtcNow }); + cmd.Parameters.Add(new DuckDBParameter { Value = ServerId }); + cmd.Parameters.Add(new DuckDBParameter { Value = eventTimeServerLocal }); + cmd.Parameters.Add(new DuckDBParameter { Value = eventClass }); + cmd.Parameters.Add(new DuckDBParameter { Value = filename }); + await cmd.ExecuteNonQueryAsync(); + } + + [Fact] + public async Task DefaultTrace_AutoGrowsForDifferentFilesInOneTick_AreTwoEvents_AndTrueCopiesStayOne() + { + /* #5644 W6: Lite, the viewer and the MCP reader key on the same full column set (filename and event_class too). */ + var service = new LocalDataService(_duckDb); + var tick = At(2026, 3, 9, 8, 0); + await SeedAutoGrowAsync(tick, "a.mdf", 92); + await SeedAutoGrowAsync(tick, "b.mdf", 92); + await SeedAutoGrowAsync(tick, "b.mdf", 93); + await SeedAutoGrowAsync(tick, "a.mdf", 92); /* a true copy */ + + var rows = await service.GetDefaultTraceEventsAsync( + ServerId, fromDate: At(2026, 3, 9, 0, 0), toDate: At(2026, 3, 10, 0, 0), serverClock: Eastern()); + + Assert.Equal(3, rows.Count(r => r.EventName == "Data File Auto Grow")); + } + // ── System Events (Default Trace): server-local event_time converted per row ── [Fact] diff --git a/Lite/Services/LocalDataService.SystemEvents.cs b/Lite/Services/LocalDataService.SystemEvents.cs index 584cb0e18b..b10e2fe94d 100644 --- a/Lite/Services/LocalDataService.SystemEvents.cs +++ b/Lite/Services/LocalDataService.SystemEvents.cs @@ -840,7 +840,9 @@ the same clock and held to the exact window below (#4766). Bind db filter params /* #5644 W6: DISTINCT drops the stored copies of one event (the Begin and Commit rows of a DDL that an older build kept, and an event re-stored by the datetime watermark round trip): they match on every column - selected here, and only the event_sequence the collector stores tells them apart. */ + selected here, and only the event_sequence the collector stores tells them apart. filename and event_class are + selected for the same reason as the MCP reader and the viewer key on them (the grid does not show them): two + Data File Auto Grow rows for different files in one tick are two events. */ command.CommandText = @" SELECT DISTINCT event_time, @@ -855,7 +857,9 @@ SELECT DISTINCT integer_data, severity, error_number, - text_data + text_data, + filename, + event_class FROM v_default_trace_events WHERE server_id = $1 AND event_time >= $2 From 4360ce2ad2a8887c02c5c621fb09039b38ccc54d Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:59:52 -0400 Subject: [PATCH 25/42] Chart gap threshold is computed per series, so a coarse overlay is one line (#5644 W7) --- .../Darling.Tests/WebChartGapBehaviourTests.cs | 15 +++++++++++++++ Darling/Darling.Tests/web-chart-gap-harness.mjs | 14 ++++++++++++++ .../wwwroot/js/charts.js | 11 +++++++---- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/Darling/Darling.Tests/WebChartGapBehaviourTests.cs b/Darling/Darling.Tests/WebChartGapBehaviourTests.cs index 67c216e9af..7219783d60 100644 --- a/Darling/Darling.Tests/WebChartGapBehaviourTests.cs +++ b/Darling/Darling.Tests/WebChartGapBehaviourTests.cs @@ -146,6 +146,21 @@ public void AMissedCycle_BreaksOnlyPastTheFloorAndThreeTimesTheMedianInterval() Assert.Equal(2, N(r, "slowCollectorBreak", "lines")); } + [Fact] + public void TheGapThreshold_IsPerSeries_ACoarseSeriesOverFineBucketsIsOneLineUnlessItHasAGapOfItsOwn() + { + // 1-minute buckets, a series sampled every 15 minutes: the rows' median interval is 1 minute (threshold 5 minutes), which + // used to cut the 15-minute series into a dot per sample. Its own spacing makes the threshold 45 minutes. + var r = Run(); + Assert.Equal(1, N(r, "coarseOverlayNoGap", "overlays")); + Assert.Equal(0, N(r, "coarseOverlayNoGap", "dots")); + Assert.Equal(1, N(r, "coarseOverlayNoGap", "lines")); + Assert.Equal(2, N(r, "coarseOverlayGap", "overlays")); // a real 7 hour silence still breaks it + Assert.Equal(0, N(r, "coarseOverlayGap", "dots")); + Assert.Equal(2, N(r, "coarsePrimary", "lines")); + Assert.Equal(0, N(r, "coarsePrimary", "dots")); + } + [Fact] public void TheThreshold_HasNothingToCompareUnderThreeBuckets_AndNeverFallsUnderTheFloor() { diff --git a/Darling/Darling.Tests/web-chart-gap-harness.mjs b/Darling/Darling.Tests/web-chart-gap-harness.mjs index 432b8a9f42..624d459621 100644 --- a/Darling/Darling.Tests/web-chart-gap-harness.mjs +++ b/Darling/Darling.Tests/web-chart-gap-harness.mjs @@ -147,6 +147,20 @@ out.bigMiss = marks(draw(seq(0, 30).concat(seq(37, 60)), { series: [{ key: "a", out.slowCollector = marks(draw(seq(0, 300, 15).concat(seq(330, 600, 15)), { series: [{ key: "a", label: "A", color: "#f00" }] })); out.slowCollectorBreak = marks(draw(seq(0, 300, 15).concat(seq(420, 600, 15)), { series: [{ key: "a", label: "A", color: "#f00" }] })); +/* The threshold is per series (#5644 review): a 15 minute overlay on 1 minute buckets is one line when it has no gap of its own, + and breaks when it does. The overlay c is set only at every 15th minute. */ +const sparseC = (minutes) => minutes.map((m, i) => ({ t: iso(T0 + m * MIN), a: 10 + (i % 7), c: m % 15 === 0 ? 5 + (i % 3) : null })); +const drawSparse = (minutes) => + charts.renderLineChart({ points: sparseC(minutes), xKey: "t", series: [{ key: "a", label: "A", color: "#f00" }], series2: { key: "c", label: "C", color: "#00f" } }); +out.coarseOverlayNoGap = marks(drawSparse(noGap)); +out.coarseOverlayGap = marks(drawSparse(withGap)); +/* a coarse PRIMARY series next to a fine one on the same rows */ +out.coarsePrimary = marks(charts.renderLineChart({ + points: sparseC(noGap).map((r) => ({ ...r, b: 1 })), + xKey: "t", + series: [{ key: "a", label: "A", color: "#f00" }, { key: "c", label: "C", color: "#0f0" }], +})); + out.threshold = { empty: String(charts.gapThresholdMs([])), one: String(charts.gapThresholdMs([0])), diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js index 890eef8a54..27f57a88e4 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js @@ -562,8 +562,10 @@ export function renderLineChart(spec) { if (v == null) continue; plottable.push(i); } - /* One run per stretch with samples (#5644 W7): a collection gap ends a run, so no line or fill crosses it. */ - for (const run of splitAtGaps(plottable, rowTimes, gapMs)) { + /* One run per stretch with samples (#5644 W7): a collection gap ends a run, so no line or fill crosses it. The + threshold comes from THIS series' own spacing: a 15-minute series on a chart of 1-minute buckets is not a + run of gaps. */ + for (const run of splitAtGaps(plottable, rowTimes, gapThresholdMs(plottable.map((i) => rowTimes[i])))) { const linePts = run.map((i) => xs[i] + "," + plotY(readVal(rows[i].r, s.key))); /* A lone plottable point has no segment to stroke, so draw it as a dot — one bucket still shows its reading. (Its resting marker matches the hover dot; static class so it does not vanish on mouseout.) */ @@ -599,8 +601,9 @@ export function renderLineChart(spec) { if (v == null || isNaN(v)) continue; plottable2.push(i); } - /* Broken at collection gaps like a primary line (#5644 W7). */ - for (const run of splitAtGaps(plottable2, rowTimes, gapMs)) { + /* Broken at collection gaps like a primary line (#5644 W7), by its own spacing (a 15-minute overlay over + 1-minute buckets draws one line, not a dot per sample). */ + for (const run of splitAtGaps(plottable2, rowTimes, gapThresholdMs(plottable2.map((i) => rowTimes[i])))) { const pts2 = run.map((i) => xs[i] + "," + Math.max(M.t, Math.min(M.t + PLOT_H, scaleY2(Number(rows[i].r[series2.key]))))); if (pts2.length === 1) { /* Same single-bucket rule as the primary series: a lone overlay reading draws as a dot, not a dropped From 77a525aa290f6e354732f066b41e9c5751c5c912 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:02:46 -0400 Subject: [PATCH 26/42] Default Trace watermark rides on the trace's EventSequence, keyed to the trace identity, so late-written grow/shrink stalls are stored (#5644 W6) --- Darling/Darling.Tests/Darling.Tests.csproj | 4 + ...DefaultTraceEventSequenceWatermarkTests.cs | 566 ++++++++++++++++++ ...aultTraceEventsCollectorDefinitionTests.cs | 17 +- .../DefaultTraceEventsCollector.cs | 196 +++++- 4 files changed, 765 insertions(+), 18 deletions(-) create mode 100644 Lite.Tests/DefaultTraceEventSequenceWatermarkTests.cs diff --git a/Darling/Darling.Tests/Darling.Tests.csproj b/Darling/Darling.Tests/Darling.Tests.csproj index b7060c7a9e..69881add18 100644 --- a/Darling/Darling.Tests/Darling.Tests.csproj +++ b/Darling/Darling.Tests/Darling.Tests.csproj @@ -57,6 +57,10 @@ brings its Lite.Tests namespace with it and uses only the BCL, xunit and PerformanceMonitor.Alerting, which both projects carry. --> + + diff --git a/Lite.Tests/DefaultTraceEventSequenceWatermarkTests.cs b/Lite.Tests/DefaultTraceEventSequenceWatermarkTests.cs new file mode 100644 index 0000000000..85cacd770a --- /dev/null +++ b/Lite.Tests/DefaultTraceEventSequenceWatermarkTests.cs @@ -0,0 +1,566 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// #5644 W6: the default trace watermark rides on the trace's EventSequence, not only on +/// StartTime. A grow/shrink stall carries the time it BEGAN but reaches the trace file when it ENDS, +/// after a poll has stored later-starting events and moved the StartTime watermark past it; "EventSequence +/// greater than the last one stored" reads it whenever it is written, and a trace restart (sequence back to 1) +/// is told apart by the trace's identity, sys.traces.start_time. +/// +/// The SQL cannot run here, so is a model of the monitored server's default trace +/// and of the host's store and collector state. It evaluates the query's WHERE in C# from the parameters the +/// REAL binds, feeds the REAL +/// a two-result-set reader, stores what it keeps and +/// persists PendingState the way both hosts do (Lite: DuckDB collector_state; Darling: PG +/// collect.collector_state, through the same StateKeys/PendingState plumbing). The text +/// pins at the end hold the SQL to the rule the model evaluates. The same file runs in Lite.Tests and, linked, +/// in Darling.Tests: the collector is one shared definition. +/// +public sealed class DefaultTraceEventSequenceWatermarkTests +{ + private static readonly DateTime s_trace1Start = new(2026, 10, 12, 11, 0, 0, DateTimeKind.Unspecified); + private static readonly DateTime s_noon = new(2026, 10, 12, 12, 0, 0, DateTimeKind.Unspecified); + private const string File5 = @"S:\MSSQL\Log\log_5.trc"; + private const string File6 = @"S:\MSSQL\Log\log_6.trc"; + private const string File7 = @"S:\MSSQL\Log\log_7.trc"; + private const string Grow = "Log File Auto Grow"; + + private sealed record TraceEvent(string File, long Sequence, DateTime Start, string Name, long DurationUs); + + private sealed class NoDeltas : ICollectorDeltaCalculator + { + public long CalculateDelta(int serverId, string collectorName, string key, long currentValue, + DateTime? collectionTime = null, int maxGapSeconds = 0) => currentValue; + + public long CalculateDeltaWithInterval(int serverId, string collectorName, string key, long currentValue, + out int intervalSeconds, DateTime? collectionTime = null, int maxGapSeconds = 0) + { + intervalSeconds = 0; + return currentValue; + } + } + + /// The monitored server's default trace plus the host's store and persisted collector state. + private sealed class TraceHost + { + public DateTime TraceStart = s_trace1Start; + public string CurrentFile = File5; + public long NextSequence = 1; + + /// Every event in the rollover set, in file order, older traces' files included. + public List Written = new(); + + /// Rows the host stored: (EventSequence, StartTime, name). + public List<(long Sequence, DateTime Start, string Name)> Store = new(); + + public Dictionary State = new(StringComparer.Ordinal); + + public DateTime? Watermark => Store.Count == 0 ? null : Store.Max(r => r.Start); + + public TraceEvent Write(DateTime start, string name = "ErrorLog", long durationUs = 0) + { + var ev = new TraceEvent(CurrentFile, NextSequence++, start, name, durationUs); + Written.Add(ev); + return ev; + } + + public void Rollover(string newFile) => CurrentFile = newFile; + + public void RestartTrace(DateTime newStart, string newFile) + { + TraceStart = newStart; + CurrentFile = newFile; + NextSequence = 1; + } + + public TraceHost Clone() => new() + { + TraceStart = TraceStart, + CurrentFile = CurrentFile, + NextSequence = NextSequence, + Written = new List(Written), + Store = new List<(long, DateTime, string)>(Store), + State = new Dictionary(State, StringComparer.Ordinal), + }; + + public CollectorContext NewContext() => new() + { + ServerId = 42, + ServerName = "test-server", + CollectionTime = new DateTime(2026, 10, 12, 12, 0, 0, DateTimeKind.Utc), + Deltas = new NoDeltas(), + Target = new CollectorTargetInfo(), + Watermark = Watermark, + HasCollectedBefore = Store.Count > 0, + State = new Dictionary(State, StringComparer.Ordinal), + }; + + /// One collection cycle. false models a state save that failed + /// after the rows were written. Returns the rows the collector kept (and stored). + public async Task> PollAsync(bool saveState = true) + { + var definition = DefaultTraceEventsCollector.Instance; + var context = NewContext(); + var query = definition.BuildQuery(context); + var p = query.Parameters.ToDictionary(x => x.Name, x => x.Value, StringComparer.Ordinal); + + var cutoff = (DateTime?)p["@cutoff_time"]; + var lastTraceStart = (DateTime?)p["@last_trace_start"]; + var lastSequence = (long?)p["@last_event_sequence"]; + var lastPath = (string?)p["@last_trace_path"]; + bool? continues = lastTraceStart is null ? null : lastTraceStart == TraceStart; + bool singleFile = lastPath == CurrentFile && continues == true; + + /* The query's FROM/WHERE: the file set (current file when steady, else the whole rollover set), + the curated filter, and StartTime >= cutoff OR (running trace AND (sequence rule OR new trace)). */ + var inSet = singleFile ? Written.Where(e => e.File == CurrentFile) : Written; + var read = inSet + .Where(e => e.Name == "ErrorLog" || e.DurationUs > 1_000_000) + .Where(e => (cutoff is { } c && e.Start >= c) + || (e.Start >= TraceStart + && ((continues == true && lastSequence is { } l && e.Sequence > l) || continues == false))) + .OrderByDescending(e => e.Start) + .ToList(); + + using var reader = BuildReader(read, CurrentFile, TraceStart); + var kept = await definition.ReadAsync(reader, context, CancellationToken.None); + foreach (var row in kept) + { + Store.Add((row.EventSequence!.Value, row.EventTime!.Value, row.EventName!)); + } + + if (saveState) + { + foreach (var pair in context.PendingState) + { + State[pair.Key] = pair.Value; + } + } + + return kept; + } + + public (long? Sequence, long? StartTicks) RecordedTrace() + { + var text = State[DefaultTraceEventsCollector.TraceSequenceStateKey]; + var parts = text.Split(':'); + return (parts[1].Length == 0 ? null : long.Parse(parts[1], CultureInfo.InvariantCulture), + long.Parse(parts[0], CultureInfo.InvariantCulture)); + } + + /// Every curated event written so far is in the store, and nothing is in it twice unless + /// (a failed state save repeats rows, which the readers collapse). + public void AssertNothingLost(bool allowRepeats = false) + { + var stored = Store.Select(r => (r.Sequence, r.Start)).ToHashSet(); + foreach (var e in Written.Where(e => e.Name == "ErrorLog" || e.DurationUs > 1_000_000)) + { + Assert.True(stored.Contains((e.Sequence, e.Start)), $"lost {e.Name} seq {e.Sequence} start {e.Start:HH:mm:ss}"); + } + + if (!allowRepeats) + { + Assert.Equal(stored.Count, Store.Count); + } + } + } + + /// Builds the two-result-set batch the real query returns: the payload, then (path, trace start). + private static System.Data.Common.DbDataReader BuildReader(List rows, string currentFile, DateTime traceStart) + { + var set = new DataSet(); + var payload = new DataTable("payload"); + var types = new[] + { + typeof(DateTime), typeof(string), typeof(int), typeof(int), typeof(string), typeof(int), + typeof(string), typeof(string), typeof(string), typeof(string), typeof(string), typeof(long), + typeof(long), typeof(string), typeof(string), typeof(int), typeof(int), typeof(int), + typeof(long), typeof(long), typeof(DateTime), + }; + for (int i = 0; i < types.Length; i++) + { + payload.Columns.Add("c" + i.ToString(CultureInfo.InvariantCulture), types[i]); + } + + foreach (var e in rows) + { + var values = new object[types.Length]; + Array.Fill(values, DBNull.Value); + values[0] = e.Start; + values[1] = e.Name; + values[18] = e.Sequence; + values[19] = e.DurationUs; + payload.Rows.Add(values); + } + + var tail = new DataTable("tail"); + tail.Columns.Add("current_trace_path", typeof(string)); + tail.Columns.Add("current_trace_start", typeof(DateTime)); + tail.Rows.Add(currentFile, traceStart); + + set.Tables.Add(payload); + set.Tables.Add(tail); + return set.CreateDataReader(); + } + + /* ---- the defect: a stall written after the poll that moved the watermark past its StartTime ---- */ + + [Fact] + public async Task LateWrittenAutogrow_IsStored_WhereTheStartTimeRuleAloneLosesIt() + { + var host = new TraceHost(); + host.Write(s_noon.AddMinutes(-30)); + host.Write(s_noon.AddSeconds(3)); /* ErrorLog at 12:00:03 */ + Assert.Equal(2, (await host.PollAsync()).Count); /* poll at 12:00:05 */ + Assert.Equal(s_noon.AddSeconds(3), host.Watermark); + + /* The log autogrow began 12:00:00 and ran 8 s: the trace writes it at 12:00:08, with a StartTime + BELOW the watermark. */ + var grow = host.Write(s_noon, Grow, durationUs: 8_000_000); + + var control = host.Clone(); + control.State.Remove(DefaultTraceEventsCollector.TraceSequenceStateKey); + Assert.Empty(await control.PollAsync()); /* the old rule: never read */ + + var kept = await host.PollAsync(); /* poll at 12:00:10 */ + var row = Assert.Single(kept); + Assert.Equal(grow.Sequence, row.EventSequence); + Assert.Equal(Grow, row.EventName); + host.AssertNothingLost(); + } + + [Fact] + public async Task ReRead_StoresNothingTwice_AcrossPollsAndARollover() + { + var host = new TraceHost(); + host.Write(s_noon.AddSeconds(3)); + await host.PollAsync(); + host.Write(s_noon, Grow, 8_000_000); + await host.PollAsync(); + int stored = host.Store.Count; + + for (int i = 0; i < 3; i++) + { + Assert.Empty(await host.PollAsync()); + } + + host.Rollover(File6); /* path moved: the whole set is read */ + Assert.Empty(await host.PollAsync()); + Assert.Empty(await host.PollAsync()); + + Assert.Equal(stored, host.Store.Count); + host.AssertNothingLost(); + } + + /* ---- rollover: one trace, several files, sequences keep increasing ---- */ + + [Fact] + public async Task Rollover_KeepsReading_AndALateStallInTheNewFileIsStillFound() + { + var host = new TraceHost(); + host.Write(s_noon.AddSeconds(1)); + host.Write(s_noon.AddSeconds(2)); + await host.PollAsync(); + Assert.Equal(2L, host.RecordedTrace().Sequence); + + host.Rollover(File6); + host.Write(s_noon.AddSeconds(6)); + host.Write(s_noon.AddSeconds(1.5), Grow, 4_000_000); /* began before the watermark */ + var kept = await host.PollAsync(); + Assert.Equal(2, kept.Count); + Assert.Equal(4L, host.RecordedTrace().Sequence); /* continues across files, no reset */ + + /* Steady again on file 6: the current file alone, and a stall still found in it. */ + host.Write(s_noon.AddSeconds(5), Grow, 3_000_000); + Assert.Single(await host.PollAsync()); + host.AssertNothingLost(); + } + + /* ---- restart: the sequence goes back to 1 under a new trace identity ---- */ + + [Fact] + public async Task TraceRestart_StoresTheNewTrace_AndNeverTheOldTracesRowsTwice() + { + var host = new TraceHost(); + host.Write(s_noon.AddSeconds(1)); + host.Write(s_noon.AddSeconds(3)); + host.Write(s_noon.AddSeconds(4)); + await host.PollAsync(); + Assert.Equal(3L, host.RecordedTrace().Sequence); + + /* Written by the old trace after the last poll, just before the server went down: sequence 4. */ + var lastOfOld = host.Write(s_noon.AddSeconds(20)); + + var newStart = s_noon.AddMinutes(30); + host.RestartTrace(newStart, File6); + host.Write(newStart.AddSeconds(5)); /* new trace, sequence 1 */ + host.Write(newStart.AddSeconds(6)); /* sequence 2 */ + int before = host.Store.Count; + + var kept = await host.PollAsync(); + Assert.Equal(3, kept.Count); /* old seq 4 once + new 1 and 2 */ + Assert.Contains(kept, r => r.EventSequence == lastOfOld.Sequence && r.EventTime == lastOfOld.Start); + Assert.Equal(before + 3, host.Store.Count); + + var (sequence, ticks) = host.RecordedTrace(); + Assert.Equal(newStart.Ticks, ticks); + Assert.Equal(2, sequence); /* the old trace's 4 does not count */ + + /* A stall of the new trace written late: found, because the bound is the new trace's 2 and not the old 4. */ + host.Write(newStart.AddSeconds(1), Grow, 6_000_000); + var late = Assert.Single(await host.PollAsync()); + Assert.Equal(3, late.EventSequence); + host.AssertNothingLost(); + } + + [Fact] + public async Task TraceRestart_WithNoCuratedEventYet_RecordsSequenceZero_AndTheFirstEventsAreRead() + { + var host = new TraceHost(); + host.Write(s_noon.AddSeconds(3)); + await host.PollAsync(); + + var newStart = s_noon.AddHours(1); + host.RestartTrace(newStart, File6); + Assert.Empty(await host.PollAsync()); /* quiet new trace */ + Assert.Equal(0L, host.RecordedTrace().Sequence); + Assert.Equal(newStart.Ticks, host.RecordedTrace().StartTicks); + + host.Write(newStart.AddSeconds(2)); + host.Write(newStart, Grow, 5_000_000); + Assert.Equal(2, (await host.PollAsync()).Count); + host.AssertNothingLost(); + } + + [Fact] + public async Task TraceRestart_AndASecondRestart_EachReadFromTheirFirstEvent() + { + var host = new TraceHost(); + host.Write(s_noon.AddSeconds(3)); + await host.PollAsync(); + + host.RestartTrace(s_noon.AddHours(1), File6); + host.Write(s_noon.AddHours(1).AddSeconds(2)); + Assert.Single(await host.PollAsync()); + + host.RestartTrace(s_noon.AddHours(2), File7); + host.Write(s_noon.AddHours(2).AddSeconds(2)); + host.Write(s_noon.AddHours(2).AddSeconds(3)); + Assert.Equal(2, (await host.PollAsync()).Count); + Assert.Equal(2L, host.RecordedTrace().Sequence); + host.AssertNothingLost(); + } + + /* ---- upgrade: no stored sequence yet ---- */ + + [Fact] + public async Task Upgrade_NoSequenceState_FallsBackToTheStartTimeRule_ThenRecordsTheSequence() + { + var host = new TraceHost(); + host.Write(s_noon.AddSeconds(1)); + host.Write(s_noon.AddSeconds(3)); + await host.PollAsync(); + host.State.Remove(DefaultTraceEventsCollector.TraceSequenceStateKey); /* state as the previous build left it */ + + var between = host.Write(s_noon.AddSeconds(5)); /* written before the upgrade poll */ + var kept = await host.PollAsync(); + var row = Assert.Single(kept); /* stored once, the newest tick not again */ + Assert.Equal(between.Sequence, row.EventSequence); + Assert.Equal(3L, host.RecordedTrace().Sequence); + + /* From the next poll the late stall is found. */ + host.Write(s_noon.AddSeconds(2), Grow, 9_000_000); + Assert.Single(await host.PollAsync()); + host.AssertNothingLost(); + } + + [Fact] + public async Task Upgrade_NoStateAtAll_RepeatsOnlyTheNewestTick_WhichTheReadersCollapse() + { + var host = new TraceHost(); + host.Write(s_noon.AddSeconds(1)); + host.Write(s_noon.AddSeconds(3)); + await host.PollAsync(); + host.State.Clear(); /* state reset or lost */ + + var kept = await host.PollAsync(); + Assert.All(kept, r => Assert.Equal(s_noon.AddSeconds(3), r.EventTime)); + Assert.Single(kept); + Assert.True(host.State.ContainsKey(DefaultTraceEventsCollector.TraceSequenceStateKey)); + host.AssertNothingLost(allowRepeats: true); + } + + [Fact] + public async Task Upgrade_AQuietFirstPoll_RecordsTheIdentityAlone_SoTheSteadyStateFileReadIsKept() + { + var host = new TraceHost(); + host.Write(s_noon.AddSeconds(3)); + await host.PollAsync(); + host.State.Remove(DefaultTraceEventsCollector.TraceSequenceStateKey); + host.Written.Clear(); /* nothing curated in the file set */ + + await host.PollAsync(); + var (sequence, ticks) = host.RecordedTrace(); + Assert.Null(sequence); /* identity, no sequence yet */ + Assert.Equal(host.TraceStart.Ticks, ticks); + + /* Identity known => steady state reads the current file alone again (#1962), sequence rule still off. */ + var query = DefaultTraceEventsCollector.Instance.BuildQuery(host.NewContext()); + Assert.Equal(host.TraceStart, (DateTime?)query.Parameters.Single(x => x.Name == "@last_trace_start").Value); + Assert.Null(query.Parameters.Single(x => x.Name == "@last_event_sequence").Value); + + host.Write(s_noon.AddSeconds(7)); + Assert.Single(await host.PollAsync()); + Assert.Equal(host.Written[0].Sequence, host.RecordedTrace().Sequence); + } + + /* ---- failure: state is saved only after a good write; a failed save repeats at worst ---- */ + + [Fact] + public async Task FailedStateSave_RepeatsAtWorst_AndNeverLosesAnEvent() + { + var host = new TraceHost(); + host.Write(s_noon.AddSeconds(1)); + await host.PollAsync(); + + host.Write(s_noon.AddSeconds(3)); + host.Write(s_noon, Grow, 8_000_000); + Assert.Equal(2, (await host.PollAsync(saveState: false)).Count); /* rows written, state not saved */ + + var again = await host.PollAsync(); /* stale state: a repeat */ + Assert.True(again.Count <= 2); + host.AssertNothingLost(allowRepeats: true); + + int settled = host.Store.Count; + Assert.Empty(await host.PollAsync()); /* the saved state ends the repeat */ + Assert.Equal(settled, host.Store.Count); + } + + /* ---- the stored text ---- */ + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("garbage")] + [InlineData("123")] + [InlineData(":5")] + [InlineData("-5:5")] + [InlineData("639000000000000000:-3")] + [InlineData("639000000000000000:abc")] + [InlineData("99999999999999999999999:5")] + public void MalformedOrAbsentState_BindsNoIdentityAndNoSequence(string? state) + { + var context = new CollectorContext + { + ServerId = 1, + ServerName = "s", + CollectionTime = DateTime.UtcNow, + Deltas = new NoDeltas(), + Target = new CollectorTargetInfo(), + State = state is null + ? CollectorContext.NoState + : new Dictionary(StringComparer.Ordinal) { [DefaultTraceEventsCollector.TraceSequenceStateKey] = state }, + }; + + var parameters = DefaultTraceEventsCollector.Instance.BuildQuery(context).Parameters; + Assert.Null(parameters.Single(x => x.Name == "@last_trace_start").Value); + Assert.Null(parameters.Single(x => x.Name == "@last_event_sequence").Value); + } + + [Fact] + public void WellFormedState_BindsTheIdentityAndTheSequence_AtTheRightTypes() + { + var start = new DateTime(2026, 10, 10, 5, 27, 27, 487, DateTimeKind.Unspecified); + var context = new CollectorContext + { + ServerId = 1, + ServerName = "s", + CollectionTime = DateTime.UtcNow, + Deltas = new NoDeltas(), + Target = new CollectorTargetInfo(), + State = new Dictionary(StringComparer.Ordinal) + { + [DefaultTraceEventsCollector.TraceSequenceStateKey] = start.Ticks.ToString(CultureInfo.InvariantCulture) + ":79116", + }, + }; + + var parameters = DefaultTraceEventsCollector.Instance.BuildQuery(context).Parameters; + var traceStart = parameters.Single(x => x.Name == "@last_trace_start"); + var sequence = parameters.Single(x => x.Name == "@last_event_sequence"); + Assert.Equal(start, traceStart.Value); + Assert.Equal(CollectorParameterType.DateTime2, traceStart.Type); + Assert.Equal(79116L, sequence.Value); + Assert.Equal(CollectorParameterType.BigInt, sequence.Type); + } + + [Fact] + public void StateKeys_DeclareTheTraceSequence_SoBothHostsLoadAndSaveIt() + { + Assert.Equal("trace_event_sequence", DefaultTraceEventsCollector.TraceSequenceStateKey); + Assert.Contains(DefaultTraceEventsCollector.TraceSequenceStateKey, DefaultTraceEventsCollector.Instance.StateKeys); + } + + [Fact] + public void QueryText_HoldsTheRuleTheModelEvaluates() + { + var text = DefaultTraceEventsCollector.Instance.BuildQuery(new CollectorContext + { + ServerId = 1, + ServerName = "s", + CollectionTime = DateTime.UtcNow, + Deltas = new NoDeltas(), + Target = new CollectorTargetInfo(), + }).Text.Replace("\r\n", "\n", StringComparison.Ordinal); + + /* The identity of the running trace, captured with the path in the same read. */ + Assert.Contains("@current_trace_start = t.start_time", text, StringComparison.Ordinal); + Assert.Contains("WHEN CONVERT(datetime, @last_trace_start) = @current_trace_start\n THEN 1", text, StringComparison.Ordinal); + + /* The StartTime floor, OR the sequence rule restricted to the running trace; a new trace from its start. */ + Assert.Contains(" ft.StartTime >= CONVERT(datetime, COALESCE(@cutoff_time", text, StringComparison.Ordinal); + Assert.Contains("ft.StartTime >= @current_trace_start", text, StringComparison.Ordinal); + Assert.Contains("(@trace_continues = 1 AND ft.EventSequence > @last_event_sequence)", text, StringComparison.Ordinal); + Assert.Contains("OR @trace_continues = 0", text, StringComparison.Ordinal); + + /* Only the same trace may use the one-file steady-state read: an unknown or changed identity reads the set. */ + Assert.Equal(2, CountOf(text, "AND @trace_continues = 1\n")); + + /* The trailing result set carries the identity next to the path. */ + Assert.Contains("current_trace_path = @current_trace_path,\n current_trace_start = @current_trace_start;", text, StringComparison.Ordinal); + + /* Unchanged by this fix: the Commit-only DDL rule, the curated list and the 1 s stall filter. */ + Assert.Contains("AND ISNULL(ft.EventSubClass, 1) = 1", text, StringComparison.Ordinal); + Assert.Contains("ISNULL(ft.Duration, 0) > 1000000", text, StringComparison.Ordinal); + Assert.Contains("te.name LIKE N'%Log File Auto Grow%'", text, StringComparison.Ordinal); + } + + private static int CountOf(string text, string needle) + { + int count = 0; + for (int at = text.IndexOf(needle, StringComparison.Ordinal); at >= 0; + at = text.IndexOf(needle, at + needle.Length, StringComparison.Ordinal)) + { + count++; + } + + return count; + } +} diff --git a/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs b/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs index 1ed85336d1..803b3a1cb5 100644 --- a/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs +++ b/Lite.Tests/DefaultTraceEventsCollectorDefinitionTests.cs @@ -202,7 +202,7 @@ public void BuildQuery_KeepsTheReconfigureLine_OutsideEveryDatabasePredicate() /* The keep is the FIRST arm of the OR that follows the StartTime bound, and every database predicate sits AFTER it — inside the other arm — so none of them can reach the 15457 row. */ - var startTime = text.IndexOf("AND ft.StartTime >= ", StringComparison.Ordinal); + var startTime = text.IndexOf("ft.StartTime >= CONVERT(datetime, ", StringComparison.Ordinal); var keepAt = text.IndexOf(keep, StringComparison.Ordinal); var systemDbs = text.IndexOf("ISNULL(ft.DatabaseID, 0) NOT IN (1, 3, 4)", StringComparison.Ordinal); var agDbs = text.IndexOf("ISNULL(ft.DatabaseID, 0) < 32761", StringComparison.Ordinal); @@ -277,7 +277,7 @@ public void BuildQuery_ExcludedDatabases_IsNullSafe_KeepsServerLevelEvents() { var noExclusion = DefaultTraceEventsCollector.Instance.BuildQuery(MakeContext()); Assert.DoesNotContain("@excl_db_0", noExclusion.Text, StringComparison.Ordinal); - Assert.Equal(3, noExclusion.Parameters.Count); /* @cutoff_time + @last_trace_path + @include_object_ddl */ + Assert.Equal(5, noExclusion.Parameters.Count); /* @cutoff_time + @last_trace_path + @last_trace_start + @last_event_sequence + @include_object_ddl */ var withExclusion = DefaultTraceEventsCollector.Instance.BuildQuery( MakeContext(excludedDatabases: new[] { "ReportingDB", "ScratchDB" })); @@ -289,7 +289,7 @@ whole category whenever any database is excluded. */ "AND (ft.DatabaseName IS NULL OR ft.DatabaseName NOT IN (@excl_db_0, @excl_db_1))", withExclusion.Text, StringComparison.Ordinal); Assert.DoesNotContain("AND ft.DatabaseName NOT IN", withExclusion.Text, StringComparison.Ordinal); /* never the bare form */ - Assert.Equal(5, withExclusion.Parameters.Count); /* @cutoff_time + @last_trace_path + @include_object_ddl + two excluded names */ + Assert.Equal(7, withExclusion.Parameters.Count); /* the five above + two excluded names */ Assert.Contains(withExclusion.Parameters, p => p.Name == "@excl_db_0" && (string?)p.Value == "ReportingDB"); Assert.Contains(withExclusion.Parameters, p => p.Name == "@excl_db_1" && (string?)p.Value == "ScratchDB"); } @@ -399,11 +399,12 @@ or fail on checkout settings rather than on the SQL. */ private static string Lf(string text) => text.Replace("\r\n", "\n", StringComparison.Ordinal); [Fact] - public void StateKeys_DeclaresTheLastSeenTracePathAndTheWatermarkTickSequences_AndTheDeclaringSetIsPinned() + public void StateKeys_DeclaresTheTracePathTheTickSequencesAndTheTraceSequence_AndTheDeclaringSetIsPinned() { Assert.Equal( - new[] { "last_trace_file_path", "watermark_tick_event_sequences" }, + new[] { "last_trace_file_path", "watermark_tick_event_sequences", "trace_event_sequence" }, DefaultTraceEventsCollector.Instance.StateKeys.ToArray()); + Assert.Equal("trace_event_sequence", DefaultTraceEventsCollector.TraceSequenceStateKey); Assert.Equal("watermark_tick_event_sequences", DefaultTraceEventsCollector.WatermarkTickSequencesStateKey); Assert.Equal("last_trace_file_path", DefaultTraceEventsCollector.LastTraceFilePathStateKey); @@ -462,8 +463,8 @@ AFTERWARDS. Measured 897 ms whole-set vs 178 ms current-file on the reporting fl Assert.Contains("@current_trace_path = t.path", text, StringComparison.Ordinal); /* Steady-state arm: the captured path itself, and ONE file. */ - Assert.Contains("WHEN @current_trace_path = @last_trace_path\n THEN @current_trace_path", text, StringComparison.Ordinal); - Assert.Contains("WHEN @current_trace_path = @last_trace_path\n THEN 1", text, StringComparison.Ordinal); + Assert.Contains("WHEN @current_trace_path = @last_trace_path\n AND @trace_continues = 1\n THEN @current_trace_path", text, StringComparison.Ordinal); + Assert.Contains("WHEN @current_trace_path = @last_trace_path\n AND @trace_continues = 1\n THEN 1", text, StringComparison.Ordinal); /* Fallback arm still reads the whole set, via the unchanged base-path normalization. */ Assert.Contains("ELSE t.max_files", text, StringComparison.Ordinal); @@ -513,7 +514,7 @@ public void BuildQuery_ReturnsTheReadPathAsATrailingResultSet_NotAPayloadColumn( cycles that most need the path recorded are the ones that collect ZERO rows (a server whose trace churns through 20 MB files without producing curated events), and a row-derived path would never notice those rollovers — leaving the collector in the expensive fallback forever. */ - Assert.Contains("OPTION(RECOMPILE);\n\nSELECT /* PerformanceMonitorLite */\n current_trace_path = @current_trace_path;", text, StringComparison.Ordinal); + Assert.Contains("OPTION(RECOMPILE);\n\nSELECT /* PerformanceMonitorLite */\n current_trace_path = @current_trace_path,\n current_trace_start = @current_trace_start;", text, StringComparison.Ordinal); Assert.DoesNotContain("current_trace_path = ft.", text, StringComparison.Ordinal); Assert.DoesNotContain("trace_path", string.Join(",", DefaultTraceEventsCollector.Instance.PayloadColumns.Select(c => c.Name)), StringComparison.Ordinal); } diff --git a/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs b/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs index 75a432e855..cf24518d30 100644 --- a/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs +++ b/PerformanceMonitor.Collectors/DefaultTraceEventsCollector.cs @@ -52,7 +52,12 @@ namespace PerformanceMonitor.Collectors; /// the host store) bounds the read at ft.StartTime >= @cutoff_time (at-or-after, #5644 W6: an event in the /// watermark's own 3.33 ms tick that the trace flushes after a poll must not be lost), and the rows already /// stored for that one tick are dropped by EventSequence (see ), -/// so a rollover file re-read across cycles never re-ingests an event. Unlike those XE ring-buffer collectors (whose 10-minute first-run fallback +/// so a rollover file re-read across cycles never re-ingests an event. A StartTime watermark alone still +/// loses late-written events (#5644 W6): the curated auto grow/shrink stalls carry the time they BEGAN but +/// reach the file when they END, after later-starting events were stored and the watermark moved past them. So +/// the read also takes every event of the running trace whose EventSequence is greater than the last one +/// recorded (), keyed to the trace's identity because the sequence restarts +/// when the trace does; that rule can only add rows to the StartTime floor, never remove one. Unlike those XE ring-buffer collectors (whose 10-minute first-run fallback /// fits their tiny recent-only buffers), the FIRST run here collects EVERYTHING still in the trace files /// (a far-past cutoff), because the default trace persists hours-to-days of history on disk that would /// otherwise be lost — the same first-run-collects-all choice the Dashboard collector makes. @@ -162,6 +167,35 @@ private DefaultTraceEventsCollector() /// public const string WatermarkTickSequencesStateKey = "watermark_tick_event_sequences"; + /// + /// The key holding the running trace's identity and the newest + /// EventSequence this collector has read from it, as "<start ticks>:<sequence>" + /// (#5644 W6). Why a sequence and not a time: the trace stamps an event with the moment it BEGAN + /// (StartTime) but writes it when it ENDS, so the curated auto grow/shrink stalls (Duration over one + /// second) reach the file seconds after later-starting events were already read and stored. A watermark on + /// StartTime has then moved past them and an at-or-after read never sees them. EventSequence + /// is assigned when the trace takes the event (it increases in file order, across the rollover files of + /// one trace), so "EventSequence greater than the last one stored" reads each event once whenever it is + /// written; the query ORs that rule onto the StartTime floor, so it can only add rows. + /// + /// Identity. The sequence restarts at 1 whenever the default trace is started again (SQL Server + /// restart, the trace switched off and on), and the rollover set can hold files of the previous trace, so + /// a sequence means nothing without the trace it came from. The identity is sys.traces.start_time + /// (the first event of a trace carries exactly that StartTime, so a row belongs to the running trace + /// when its StartTime is at or after it). A changed identity makes the query read the running trace + /// from its first event: the store cannot hold any of it, because the poll that first sees a new trace is the + /// first that can read it. The older trace's files, which that read also returns, fall under the + /// StartTime floor and the tick dedup as before. + /// + /// Absent, malformed or half-known state (first poll after an upgrade, a reset state table, a + /// poll whose state save failed): no identity makes the sequence rule off and the whole rollover set be + /// read once, so that poll behaves exactly as the StartTime rule alone did; an identity with no + /// sequence (the format "<start ticks>:", written when a poll read no row of the running + /// trace) keeps the sequence rule off until the first row records one. A state that is behind the store + /// costs a repeat of the rows since, which both readers collapse; it never costs an event. + /// + public const string TraceSequenceStateKey = "trace_event_sequence"; + public sealed class Row { public DateTime? EventTime { get; set; } @@ -232,14 +266,29 @@ wildly between the all-history first run and the tiny steady-state windows. */ SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; DECLARE - @current_trace_path nvarchar(260); + @current_trace_path nvarchar(260), + @current_trace_start datetime, + @trace_continues bit; SELECT /* PerformanceMonitorLite */ - @current_trace_path = t.path + @current_trace_path = t.path, + @current_trace_start = t.start_time FROM sys.traces AS t WHERE t.is_default = 1 AND t.status = 1; +/*#5644 W6: is the running trace the one the stored EventSequence came from? NULL = no stored identity (the + sequence rule stays off, and the whole rollover set is read); 1 = the same trace; 0 = a new trace, whose + EventSequence restarted at 1.*/ +SET @trace_continues = + CASE + WHEN @last_trace_start IS NULL + THEN NULL + WHEN CONVERT(datetime, @last_trace_start) = @current_trace_start + THEN 1 + ELSE 0 + END; + SELECT /* PerformanceMonitorLite */ event_time = ft.StartTime, event_name = te.name, @@ -266,9 +315,10 @@ FROM sys.traces AS t CROSS APPLY sys.fn_trace_gettable ( CASE - /*Steady state: the trace has not rolled since the previous cycle, so every post-watermark - event is in the file we are already pointed at*/ + /*Steady state: the trace has not rolled since the previous cycle (and is the same trace), so every + post-watermark event is in the file we are already pointed at*/ WHEN @current_trace_path = @last_trace_path + AND @trace_continues = 1 THEN @current_trace_path WHEN CHARINDEX(N'_', REVERSE(@current_trace_path)) > 0 AND (CHARINDEX(N'\', REVERSE(@current_trace_path)) = 0 OR CHARINDEX(N'_', REVERSE(@current_trace_path)) < CHARINDEX(N'\', REVERSE(@current_trace_path))) @@ -278,6 +328,7 @@ ELSE @current_trace_path END, CASE WHEN @current_trace_path = @last_trace_path + AND @trace_continues = 1 THEN 1 ELSE t.max_files END @@ -286,7 +337,23 @@ JOIN sys.trace_events AS te ON ft.EventClass = te.trace_event_id WHERE t.is_default = 1 AND t.status = 1 -AND ft.StartTime >= CONVERT(datetime, {1}) +AND +( + ft.StartTime >= CONVERT(datetime, {1}) + OR + /*#5644 W6: events the trace WROTE after the last one stored, whatever their StartTime. A grow/shrink + stall starts at StartTime but reaches the file when it ends, by which time the StartTime watermark + has moved past it. Only rows of the running trace count (an earlier trace's files can be in the + rollover set, with sequences from a different run); a changed trace is read from its first event.*/ + ( + ft.StartTime >= @current_trace_start + AND + ( + (@trace_continues = 1 AND ft.EventSequence > @last_event_sequence) + OR @trace_continues = 0 + ) + ) +) AND ( /*The sp_configure line (msg 15457, an ErrorLog write): kept regardless of the session's database @@ -341,7 +408,8 @@ ft.StartTime DESC OPTION(RECOMPILE); SELECT /* PerformanceMonitorLite */ - current_trace_path = @current_trace_path;"; + current_trace_path = @current_trace_path, + current_trace_start = @current_trace_start;"; public override string Name => "default_trace_events"; @@ -354,7 +422,10 @@ ft.StartTime DESC /// Events at or after the newest already-collected StartTime are fetched and the ones already stored for /// that tick are dropped by EventSequence (), so re-reading the /// rolling trace files across cycles never re-ingests an event (mirrors the deadlock / system_health - /// collectors' event_time watermark). First run has no watermark → collect all on-disk history. + /// collectors' event_time watermark). Events the trace WROTE after the last recorded EventSequence are + /// fetched whatever their StartTime (, #5644 W6): a stall's StartTime is + /// when it began, so a StartTime watermark alone would never see one written late. First run has no + /// watermark → collect all on-disk history. /// public override string? WatermarkColumn => "event_time"; @@ -366,7 +437,7 @@ ft.StartTime DESC /// before the query is built and persists whatever this cycle observed once the cycle completes. /// The only collector that declares state — every other one's dedup is derivable from its rows. /// - public override IReadOnlyList StateKeys { get; } = new[] { LastTraceFilePathStateKey, WatermarkTickSequencesStateKey }; + public override IReadOnlyList StateKeys { get; } = new[] { LastTraceFilePathStateKey, WatermarkTickSequencesStateKey, TraceSequenceStateKey }; /// /// Collects on SQL Server, Azure SQL Managed Instance, and AWS RDS — everywhere the built-in default @@ -407,10 +478,21 @@ express this window against a server-local ft.StartTime. */ knowing you missed nothing (#1962). */ context.State.TryGetValue(LastTraceFilePathStateKey, out var lastTracePath); - var parameters = new List(exclusionParameters.Count + 3) + /* The trace identity and newest EventSequence from the previous cycle (#5644 W6), or null/null when + there is none or it does not parse: the query then leaves the sequence rule off and reads the + whole rollover set, which is the StartTime rule alone, as before. */ + context.State.TryGetValue(TraceSequenceStateKey, out var traceSequenceState); + TryParseTraceSequence(traceSequenceState, out var lastTraceStartTicks, out var lastEventSequence); + DateTime? lastTraceStart = lastTraceStartTicks is { } startTicks + ? new DateTime(startTicks, DateTimeKind.Unspecified) + : null; + + var parameters = new List(exclusionParameters.Count + 5) { new("@cutoff_time", cutoffTime, CollectorParameterType.DateTime2), new("@last_trace_path", lastTracePath, CollectorParameterType.NVarChar260), + new("@last_trace_start", lastTraceStart, CollectorParameterType.DateTime2), + new("@last_event_sequence", lastEventSequence, CollectorParameterType.BigInt), /* Operator toggle mirroring the Dashboard proc's @include_object_events: 1 keeps the Object DDL slice (today's behavior), 0 drops it so a create/drop-happy workload can't flood the tab. */ new("@include_object_ddl", context.CollectSchemaChangeEvents ? 1 : 0, CollectorParameterType.Int32), @@ -512,16 +594,110 @@ public override async ValueTask> ReadAsync(DbDataReader reader, Collec depend on it hardest are the ones that returned no rows at all — a quiet server still has to notice its trace rolled. A missing/NULL path (no running default trace) records nothing, which leaves the next cycle on the safe fallback. */ + DateTime? traceStart = null; if (await reader.NextResultAsync(cancellationToken) && await reader.ReadAsync(cancellationToken) && !reader.IsDBNull(0)) { context.PendingState[LastTraceFilePathStateKey] = reader.GetString(0); + + /* The running trace's identity (sys.traces.start_time), read in the same statement batch as the + path and the rows. A trailing set without the column records no identity (#5644 W6). */ + if (reader.FieldCount > 1 && !reader.IsDBNull(1)) + { + traceStart = reader.GetDateTime(1); + } } + RecordTraceSequence(rows, traceStart, context); return DropStoredWatermarkTickRows(rows, context); } + /// + /// #5644 W6: records the running trace's identity and the newest EventSequence read from it, for the next + /// cycle's "EventSequence greater than the last one" rule (see ). Only + /// rows whose StartTime is at or after the trace's start count: the rollover set can also return an earlier + /// trace's files, whose sequences come from a different run and would push the bound past the running + /// trace's events. The newest sequence never moves backwards within one trace. A new trace read to the + /// end with no curated row in it records sequence 0 (nothing of it can be stored yet); a first record with + /// no row to take a sequence from records the identity alone, which keeps the sequence rule off. + /// + private static void RecordTraceSequence(List rows, DateTime? traceStart, CollectorContext context) + { + if (traceStart is not { } start) + { + return; + } + + long? newest = null; + foreach (var row in rows) + { + if (row.EventTime is { } time && time >= start && row.EventSequence is { } sequence + && (newest is null || sequence > newest.Value)) + { + newest = sequence; + } + } + + context.State.TryGetValue(TraceSequenceStateKey, out var stored); + bool haveStored = TryParseTraceSequence(stored, out var storedStartTicks, out var storedSequence) + && storedStartTicks.HasValue; + bool sameTrace = haveStored && storedStartTicks == start.Ticks; + + long? recorded; + if (sameTrace) + { + recorded = storedSequence is { } known && (newest is null || known > newest.Value) ? known : newest; + } + else if (haveStored) + { + /* A different trace than the one the stored sequence came from: its events are read from the + first one, so with none curated the bound is 0, not "unknown". */ + recorded = newest ?? 0; + } + else + { + recorded = newest; + } + + context.PendingState[TraceSequenceStateKey] = + start.Ticks.ToString(CultureInfo.InvariantCulture) + ":" + + (recorded is { } value ? value.ToString(CultureInfo.InvariantCulture) : string.Empty); + } + + /// + /// Parses ("ticks:sequence", sequence optional). False and both + /// outputs null for absent or malformed text, which every caller treats as "no stored identity". + /// + private static bool TryParseTraceSequence(string? value, out long? startTicks, out long? sequence) + { + startTicks = null; + sequence = null; + if (string.IsNullOrEmpty(value)) + { + return false; + } + + int colon = value.IndexOf(':', StringComparison.Ordinal); + if (colon <= 0 + || !long.TryParse(value.AsSpan(0, colon), NumberStyles.None, CultureInfo.InvariantCulture, out var ticks) + || ticks < DateTime.MinValue.Ticks || ticks > DateTime.MaxValue.Ticks) + { + return false; + } + + var tail = value.AsSpan(colon + 1); + long parsedSequence = 0; + if (!tail.IsEmpty && !long.TryParse(tail, NumberStyles.None, CultureInfo.InvariantCulture, out parsedSequence)) + { + return false; + } + + startTicks = ticks; + sequence = tail.IsEmpty ? null : parsedSequence; + return true; + } + /// /// #5644 W6: the query reads at-or-after the watermark tick, so rows already stored for that tick come /// back with any new event of the same tick. Drops exactly the stored ones (same tick, EventSequence in From f2fbc892168b3f303a8af384a203350eb1983bd9 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:09:07 -0400 Subject: [PATCH 27/42] Collection Health run-time column pin follows the opted-in localizeText flag (#5644) --- Darling/Darling.Tests/WebCollectionHealthRunAtColumnTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/WebCollectionHealthRunAtColumnTests.cs b/Darling/Darling.Tests/WebCollectionHealthRunAtColumnTests.cs index ef615c3214..41b691b142 100644 --- a/Darling/Darling.Tests/WebCollectionHealthRunAtColumnTests.cs +++ b/Darling/Darling.Tests/WebCollectionHealthRunAtColumnTests.cs @@ -64,7 +64,7 @@ that lists every collector carries the run time too. */ Assert.Contains("{ key: \"run_at\", label: \"Run at\", hideWhenEmpty: true },", columns, StringComparison.Ordinal); Assert.Contains("{ key: \"next_run_utc\", label: \"Next run\", format: \"time\", hideWhenEmpty: true },", columns, StringComparison.Ordinal); - Assert.Contains("{ key: \"run_time_note\", label: \"Run time\", clamp: true, hideWhenEmpty: true },", columns, StringComparison.Ordinal); + Assert.Contains("{ key: \"run_time_note\", label: \"Run time\", clamp: true, localizeText: true, hideWhenEmpty: true },", columns, StringComparison.Ordinal); Assert.True(columns.IndexOf("key: \"last_success\"", StringComparison.Ordinal) < columns.IndexOf("key: \"run_at\"", StringComparison.Ordinal)); } From 2b221393017e5d71c175dc23953864f7aba486a6 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:37:29 -0400 Subject: [PATCH 28/42] Tag FinOpsTopAvgRankingTests with the Reads=Lite trait the guard requires (#5644) --- Darling/Darling.Tests/FinOpsTopAvgRankingTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Darling/Darling.Tests/FinOpsTopAvgRankingTests.cs b/Darling/Darling.Tests/FinOpsTopAvgRankingTests.cs index 8d99957393..b3b494990b 100644 --- a/Darling/Darling.Tests/FinOpsTopAvgRankingTests.cs +++ b/Darling/Darling.Tests/FinOpsTopAvgRankingTests.cs @@ -21,6 +21,7 @@ namespace Darling.Tests; /// 2.3, 1.95 and 1.56 into 2, so the tie broke on database name and the top-N cut dropped the busiest database. /// The average now travels unrounded (AvgCpuMs) and is what the ranking and the wire row use. /// +[Trait("Reads", "Lite")] public sealed class FinOpsTopAvgRankingTests { private static string Read(string relative, [CallerFilePath] string thisFile = "") => From fcb1e397df25dfa108f31842a9494a8fdd46532b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:05:41 -0400 Subject: [PATCH 29/42] Web: show the alert text, sweep notes and regression text in local time (#5644 W20) Alert History and Triage bodies, Fleet Sweeps reasons and notes, and the space-separated Z/UTC form localize on the page; stored and emailed text is unchanged. --- .../WebClampCellBehaviourTests.cs | 37 +++++++++++++++++ .../WebPartialWindowBehaviourTests.cs | 41 +++++++++++++++++++ .../Darling.Tests/web-clamp-cell-harness.mjs | 27 ++++++++++++ .../web-partial-window-harness.mjs | 33 ++++++++++++++- .../wwwroot/js/pages/alerts.js | 13 +++--- .../wwwroot/js/pages/sweeps.js | 8 ++-- .../wwwroot/js/pages/triage.js | 4 +- .../wwwroot/js/pages/views.js | 4 +- .../wwwroot/js/util.js | 41 ++++++++++++++++--- 9 files changed, 189 insertions(+), 19 deletions(-) diff --git a/Darling/Darling.Tests/WebClampCellBehaviourTests.cs b/Darling/Darling.Tests/WebClampCellBehaviourTests.cs index f7d422dc70..6e604b9cdb 100644 --- a/Darling/Darling.Tests/WebClampCellBehaviourTests.cs +++ b/Darling/Darling.Tests/WebClampCellBehaviourTests.cs @@ -100,6 +100,43 @@ public void OnlyAnOptedInMessageColumn_LocalizesAnIsoInstant_SqlTextAndDetailsPr Assert.DoesNotContain("T14:37", expected); } + [Fact] + public void EachAlertSweepAndRegressionShape_IsLocalizedInAnOptedInCell_AndPrintedAsWrittenElsewhere() + { + // #5644 W20 (H2): "until 2026-10-11 12:48:52Z", "service started 2026-10-12T23:01:13.7792072Z", "Failed At: 2026-10-12 07:00:00Z". + var rows = Run().GetProperty("shapes").EnumerateArray().ToList(); + Assert.Equal(6, rows.Count); + var sqlZ = "SELECT * FROM t WHERE d = '2026-01-01 00:00:00Z'"; + foreach (var r in rows) + { + Assert.Equal(sqlZ, r.GetProperty("sql").GetString()); + Assert.Equal(r.GetProperty("raw").GetString(), r.GetProperty("notOpted").GetString()); + } + + foreach (var r in rows.Take(5)) + { + var expected = r.GetProperty("expected").GetString(); + Assert.Equal(expected, r.GetProperty("opted").GetString()); + Assert.Equal(expected, r.GetProperty("clamped").GetString()); + Assert.NotEqual(r.GetProperty("raw").GetString(), r.GetProperty("opted").GetString()); + } + + // A server's own wall clock (no Z, no UTC) is never shifted, even in an opted-in cell. + Assert.Equal("start time 2026-09-27 21:32:43 (server clock)", rows[5].GetProperty("opted").GetString()); + } + + [Fact] + public void TheFleetSweepReasonsAndNotes_AreOptedIn_AndTheAlertPagesLocalizeTheirBodies() + { + var sweeps = Wwwroot("js", "pages", "sweeps.js"); + Assert.Equal(2, Regex.Matches(sweeps, "key: \"reason\", label: \"Reason\", wrap: true, localizeText: true").Count); + Assert.Contains("el(\"li\", { text: localizeIsoText(String(n)) })", sweeps, StringComparison.Ordinal); + foreach (var page in new[] { "alerts.js", "triage.js", "views.js" }) + { + Assert.Contains("localizeAlertDetail(", Wwwroot("js", "pages", page), StringComparison.Ordinal); + } + } + [Fact] public void TheCollectionHealthMessageColumns_AreOptedIn_AndTheCustomViewAndDefaultTraceTextColumnsAreNot() { diff --git a/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs b/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs index fefb61d98e..7fb9543c74 100644 --- a/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs +++ b/Darling/Darling.Tests/WebPartialWindowBehaviourTests.cs @@ -124,6 +124,47 @@ public void AnIsoInstantInASentence_PrintsInTheGridsLocalTimeFormat() Assert.DoesNotContain("T", a); } + [Fact] + public void ASpaceSeparatedOrUtcMarkedInstant_PrintsLocal_AndAServerClockOneIsLeftAlone() + { + // #5644 W20 (H2): the alert text spells an instant "2026-10-11 12:48:52Z" and "... 23:13:00 UTC"; the old pattern needed a "T". + var h = Run().GetProperty("h2"); + string G(string n) => h.GetProperty(n).GetString()!; + Assert.Equal(G("expSpaceZ"), G("spaceZ")); + Assert.Equal(G("expSpaceUtc"), G("spaceUtc")); + Assert.Equal(G("expFrac7"), G("frac7")); + Assert.Equal(G("expSpaceFrac7"), G("spaceFrac7")); + Assert.Equal(G("expSpaceZ"), G("stripUtc")); + Assert.Equal("start time 2026-09-27 21:32:43 (server clock)", G("serverClock")); + } + + [Fact] + public void AnAlertBodyOnTheWeb_ShowsItsTimeLinesLocal_AndLeavesStatementsAndServerClockLinesAlone() + { + // #5644 W20 (H2): Triage and Alert History printed "Incident Since: 2026-10-12 23:13:00Z", "Failed At: ...Z" and a time-of-day-only "23:12:45 UTC". + var h = Run().GetProperty("h2"); + string G(string n) => h.GetProperty(n).GetString()!; + var here = h.GetProperty("here"); + string H(string n) => here.GetProperty(n).GetString()!; + var expected = string.Join("\n", new[] + { + "Blocking chain (x2)", + " Incident Since: " + H("inc"), + " Blocked Query: SELECT 1 WHERE d = '2026-01-01 00:00:00Z'", + " Failed At: " + H("fail"), + " Started: 2026-09-27 21:32:43 (server clock, UTC offset not collected)", + " Window: " + H("fail") + " → " + H("end"), + "", + "Newest blocked process sample: " + H("samp") + ", 5m 20s old", + }); + Assert.Equal(expected, G("body")); + // No newest_sample on the row, or one whose time of day is not the line's: the time-of-day-only line is left as stored. + Assert.Contains("Newest blocked process sample: 23:12:45 UTC, 5m 20s old", G("bodyNoSample")); + Assert.Contains("Newest blocked process sample: 23:12:45 UTC, 5m 20s old", G("bodyOtherClock")); + Assert.Contains(" Incident Since: " + H("inc"), G("bodyNoSample")); + Assert.Contains("Incident Since: 2026-10-12 23:13:00Z", G("bodyRaw")); + } + [Fact] public void ACappedPanelsHeading_DropsItsWindowLabel_AndAnyOtherKeepsIt() { diff --git a/Darling/Darling.Tests/web-clamp-cell-harness.mjs b/Darling/Darling.Tests/web-clamp-cell-harness.mjs index e8e0bedfa7..ba3d4ef985 100644 --- a/Darling/Darling.Tests/web-clamp-cell-harness.mjs +++ b/Darling/Darling.Tests/web-clamp-cell-harness.mjs @@ -108,4 +108,31 @@ out.localize = { expected: "not collected since " + new Date("2026-10-09T14:37:35Z").toLocaleString(), }; +/* H2 (#5644 W20 leftovers): the shapes the alert text, the sweep notes and the collection-health regression text use. Each is localized in an + opted-in cell and printed as written in a cell that is not opted in; a SQL text cell holding a Z-marked literal stays as written. */ +const shapes = [ + { raw: "until 2026-10-11 12:48:52Z", stamp: "2026-10-11T12:48:52Z", lead: "until " }, + { raw: "service started 2026-10-12T23:01:13.7792072Z", stamp: "2026-10-12T23:01:13.779Z", lead: "service started " }, + { raw: "Failed At: 2026-10-12 07:00:00Z", stamp: "2026-10-12T07:00:00Z", lead: "Failed At: " }, + { raw: "Incident Since: 2026-10-12 23:13:00 UTC", stamp: "2026-10-12T23:13:00Z", lead: "Incident Since: " }, + { raw: "at 2026-10-12 23:13:00.1234567Z", stamp: "2026-10-12T23:13:00.123Z", lead: "at " }, +]; +const sqlZ = "SELECT * FROM t WHERE d = '2026-01-01 00:00:00Z'"; +const serverClock = "start time 2026-09-27 21:32:43 (server clock)"; +const l2 = VIZ.table({ rows: [...shapes.map((x) => ({ query_text: sqlZ, message: x.raw, other: x.raw })), { query_text: sqlZ, message: serverClock, other: serverClock }] }, { + rowsKey: "rows", title: "Localize shapes", tools: false, + columns: [ + { key: "query_text", label: "Query", wrap: true }, + { key: "other", label: "Other", wrap: true }, + { key: "message", label: "Message", wrap: true, localizeText: true }, + { key: "message", label: "Clamped message", clamp: true, localizeText: true }, + ], +}); +const l2rows = l2.findAll((n) => n.tag === "tr").filter((n) => n.parent && n.parent.tag === "tbody"); +out.shapes = l2rows.map((tr, i) => { + const t = tr.children.map((c) => c.textContent); + return { sql: t[0], notOpted: t[1], opted: t[2], clamped: t[3], expected: i < shapes.length ? shapes[i].lead + new Date(shapes[i].stamp).toLocaleString() : null, raw: i < shapes.length ? shapes[i].raw : serverClock }; +}); +out.sqlZ = sqlZ; + console.log(JSON.stringify(out)); diff --git a/Darling/Darling.Tests/web-partial-window-harness.mjs b/Darling/Darling.Tests/web-partial-window-harness.mjs index 0dc4490424..54e25957d4 100644 --- a/Darling/Darling.Tests/web-partial-window-harness.mjs +++ b/Darling/Darling.Tests/web-partial-window-harness.mjs @@ -31,7 +31,7 @@ globalThis.document = { createElement: (t) => new FakeNode(t), createTextNode: ( globalThis.location = { hash: "#/server/a/events" }; const root = process.argv[2]; -const { windowFloorStrip, cappedReadNote, partialWindowSubtitle, markPartialSubtitle, localizeIsoText, emptyStrip, noticeStrip, el, localTime } = await import(pathToFileURL(root + "/util.js").href); +const { windowFloorStrip, cappedReadNote, partialWindowSubtitle, markPartialSubtitle, localizeIsoText, localizeAlertDetail, emptyStrip, noticeStrip, el, localTime } = await import(pathToFileURL(root + "/util.js").href); const text = (n) => (n ? n.textContent : null); const OLDEST = "2026-10-12T13:29:59.5027090Z"; @@ -101,4 +101,35 @@ out.bothNotice = noticeStrip("TRUNCATED: " + both).textContent; out.bannerText = "baseline discontinuity at 10/12/2026, 8:15:00 AM (restart) — instance identity changed — start time 2026-10-09 14:00:00 → 2026-10-12 08:15:00 (server clock); every delta baseline for this server was forgotten"; out.bannerNotice = noticeStrip(out.bannerText).textContent; +/* H2: the alert text and the sweep notes (#5644 W20 leftovers) */ +const here = (s) => new Date(s).toLocaleString(); +const alertBody = [ + "Blocking chain (x2)", + " Incident Since: 2026-10-12 23:13:00Z", + " Blocked Query: SELECT 1 WHERE d = '2026-01-01 00:00:00Z'", + " Failed At: 2026-10-12 07:00:00Z", + " Started: 2026-09-27 21:32:43 (server clock, UTC offset not collected)", + " Window: 2026-10-12 07:00:00Z → 2026-10-12 08:00:00Z", + "", + "Newest blocked process sample: 23:12:45 UTC, 5m 20s old", +].join("\n"); +const sample = { source: "blocked process", instant_utc: "2026-10-12T23:12:45.0000000Z", age_seconds: 320 }; +out.h2 = { + spaceZ: localizeIsoText("until 2026-10-11 12:48:52Z"), + spaceUtc: localizeIsoText("at 2026-10-12 23:13:00 UTC."), + frac7: localizeIsoText("service started 2026-10-12T23:01:13.7792072Z"), + spaceFrac7: localizeIsoText("at 2026-10-12 23:13:00.1234567Z"), + serverClock: localizeIsoText("start time 2026-09-27 21:32:43 (server clock)"), + expSpaceZ: "until " + here("2026-10-11T12:48:52Z"), + expSpaceUtc: "at " + here("2026-10-12T23:13:00Z") + ".", + expFrac7: "service started " + here("2026-10-12T23:01:13.779Z"), + expSpaceFrac7: "at " + here("2026-10-12T23:13:00.123Z"), + body: localizeAlertDetail(alertBody, sample), + bodyNoSample: localizeAlertDetail(alertBody, null), + bodyOtherClock: localizeAlertDetail(alertBody, { ...sample, instant_utc: "2026-10-12T23:12:46Z" }), + stripUtc: noticeStrip("until 2026-10-11 12:48:52Z").textContent, + here: { inc: here("2026-10-12T23:13:00Z"), fail: here("2026-10-12T07:00:00Z"), end: here("2026-10-12T08:00:00Z"), samp: here("2026-10-12T23:12:45Z") }, + bodyRaw: alertBody, +}; + console.log(JSON.stringify(out)); diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js index c13a7647ab..5afdf3f8cb 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js @@ -24,7 +24,7 @@ */ import { el, mount, readTool, apiSend, buildQuery, loadingStrip, errorStrip, emptyStrip, noticeStrip, disclosure, - ALERT_STATE_LABELS, alertDeliveryState, parseUtc } from "../util.js"; + ALERT_STATE_LABELS, alertDeliveryState, parseUtc, localizeAlertDetail } from "../util.js"; import { VIZ, reapplyGridSort, gridRowOf } from "../panels.js"; import { copyText } from "../grid-tools.js"; import { pageRangePicker, windowResultOfSpec } from "../page-range.js"; @@ -221,7 +221,7 @@ function detailCell(a) { const hasDetail = a.detail_text != null && String(a.detail_text).trim().length > 0; if (!hasDetail && !a.send_error) return el("span", { class: "muted", text: "—" }); - const summary = hasDetail ? a.detail_text : "Delivery error: " + a.send_error; + const summary = hasDetail ? localizeAlertDetail(a.detail_text, a.newest_sample) : "Delivery error: " + a.send_error; const placeholder = el("div", {}); const node = disclosure(summary, placeholder, { max: 120 }); @@ -295,7 +295,10 @@ function detailBody(a, details) { * the fix script. They replace the parsed detail_text, which is the same headings and fields without the prose. * A row without them (not an Analysis alert, no advice, a failed fetch) keeps the detail_text rendering below. */ const items = Array.isArray(details) && details.length > 0 ? details : null; - const fields = !items && hasDetail ? parseDetailFields(a.detail_text) : null; + /* #5644 W20: the stored text names UTC instants ("Incident Since: ...Z", the sample-age line's time of day); the page shows them + * in the grids' local format. The stored text, the copy and the CSV keep the stored form. */ + const shownDetail = hasDetail ? localizeAlertDetail(a.detail_text, a.newest_sample) : null; + const fields = !items && hasDetail ? parseDetailFields(shownDetail) : null; const body = []; if (items) { for (const item of items) body.push(...detailItem(item)); @@ -304,11 +307,11 @@ function detailBody(a, details) { * engine persisted when the alert fired, never re-measured against this page's clock) - once, and only when the * carrier is set. The detail_text branches below already carry the line in the text and must not add it again. */ const ageLine = sampleAgeLine(a.newest_sample); - if (ageLine !== null) body.push(el("div", { class: "detail-sample-age", text: ageLine })); + if (ageLine !== null) body.push(el("div", { class: "detail-sample-age", text: localizeAlertDetail(ageLine, a.newest_sample) })); } else if (fields) { body.push(el("div", { class: "detail-fields" }, fields.map(fieldRow))); } else if (hasDetail) { - body.push(el("div", { class: "detail-raw", text: a.detail_text })); + body.push(el("div", { class: "detail-raw", text: shownDetail })); } if (a.send_error) { body.push(el("div", { class: "detail-fields detail-error" }, [fieldRow(["Delivery error", a.send_error])])); diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/sweeps.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/sweeps.js index ec7429ac9b..e48c44d2e4 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/sweeps.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/sweeps.js @@ -24,7 +24,7 @@ */ import { el, mount, apiGet, readTool, loadingStrip, errorStrip, emptyStrip, localTime, relTime, rollupTextId, - fmtInt, fmtMb, fmtPct, fmtBool, fmtText, bandClass, disclosure } from "../util.js"; + fmtInt, fmtMb, fmtPct, fmtBool, fmtText, bandClass, disclosure, localizeIsoText } from "../util.js"; import { gridTable } from "../panels.js"; import { pageRangePicker, windowOfSpec } from "../page-range.js"; import { relativeSpec } from "../time-range.js"; @@ -230,7 +230,7 @@ function sweepDocument(d) { { key: "server", label: "Server" }, bandColumn("from", "From"), bandColumn("to", "To"), - { key: "reason", label: "Reason", wrap: true, display: (t) => t.reason || "—" }, + { key: "reason", label: "Reason", wrap: true, localizeText: true, display: (t) => t.reason || "—" }, ], transitions)); } for (const [label, list] of [["New servers", changes.new_servers], ["Departed servers", changes.departed_servers]]) { @@ -262,7 +262,7 @@ function sweepDocument(d) { { key: "server", label: "Server" }, bandColumn("previous_band", "Previous", (v) => v.previous_band || "—"), bandColumn("band", "Band", (v) => v.band, (v) => v.band_changed), - { key: "reason", label: "Reason", wrap: true, display: (v) => v.reason || "—" }, + { key: "reason", label: "Reason", wrap: true, localizeText: true, display: (v) => v.reason || "—" }, ], verdicts) : emptyStrip("This sweep carries no per-server verdicts.")); @@ -297,7 +297,7 @@ function livenessBlock(d, lv) { + faults.map((f) => f.server + " (" + f.error + ")").join("; ") }) : null, notes.length - ? el("ul", { class: "liveness-notes" }, notes.map((n) => el("li", { text: String(n) }))) + ? el("ul", { class: "liveness-notes" }, notes.map((n) => el("li", { text: localizeIsoText(String(n)) }))) : null, ]); } diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/triage.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/triage.js index 63995b0ffd..70313742b1 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/triage.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/triage.js @@ -21,7 +21,7 @@ */ import { el, mount, apiGet, buildQuery, loadingStrip, errorStrip, emptyStrip, noticeStrip, localTime, fmtNum, - alertDeliveryState } from "../util.js"; + alertDeliveryState, localizeAlertDetail } from "../util.js"; import { VIZ } from "../panels.js"; import { suggestViz, deriveVizConfig } from "../derive.js"; import { renderNotebookDoc } from "./views.js"; @@ -61,7 +61,7 @@ function alertFields(a) { ]) )); if (a.detail_text) { - return [box, el("pre", { class: "code", text: a.detail_text })]; + return [box, el("pre", { class: "code", text: localizeAlertDetail(a.detail_text, a.newest_sample) })]; } return [box]; } diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/views.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/views.js index 4b81c7ff9d..04f6589d32 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/views.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/views.js @@ -25,7 +25,7 @@ * descriptions) reaches the DOM through el()/textContent (R4 — never innerHTML). */ -import { el, mount, apiGetFleet, loadingStrip, errorStrip, emptyStrip, noticeStrip, relTime, localTime, localizeIsoText, fmtNum } from "../util.js"; +import { el, mount, apiGetFleet, loadingStrip, errorStrip, emptyStrip, noticeStrip, relTime, localTime, localizeIsoText, localizeAlertDetail, fmtNum } from "../util.js"; import { rollingHoursPicker } from "../page-range.js"; import { renderPanel, setPanelSignal, VIZ } from "../panels.js"; import { orderServers } from "../server-order.js"; @@ -896,7 +896,7 @@ function renderAlertHeaderCell(cell, opts) { )), /* detail_text (#4368): the same 
 block triage.js's alertFields uses — through el()'s
            text path (textContent), never innerHTML, since this is alert/query text the server only echoes. */
-        a.detail_text ? el("pre", { class: "code", text: a.detail_text }) : null,
+        a.detail_text ? el("pre", { class: "code", text: localizeAlertDetail(a.detail_text, a.newest_sample) }) : null,
       ]
     : emptyStrip("No matching alert history row for this link.");
   return el("div", { class: "notebook-panel panel card" }, [
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js
index 6882357466..290915c978 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js
@@ -188,17 +188,48 @@ export function metricNote(text) {
 }
 
 /**
- * Rewrites every ISO-8601 instant inside a sentence (a server message, a status line) into the grids' local-time format
- * (#5644 W20): "stored at 2026-10-09T14:37:35.0158630" and "since 2026-10-12T19:18:38Z" printed raw UTC beside grids in local
- * time. A stamp with no zone is UTC, as parseUtc reads it everywhere. Text with no such stamp, and a non-string, come back as given.
+ * Rewrites every UTC instant inside a sentence (a server message, a status line, a note, an alert body) into the grids'
+ * local-time format (#5644 W20): "stored at 2026-10-09T14:37:35.0158630", "since 2026-10-12T19:18:38Z", "until
+ * 2026-10-11 12:48:52Z" (the alert text's space-separated form) and "2026-10-12 23:13:00 UTC" printed raw UTC beside grids
+ * in local time. The separator is a "T" or a space and the fraction is 0-7 digits. A "T" stamp with no zone is UTC, as
+ * parseUtc reads it everywhere; a space-separated one is rewritten ONLY with a "Z" or " UTC" after it, because the same
+ * shape without one is a monitored server's own wall clock ("2026-09-27 21:32:43 (server clock)") and shifting it would
+ * state the wrong hour. Text with no such stamp, and a non-string, come back as given.
  */
 export function localizeIsoText(text) {
   if (typeof text !== "string") return text;
-  return text.replace(/\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?(?:Z|[+-]\d\d:?\d\d)?/g, (m) => {
-    const d = parseUtc(m);
+  return text.replace(/(\d{4}-\d\d-\d\d)([T ])(\d\d:\d\d:\d\d)(?:\.(\d{1,7}))?(Z(?![A-Za-z])| UTC(?![A-Za-z])|[+-]\d\d:?\d\d)?/g, (m, date, sep, time, frac, zone) => {
+    if (sep === " " && !zone) return m;
+    const iso = date + "T" + time + (frac ? "." + frac.slice(0, 3).padEnd(3, "0") : "") + (zone === " UTC" ? "Z" : zone || "");
+    const d = parseUtc(iso);
     return d ? d.toLocaleString() : m;
   });
 }
+
+/**
+ * An alert's stored body text in the grids' local-time format (#5644 W20, Alert History and Triage). The instants the alert
+ * builders write ("Incident Since: 2026-10-12 23:13:00Z", "Failed At: 2026-10-12 07:00:00Z", and the Started, Snapshot, From,
+ * To and Window lines) go through localizeIsoText. Only those labelled lines do: the same body carries "Blocked Query", "Query"
+ * and "Victim SQL" lines, and a literal such as '2026-01-01 00:00:00Z' inside a statement is data. The sample-age line ("Newest blocked process sample: 23:12:45 UTC, 5m 20s old") carries a time of day
+ * with no date, which cannot be shifted safely alone, so it takes its date from the row's own `newest_sample.instant_utc`
+ * (the carrier the engine persisted when the alert fired) and only when that instant's UTC time of day IS the one written
+ * on the line. The stored and emailed text is not changed (the same line feeds email, webhooks and Lite's windows); only
+ * what this page shows is. A copy of the text for the clipboard or CSV keeps the stored form.
+ */
+export function localizeAlertDetail(text, sample) {
+  if (typeof text !== "string") return text;
+  let out = text.replace(/^(\s*(?:Incident Since|Started|Failed At|Snapshot|From|To|Window): )(.*)$/gm,
+    (m, lead, value) => lead + localizeIsoText(value));
+  const at = sample && typeof sample.instant_utc === "string" ? parseUtc(sample.instant_utc) : null;
+  if (at) {
+    const two = (n) => String(n).padStart(2, "0");
+    const clock = two(at.getUTCHours()) + ":" + two(at.getUTCMinutes()) + ":" + two(at.getUTCSeconds());
+    out = out.replace(/^(\s*(?:Newest .+? sample|.+? reading)): (\d\d:\d\d:\d\d) UTC(?=, .* old\s*$)/gm,
+      (m, lead, time) => (time === clock ? lead + ": " + at.toLocaleString() : m));
+  }
+  return out;
+}
+
 /**
  * The two collectors the blocking reads draw from (#5244): the blocked-process-report XE session when it holds a row in the window
  * for the chosen databases, and the DMV snapshot only when it holds none. The MCP answers name the one that answered in `source`;

From e4c156f5deba1110d469c0516c58f58a3750f61b Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Mon, 12 Oct 2026 22:25:27 -0400
Subject: [PATCH 30/42] WIP #5644 H1: live-check wording fixes (uncommitted
 work of a hung lane; unverified)

---
 .../ChartIntegerTicksPinTests.cs              |  6 +--
 .../Darling.Tests/DarlingWebEndpointsTests.cs | 39 ++++++++++++++
 .../WebBlockingSourceBehaviourTests.cs        | 52 ++++++++++++++++++-
 .../WebChartAtTimeBehaviourTests.cs           |  2 +-
 Darling/Darling.Tests/WebPlainTextTests.cs    | 27 +++++++++-
 Darling/Darling.Tests/WebWordingFixesTests.cs |  9 ++++
 .../web-blocking-source-harness.mjs           | 22 +++++++-
 .../web-chart-at-time-harness.mjs             |  2 +-
 .../DarlingWebEndpoints.cs                    | 42 +++++++++++++++
 .../wwwroot/index.html                        |  2 +
 .../wwwroot/js/pages/server-tabs.js           | 34 ++++++++----
 .../wwwroot/js/panels.js                      | 11 +++-
 .../wwwroot/js/plain-text.js                  | 13 +++++
 .../wwwroot/js/view-templates.js              |  6 +--
 14 files changed, 243 insertions(+), 24 deletions(-)

diff --git a/Darling/Darling.Tests/ChartIntegerTicksPinTests.cs b/Darling/Darling.Tests/ChartIntegerTicksPinTests.cs
index bd486d2892..e67c2f5b3f 100644
--- a/Darling/Darling.Tests/ChartIntegerTicksPinTests.cs
+++ b/Darling/Darling.Tests/ChartIntegerTicksPinTests.cs
@@ -71,9 +71,9 @@ public void EveryCountChartOnTheServerPage_DeclaresWholeNumbers_AndVizLinePasses
 
         /* The four Blocking Events / Deadlocks line panels (Overview and Locking tabs): each one that charts
            COUNT_SERIES carries the format, counted against the panels that chart it. */
-        var countPanels = Regex.Matches(tabs, @"COUNT_SERIES, \{").Count;
+        var countPanels = Regex.Matches(tabs, @"(?:COUNT_SERIES|BLOCKING_REPORT_SERIES), \{").Count;
         Assert.Equal(4, countPanels);
-        Assert.Equal(countPanels, Regex.Matches(tabs, @"COUNT_SERIES, \{\n\s+subtitle: ctx\.label,\n\s+format: ""int"",").Count);
+        Assert.Equal(countPanels, Regex.Matches(tabs, @"(?:COUNT_SERIES|BLOCKING_REPORT_SERIES), \{\n\s+subtitle: ctx\.label,\n\s+format: ""int"",").Count);
 
         /* The Blocked Sessions chart is a fanout spec rather than a line() call. */
         Assert.Matches(@"series: BLOCKED_SESSION_SERIES,\n\s+format: ""int"",", tabs);
@@ -85,7 +85,7 @@ public void EveryCountChartOnTheServerPage_DeclaresWholeNumbers_AndVizLinePasses
     public void TheStarterDashboardAndComposedCountCharts_AskForWholeNumberTicks()
     {
         var templates = Js("view-templates.js");
-        Assert.Matches(@"series: \[\{ key: ""count"", label: ""Events"" \}\],\n\s+format: ""int"",", templates);
+        Assert.Matches(@"series: \[\{ key: ""count"", label: ""Reports"" \}\],\n\s+format: ""int"",", templates);
         Assert.Contains("...READ_FIELDS.get_deadlock_trend.line,", templates, StringComparison.Ordinal);
         Assert.Matches(@"get_deadlock_trend: \{[\s\S]*?line: \{[^}]*?format: ""int"",[^}]*?label: ""Deadlocks""", Js("read-fields.js"));
 
diff --git a/Darling/Darling.Tests/DarlingWebEndpointsTests.cs b/Darling/Darling.Tests/DarlingWebEndpointsTests.cs
index c02c1c2f28..8e1381ccf2 100644
--- a/Darling/Darling.Tests/DarlingWebEndpointsTests.cs
+++ b/Darling/Darling.Tests/DarlingWebEndpointsTests.cs
@@ -506,6 +506,45 @@ public void NonIntegerParamRefusal_LeavesAbsentEmptyAndWholeNumbersToTheTool(str
         }
     }
 
+    /* #5644 W22: the tools refuse an out-of-range window as hours_back; a web caller that sent hours is told about hours. */
+    private static string HoursBackRefusal(int value) =>
+        McpHelpers.ValidateHoursBack(value)!;
+
+    [Theory]
+    [InlineData(-5)]
+    [InlineData(500)]
+    public void WindowRefusal_NamesTheHoursParameterTheWebCallerSent(int value)
+    {
+        var refusal = HoursBackRefusal(value);
+        Assert.Contains("hours_back", refusal);
+
+        var named = DarlingWebEndpoints.NameWindowParameterTheCallerSent(refusal, k => k == "hours" ? value.ToString() : null);
+
+        var doc = System.Text.Json.JsonDocument.Parse(named).RootElement;
+        Assert.Equal("invalid", doc.GetProperty("status").GetString());
+        Assert.Equal("hours", doc.GetProperty("hints").GetProperty("parameter").GetString());
+        var message = doc.GetProperty("message").GetString()!;
+        Assert.DoesNotContain("hours_back", message);
+        Assert.Contains($"hours value '{value}'", message);
+    }
+
+    [Fact]
+    public void WindowRefusal_ForAnMcpStyleRequest_OrAnotherParameter_IsLeftAlone()
+    {
+        var refusal = HoursBackRefusal(-5);
+
+        /* The caller sent hours_back (alone or beside hours): the tool's own wording is right. */
+        Assert.Equal(refusal, DarlingWebEndpoints.NameWindowParameterTheCallerSent(refusal, k => k == "hours_back" ? "-5" : null));
+        Assert.Equal(refusal, DarlingWebEndpoints.NameWindowParameterTheCallerSent(refusal, k => k is "hours" or "hours_back" ? "-5" : null));
+        /* No window parameter sent, and a result that is data rather than a refusal. */
+        Assert.Equal(refusal, DarlingWebEndpoints.NameWindowParameterTheCallerSent(refusal, _ => null));
+        const string Data = "{\"status\":\"ok\",\"hints\":{\"parameter\":\"hours_back\"}}";
+        Assert.Equal(Data, DarlingWebEndpoints.NameWindowParameterTheCallerSent(Data, k => k == "hours" ? "1" : null));
+        /* A refusal for a different parameter keeps its wording, even when it mentions hours_back. */
+        var other = McpHelpers.Refusal("baseline_hours_back", "baseline_hours_back must be greater than hours_back.");
+        Assert.Equal(other, DarlingWebEndpoints.NameWindowParameterTheCallerSent(other, k => k == "hours" ? "4" : null));
+    }
+
     [Fact]
     public void NonIntegerParamRefusal_ThroughTheQueryString_ReadsTheFirstNonEmptyValue()
     {
diff --git a/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs b/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs
index 4932610af8..acfb462840 100644
--- a/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs
+++ b/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs
@@ -76,7 +76,7 @@ public void BlockingEventsAndSeverity_NameTheCollectorThatAnswered(string scenar
         var r = Run(scenario);
 
         /* Blocking Events is on the Overview tab and the Blocking tab; Blocking Severity only on the Blocking tab. */
-        var events = Lines(r, "Blocking Events");
+        var events = Lines(r, "Blocked process report trend");
         Assert.Equal(2, events.Length);
         Assert.All(events, lines => Assert.Equal(new[] { expected }, lines));
         var severity = Lines(r, "Blocking Severity");
@@ -96,6 +96,54 @@ public void NoRows_OrNoRecognizedSource_DrawsNoSourceLine(string scenario)
         Assert.All(r.GetProperty("panels").EnumerateArray(), p => Assert.Empty(p.GetProperty("strips").EnumerateArray()));
     }
 
+    /// The empty sentences every panel of one tab drew, for the panels with that title.
+    private static string[][] Empties(JsonElement r, string tab, string title) =>
+        r.GetProperty("panels").EnumerateArray()
+            .Where(p => p.GetProperty("tab").GetString() == tab && p.GetProperty("title").GetString() == title)
+            .Select(p => p.GetProperty("empties").EnumerateArray().Select(s => s.GetString()!).ToArray())
+            .ToArray();
+
+    /// #5644 W16: each deadlock read answers its own empty message (the trend's carries the collector-run count, the table's says
+    /// "specified time range"), and only the Deadlock Severity panel showed the page's sentence. Every deadlock empty state on the
+    /// Blocking tab now says that one sentence, whatever the read said.
+    [Fact]
+    public void EveryDeadlockEmptyState_OnTheBlockingTab_SaysTheOneSentence()
+    {
+        var r = Run("deadlockEmpty");
+        const string One = "No deadlocks in this window.";
+
+        /* The Blocking tab: the chart (titled Deadlock trend), the Deadlocks grid and the Deadlock Severity panel. */
+        Assert.Equal(new[] { One }, Assert.Single(Empties(r, "blocking", "Deadlock trend")));
+        Assert.Equal(new[] { One }, Assert.Single(Empties(r, "blocking", "Deadlocks")));
+        Assert.Equal(new[] { One }, Assert.Single(Empties(r, "blocking", "Deadlock Severity")));
+    }
+
+    /// #5644 W16: the panels that are NOT deadlock states keep the read's own message (the sentence is a per-panel opt-in).
+    [Fact]
+    public void ABlockingEmptyState_StillShowsTheReadsOwnMessage()
+    {
+        var r = Run("deadlockEmpty");
+
+        /* get_blocking_stats answered the envelope, so Blocking Severity (same read as Deadlock Severity) shows the read's message. */
+        Assert.Equal(new[] { "No blocking or deadlock data in the specified time range." }, Assert.Single(Empties(r, "blocking", "Blocking Severity")));
+    }
+
+    /// #5644 W16: the chart counts one point per blocked-process REPORT while the tile beside it counts one event per wait, so
+    /// the chart says reports (title, legend and empty sentence), on the Overview and Blocking tabs and in the starter dashboard.
+    [Fact]
+    public void TheBlockingTrendChart_SaysItCountsBlockedProcessReports()
+    {
+        var tabs = ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js", "pages", "server-tabs.js");
+        var templates = ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js", "view-templates.js");
+
+        Assert.DoesNotContain("\"Blocking Events\"", tabs);
+        Assert.DoesNotContain("\"Blocking Events\"", templates);
+        Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(tabs, "line\\(\"Blocked process report trend\", \"get_blocking_trend\"").Count);
+        Assert.Contains("const BLOCKING_REPORT_SERIES = [{ key: \"count\", label: \"Reports\" }];", tabs);
+        Assert.DoesNotContain("No blocking events in this window — an empty trend", tabs);
+        Assert.Contains("label: \"Reports\"", templates);
+    }
+
     [Fact]
     public void OnlyTheBlockingPanelsGetOne()
     {
@@ -105,6 +153,6 @@ public void OnlyTheBlockingPanelsGetOne()
             .Where(p => p.GetProperty("strips").GetArrayLength() > 0)
             .Select(p => p.GetProperty("title").GetString())
             .ToArray();
-        Assert.Equal(new[] { "Blocking Events", "Blocking Events", "Blocking Severity" }, titles);
+        Assert.Equal(new[] { "Blocked process report trend", "Blocked process report trend", "Blocking Severity" }, titles);
     }
 }
diff --git a/Darling/Darling.Tests/WebChartAtTimeBehaviourTests.cs b/Darling/Darling.Tests/WebChartAtTimeBehaviourTests.cs
index f56f9e658c..a786b8e389 100644
--- a/Darling/Darling.Tests/WebChartAtTimeBehaviourTests.cs
+++ b/Darling/Darling.Tests/WebChartAtTimeBehaviourTests.cs
@@ -355,7 +355,7 @@ public void TheBlockingAndDeadlockCharts_NameTheirOwnItem_AndNoOtherChartDoes()
 
         // line() panels: Blocking Events sits on both the Overview tab and the Blocking tab; the deadlock chart is "Deadlocks" on Overview and
         // "Deadlock trend" on the Blocking tab (#5644 W16, so the Deadlocks grid below it is the only panel with that title there).
-        foreach (var (title, item, count) in new[] { ("Blocking Events", "blocking", 2), ("Deadlocks", "deadlocks", 1), ("Deadlock trend", "deadlocks", 1), ("Lock Waits", "blocking", 1) })
+        foreach (var (title, item, count) in new[] { ("Blocked process report trend", "blocking", 2), ("Deadlocks", "deadlocks", 1), ("Deadlock trend", "deadlocks", 1), ("Lock Waits", "blocking", 1) })
         {
             var calls = Regex.Matches(tabs, @"line\(""" + title + @""",");
             Assert.Equal(count, calls.Count);
diff --git a/Darling/Darling.Tests/WebPlainTextTests.cs b/Darling/Darling.Tests/WebPlainTextTests.cs
index eae57e27bc..16750687f2 100644
--- a/Darling/Darling.Tests/WebPlainTextTests.cs
+++ b/Darling/Darling.Tests/WebPlainTextTests.cs
@@ -97,6 +97,31 @@ public void CollectorMeasurementCounts_ReadAsWords()
         Assert.Equal("latest run: shred gated: 1, events read: 0, report xml empty: 0", o[0]);
     }
 
+    /* #5644 W8 (live check): the CPU and Memory pages have no bucket control, the Waits and Activity notes print raw field names, and a
+       window count printed "24 hour(s)". */
+    [Fact]
+    public void TrendNotes_NameTheTimeRange_TheColumnLabels_AndAgreeInNumber()
+    {
+        var o = Run(
+        [
+            "Each point averages the samples in one 5-minute bucket and is stamped at the bucket's start. A level is averaged, never summed, so a spike inside a bucket is smoothed into its average; narrow the bucket to see it. The width was chosen to keep this answer near 1500 points; pass bucket_minutes (1-1440) for another width, or narrow hours_back for finer points.",
+            "wait_time_ms_per_second is recomputed from the summed wait time over the seconds the collections covered.",
+            "collisions_per_second is recomputed from the summed delta.",
+            "The session counts are levels, averaged, never summed; top_application_name, top_application_connections, top_host_name and top_host_connections are the newest collection's own values in the bucket.",
+            "No query-stats group on SRV in the last 24 hour(s) has a cached plan.",
+            "No blocking was recorded in the last 1 hour(s). 288 collector run(s) DID execute; 1 collector run(s) failed.",
+        ], out _);
+        Assert.DoesNotContain("narrow the bucket", o[0]);
+        Assert.Contains("narrow the time range to see it", o[0]);
+        Assert.DoesNotContain("wait_time_ms_per_second", o[1]);
+        Assert.StartsWith("Wait ms/s is recomputed", o[1]);
+        Assert.StartsWith("Coll/s is recomputed", o[2]);
+        Assert.DoesNotContain("top_application_name", o[3]);
+        Assert.Contains("the top application, its connection count, the top host and its connection count are", o[3]);
+        Assert.Equal("No query-stats group on SRV in the last 24 hours has a cached plan.", o[4]);
+        Assert.Equal("No blocking was recorded in the last 1 hour. 288 collector runs DID execute; 1 collector run failed.", o[5]);
+    }
+
     /* #5644 W8: the clauses that point an MCP caller at a parameter the page has no control for. */
     [Fact]
     public void ParameterClauses_ForAnMcpCaller_AreDroppedOrSaidInThePagesWords()
@@ -123,7 +148,7 @@ public void ParameterClauses_ForAnMcpCaller_AreDroppedOrSaidInThePagesWords()
         Assert.Equal(" truncated: 5 of 9 returned.", o[4]);
         Assert.EndsWith("still cover every database.", o[5]);
         Assert.Contains("per database - so its totals", o[6]);
-        Assert.Equal("No active query snapshots on SQL2025 in the last 1 hour(s) matched blocking queries only. The filters were applied in SQL over the whole window.", o[7]);
+        Assert.Equal("No active query snapshots on SQL2025 in the last 1 hour matched blocking queries only. The filters were applied in SQL over the whole window.", o[7]);
         Assert.Contains("matched database 'Sales' with blocking queries only with wait type 'LCK_M_X'.", o[8]);
         Assert.Contains("so this is a normal result, not missing data.", o[9]);
         Assert.EndsWith("Select a single server to see the rest.", o[10]);
diff --git a/Darling/Darling.Tests/WebWordingFixesTests.cs b/Darling/Darling.Tests/WebWordingFixesTests.cs
index ac834060fd..177b925fc3 100644
--- a/Darling/Darling.Tests/WebWordingFixesTests.cs
+++ b/Darling/Darling.Tests/WebWordingFixesTests.cs
@@ -12,6 +12,15 @@ public sealed class WebWordingFixesTests
     private static string ServerTabs() =>
         ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js", "pages", "server-tabs.js").ReplaceLineEndings("\n");
 
+    [Fact]
+    public void TheAppShell_DeclaresAnIcon_SoThePageLoadDoesNotRequestFaviconIco()
+    {
+        /* #5644: every page load asked for /favicon.ico and got a 404 (a console error). The shell names an inline empty icon, which the
+           page's img-src (self and data:) allows; no request is made. */
+        var html = ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "index.html");
+        Assert.Contains("", html, StringComparison.Ordinal);
+    }
+
     [Fact]
     public void ThePickerPlurals_AreWrittenOut_NotBuiltWithAnS()
     {
diff --git a/Darling/Darling.Tests/web-blocking-source-harness.mjs b/Darling/Darling.Tests/web-blocking-source-harness.mjs
index e1e4d2b3b9..44a35f4923 100644
--- a/Darling/Darling.Tests/web-blocking-source-harness.mjs
+++ b/Darling/Darling.Tests/web-blocking-source-harness.mjs
@@ -116,6 +116,14 @@ const BODIES = {
     trend: { status: "empty", message: "No blocking." },
     stats: { status: "empty", message: "No blocking." },
   },
+  /* #5644 W16: every deadlock read answers the {status, message} envelope with ITS OWN wording (the trend's carries the collector-run
+     count, the table's says "specified time range"); the stats read, which draws Deadlock Severity, says a third thing. */
+  deadlockEmpty: {
+    trend: { source: null, trend: [] },
+    stats: { status: "empty", message: "No blocking or deadlock data in the specified time range." },
+    deadlockTrend: { status: "empty", message: "No deadlock was recorded for SRV1 in the last 24 hour(s). 288 collector run(s) DID execute." },
+    deadlocks: { status: "empty", message: "No deadlocks found in the specified time range." },
+  },
 };
 
 const settle = async () => {
@@ -134,10 +142,12 @@ const panelsOf = (root) => {
     if (!hasClass(node, "panel")) return;
     const h3 = node.children.find((c) => c.tag === "h3");
     const strips = [];
+    const empties = [];
     walk(node, (x) => {
       if (hasClass(x, "source")) strips.push(x.textContent);
+      if (hasClass(x, "empty")) empties.push(x.textContent);
     });
-    found.push({ title: h3 && h3.children[0] ? h3.children[0].textContent : "", strips });
+    found.push({ title: h3 && h3.children[0] ? h3.children[0].textContent : "", strips, empties });
   });
   return found;
 };
@@ -146,7 +156,15 @@ const body = BODIES[scenario];
 if (!body) throw new Error("unknown scenario " + scenario);
 globalThis.fetch = async (url) => {
   const u = String(url);
-  const answer = u.includes("/get_blocking_trend") ? body.trend : u.includes("/get_blocking_stats") ? body.stats : {};
+  const answer = u.includes("/get_blocking_trend")
+    ? body.trend
+    : u.includes("/get_blocking_stats")
+      ? body.stats
+      : u.includes("/get_deadlock_trend")
+        ? body.deadlockTrend || {}
+        : u.includes("/get_deadlocks")
+          ? body.deadlocks || {}
+          : {};
   return { status: 200, ok: true, text: async () => JSON.stringify(answer) };
 };
 
diff --git a/Darling/Darling.Tests/web-chart-at-time-harness.mjs b/Darling/Darling.Tests/web-chart-at-time-harness.mjs
index 347651abe9..49e144a448 100644
--- a/Darling/Darling.Tests/web-chart-at-time-harness.mjs
+++ b/Darling/Darling.Tests/web-chart-at-time-harness.mjs
@@ -99,7 +99,7 @@ const panelHead = (title, index) => {
 const tabPanels = {
   queries: ["Active Queries"],
   /* The Blocking tab in its order: the Deadlocks chart (1) and the Deadlocks grid (3) share a title. */
-  blocking: ["Blocking Events", "Deadlocks", "Blocking", "Deadlocks", "Waiting Tasks", "Blocked Sessions", "Lock Waits", "Blocking Severity", "Deadlock Severity", "Deadlock Graphs", "Blocked Process Reports"],
+  blocking: ["Blocked process report trend", "Deadlocks", "Blocking", "Deadlocks", "Waiting Tasks", "Blocked Sessions", "Lock Waits", "Blocking Severity", "Deadlock Severity", "Deadlock Graphs", "Blocked Process Reports"],
 };
 const runHashchange = () => {
   panelHeads = (tabPanels[hashValue.split("/").pop()] || []).map((title, index) => panelHead(title, index));
diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
index 27a34bd5e8..f71257aa29 100644
--- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
@@ -16,6 +16,7 @@
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
@@ -416,6 +417,10 @@ is left as sent. */
 
                     result = await handler(context, postgres, analysis);
 
+                    /* #5644 W22: the tools' own window parameter is hours_back, but a web caller sends hours; a refusal for an
+                       out-of-range one (?hours=-5, ?hours=500) is worded in the caller's own parameter. */
+                    result = NameWindowParameterTheCallerSent(result, key => First(context, key));
+
                     /* #4966: a grid over a window says where its table's data starts when that is after the window's
                        start. Added to the web mirror only, never to the tool, so the MCP payloads are unchanged. A
                        failed probe returns the tool's answer as it was. */
@@ -6466,6 +6471,43 @@ private static Task UnparseableParam(string key, string? expected = null
         return null;
     }
 
+    /// 
+    /// #5644 W22: the window refusal (hints.parameter hours_back) a web caller got for ?hours= out of range, said in
+    /// the parameter the caller sent. The tools name hours_back (the MCP parameter, kept as it is for MCP callers); the web
+    /// accepts both spellings, and a request that sent only hours was told off for a parameter it never used. A request that
+    /// sent hours_back (alone or beside hours), a refusal for any other parameter, and a result that is not a refusal
+    /// envelope come back unchanged. Only the standalone word is renamed: baseline_hours_back stays.
+    /// 
+    internal static string NameWindowParameterTheCallerSent(string result, Func lookup)
+    {
+        if (!McpHelpers.IsRefusalEnvelope(result) || string.IsNullOrEmpty(lookup("hours")) || !string.IsNullOrEmpty(lookup("hours_back")))
+        {
+            return result;
+        }
+
+        try
+        {
+            if (JsonNode.Parse(result) is not JsonObject envelope
+                || envelope["hints"] is not JsonObject hints
+                || hints["parameter"]?.GetValue() != "hours_back")
+            {
+                return result;
+            }
+
+            hints["parameter"] = "hours";
+            if (envelope["message"]?.GetValue() is { } message)
+            {
+                envelope["message"] = Regex.Replace(message, @"(?
         NonIntegerParamRefusal(key => First(context, key));
 
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/index.html b/Darling/PerformanceMonitor.Darling.Service/wwwroot/index.html
index 24c2170e31..989429020a 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/index.html
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/index.html
@@ -9,6 +9,8 @@
     
     
     Darling Web
+    
+    
     
     
     
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js
index 6d9d074323..8f79a37c1e 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js
@@ -40,7 +40,7 @@
  */
 
 import { el, makeActivatable, readTool, readToolWithinKeptHistory, keptWindowStrip, windowFloorStrip, markPartialSubtitle, mount, truncate, loadingStrip, errorStrip, readErrorStrip, emptyStrip, disclosure, noticeStrip, metricNote, getPath, fmtMs, fmtRate, localTime, parseUtc, windowFromHours, daysText, dbScopeChip, dbFilteredEmptyText, sourceStrip, clampText, getActiveDatabaseFilter, queryWaitFilter, setQueryWaitFilter, waitIsLinked } from "../util.js";
-import { renderPanel, setPanelSignal, getPanelSignal, VIZ, worstFirst, newestFirst } from "../panels.js";
+import { renderPanel, emptyHeadline, setPanelSignal, getPanelSignal, VIZ, worstFirst, newestFirst } from "../panels.js";
 import { zoomableLineChart, chartZoomScope, SERIES_COLORS, CATEGORICAL_COLORS } from "../charts.js";
 import { multiPicker, mergeSeriesRows } from "../multi-picker.js";
 import { pgPlanColumn } from "./pg-plan-viewer.js";
@@ -217,7 +217,7 @@ function fanout(read, params, specs) {
       const body = shells[i].body;
       if (hidesPanel(spec, res, shells[i], keys[i])) return;
       if (res.kind === "error") return mount(body, readErrorStrip(res.message));
-      if (res.kind === "empty") return mount(body, [keptWindowStrip(res), windowFloorStrip(res.data, spec), emptyStrip(res.message)]);
+      if (res.kind === "empty") return mount(body, [keptWindowStrip(res), windowFloorStrip(res.data, spec), emptyStrip(emptyHeadline(spec, res))]);
       try {
         /* #2802: a fanout spec carries no `params` of its own (the window lives on the shared fetch above), so
            hand vizLine the fetch's `hours` as `windowHours` — otherwise a fanout line panel (Current Waits,
@@ -1000,9 +1000,9 @@ function pivot(rows, { xKey, seriesKey, valueKey }, maxSeries = 8) {
  * and its process rows follow the database filter.
  * `control` (#5226) is a node drawn under the title, before the rows: the ranking selector on the Top Queries and Top Procedures cards.
  */
-function table(title, read, params, rowsKey, columns, subtitle, emptyText, span = 2, noteKey = null, moreNoteKeys = null, columnGroups = null, control = null, dbScope = null, extensionMissingLine = null) {
+function table(title, read, params, rowsKey, columns, subtitle, emptyText, span = 2, noteKey = null, moreNoteKeys = null, columnGroups = null, control = null, dbScope = null, extensionMissingLine = null, emptyWins = false) {
   if (!emptyText) throw new Error("table(" + title + "): a table panel must explain its own empty state.");
-  const desc = { title, subtitle, read, params, viz: "table", rowsKey, columns, emptyText, moreNoteKeys, span, noteKey, dbScope, extensionMissingLine };
+  const desc = { title, subtitle, read, params, viz: "table", rowsKey, columns, emptyText, moreNoteKeys, span, noteKey, dbScope, extensionMissingLine, emptyWins };
   if (columnGroups) Object.assign(desc, { groups: columnGroups.groups, defaultGroups: columnGroups.defaultGroups });
   if (control) desc.control = control;
   return renderPanel(desc);
@@ -1054,6 +1054,8 @@ function line(title, read, params, rowsKey, xKey, series, opts = {}) {
     format: opts.format,
     unit: opts.unit,
     emptyText: opts.emptyText,
+    /* #5644 W16: the page's one sentence replaces the read's own empty message. */
+    emptyWins: opts.emptyWins === true,
     /* #5244: the answer's field naming the collector that answered (get_blocking_trend's `source`); absent on every other line. */
     sourceKey: opts.sourceKey,
     /* The panel is not drawn when the server answers "not collected" (it does not apply to this kind of server). */
@@ -1095,12 +1097,12 @@ export const SERVER_TABS = [
         format: "mb",
         emptyText: "No memory samples in this window.",
       }),
-      line("Blocking Events", "get_blocking_trend", { server, hours: ctx.hours }, "trend", "time", COUNT_SERIES, {
+      line("Blocked process report trend", "get_blocking_trend", { server, hours: ctx.hours }, "trend", "time", BLOCKING_REPORT_SERIES, {
         subtitle: ctx.label,
         format: "int",
         atTimeItem: "blocking",
         sourceKey: "source",
-        emptyText: "No blocking events in this window — an empty trend here means none happened, not that nothing was collected.",
+        emptyText: "No blocked process reports in this window — an empty trend here means none were captured, not that nothing was collected.",
       }),
       line("Deadlocks", "get_deadlock_trend", { server, hours: ctx.hours }, "trend", "time", COUNT_SERIES, {
         subtitle: ctx.label,
@@ -1316,18 +1318,19 @@ export const SERVER_TABS = [
       daysText(widestHours) +
       ". A Custom View can show more blocking and deadlock history.",
     build: (server, ctx) => [
-      line("Blocking Events", "get_blocking_trend", { server, hours: ctx.hours }, "trend", "time", COUNT_SERIES, {
+      line("Blocked process report trend", "get_blocking_trend", { server, hours: ctx.hours }, "trend", "time", BLOCKING_REPORT_SERIES, {
         subtitle: ctx.label,
         format: "int",
         atTimeItem: "blocking",
         sourceKey: "source",
-        emptyText: "No blocking events in this window — an empty trend here means none happened, not that nothing was collected.",
+        emptyText: "No blocked process reports in this window — an empty trend here means none were captured, not that nothing was collected.",
       }),
       line("Deadlock trend", "get_deadlock_trend", { server, hours: ctx.hours }, "trend", "time", COUNT_SERIES, {
         subtitle: ctx.label,
         format: "int",
         atTimeItem: "deadlocks",
         emptyText: DEADLOCKS_EMPTY_TEXT,
+        emptyWins: true,
       }),
       table(
         "Blocking",
@@ -1351,7 +1354,13 @@ export const SERVER_TABS = [
         ctx.label,
         DEADLOCKS_EMPTY_TEXT,
         2,
-        "separately_monitored_note"
+        "separately_monitored_note",
+        null,
+        null,
+        null,
+        null,
+        null,
+        true
       ),
       /* #2484: the Current Waits tab the viewer has and the browser did not. ONE read, two panels --
          via fanout, not two line() calls, because the tab must not fetch the same read twice (there is
@@ -1418,7 +1427,8 @@ export const SERVER_TABS = [
           rowsKey: "deadlock_severity",
           xKey: "time",
           series: DEADLOCK_SEVERITY_SERIES,
-          emptyText: "No deadlocks in this window.",
+          emptyText: DEADLOCKS_EMPTY_TEXT,
+          emptyWins: true,
         },
       ]),
       deadlockGraphsPanel(server, ctx),
@@ -3166,6 +3176,10 @@ const MEMORY_SERIES = [
    put their gridlines on whole numbers (charts.js integerTicks) instead of repeating one label down a 0-1 axis. */
 const COUNT_SERIES = [{ key: "count", label: "Events" }];
 
+/* #5644 W16: get_blocking_trend counts one point per blocked-process REPORT (a single wait can produce several), so the
+   chart says "reports" where the tile beside it counts one event per wait. Same read and same count as COUNT_SERIES. */
+const BLOCKING_REPORT_SERIES = [{ key: "count", label: "Reports" }];
+
 /* #2484: the aggregate lock-wait rate. One numeric key, per the same reasoning as the Current Waits series
    below — the LCK wait type is the grouping the read applied, not a second axis. */
 /* Stays local: this page labels the series "Lock wait (ms/sec)", not the catalog's label. */
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js
index 72cc8a7462..72d4c89953 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js
@@ -88,6 +88,15 @@ export function getPanelSignal() {
   return panelSignal;
 }
 
+/**
+ * The sentence an empty envelope shows (#5644 W16). A panel that sets `emptyWins` has ONE agreed sentence for its
+ * empty state (the Blocking tab's deadlock panels, whose reads each worded "nothing happened" differently), and it
+ * replaces the read's own message; every other panel keeps the server's message.
+ */
+export function emptyHeadline(desc, res) {
+  return desc && desc.emptyWins && desc.emptyText ? desc.emptyText : res.message;
+}
+
 /**
  * Build a panel node. It returns immediately with a loading strip and fills itself once the fetch resolves,
  * mapping the API response kinds (data / empty envelope / error / aborted / auth) to the right UI.
@@ -168,7 +177,7 @@ async function loadPanelBody(desc, body, signal) {
        window's start (the server adds the note to that envelope only, never to an unavailable or not_collected one), so a
        new server's empty week does not read as a quiet one. windowFloorStrip is null for a chart and for an envelope
        without the note. */
-    mount(body, [kept, windowFloorStrip(res.data, desc), emptyStrip(res.message)]);
+    mount(body, [kept, windowFloorStrip(res.data, desc), emptyStrip(emptyHeadline(desc, res))]);
     return;
   }
 
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js
index a165249b54..2a478f9744 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js
@@ -185,6 +185,19 @@ export function plainText(text) {
   /* A trend's width note: "...near 1500 points; pass bucket_minutes (1-1440) for another width, or narrow hours_back for finer
      points." The page picks the width, so only the narrower range is left to say. */
   t = t.replace(/pass bucket_minutes \(1-\d+\) for another width, or /g, "");
+  /* #5644 W8: a trend note's closing advice, and the field names it explains. The CPU and Memory pages have no bucket control (the
+     page picks the width), so "narrow the bucket" is said as the control that exists, the time range; the Waits, Latch/Spinlock
+     and Activity notes name their fields by the column or tile the page shows. */
+  t = t.replace(/\bnarrow the bucket to see it\b/g, "narrow the time range to see it");
+  t = t.replace(/\bwait_time_ms_per_second is recomputed\b/g, "Wait ms/s is recomputed");
+  t = t.replace(/\bcollisions_per_second is recomputed\b/g, "Coll/s is recomputed");
+  t = t.replace(
+    /\btop_application_name, top_application_connections, top_host_name and top_host_connections are\b/g,
+    "the top application, its connection count, the top host and its connection count are",
+  );
+  /* "24 hour(s)", "288 collector run(s)": the count is known, so the word agrees with it ("1 hour", "24 hours"). */
+  t = t.replace(/(? n + " " + word + (n === "1" ? "" : "s"));
+
   /* "raise limit": the pages have no row-limit control. "...filter - raise limit to see more." and "...returned; raise limit
      (now 50) for more." lose the clause; ", or raise limit," and ", and raise limit if none ... shows one" lose theirs. */
   t = t.replace(/\s*[;—–-]+\s*raise (?:the )?limit(?: \(now [\d,]+\))?(?: (?:to see|for) (?:more|the rest(?: of the rows)?))?(?=\.)/g, "");
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js
index a383019860..b1f7090754 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js
@@ -183,14 +183,14 @@ export const DASHBOARD_TEMPLATES = [
       definition: {
         panels: [
           {
-            title: "Blocking Events",
+            title: "Blocked process report trend",
             read: "get_blocking_trend",
             params: { server, hours: 24 },
             viz: "line",
             rowsKey: "trend",
             xKey: "time",
-            emptyText: "No blocking events in this window — an empty trend here means none happened, not that nothing was collected.",
-            series: [{ key: "count", label: "Events" }],
+            emptyText: "No blocked process reports in this window — an empty trend here means none were captured, not that nothing was collected.",
+            series: [{ key: "count", label: "Reports" }],
             format: "int",
           },
           {

From dcdafef2b88da7dbacd43398f93438116cdb4356 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Mon, 12 Oct 2026 22:53:06 -0400
Subject: [PATCH 31/42] Default Trace readers keep two real events with
 identical columns; event series do not break into dots (#5644)

---
 ...rlingDefaultTraceReaderServerClockTests.cs | 32 +++++++++++++++++--
 .../StoredServerClockRowTextTests.cs          |  5 +--
 .../ViewerServerClockEventReadsTests.cs       | 29 +++++++++++++++--
 .../ViewerSystemEventsDataStartTests.cs       |  3 +-
 .../WebChartGapBehaviourTests.cs              | 22 +++++++++++++
 .../Darling.Tests/web-chart-gap-harness.mjs   | 10 ++++++
 .../Mcp/DarlingDefaultTraceReader.cs          | 24 ++++++++++----
 .../wwwroot/js/charts.js                      | 11 +++++--
 .../ViewerDataService.SystemEvents.cs         | 16 ++++++----
 Lite.Tests/DefaultTraceServerClockTests.cs    | 27 ++++++++++++++--
 .../Services/LocalDataService.SystemEvents.cs |  9 ++++--
 11 files changed, 160 insertions(+), 28 deletions(-)

diff --git a/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs b/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs
index d192962816..712b1fb1ab 100644
--- a/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs
+++ b/Darling/Darling.Tests/DarlingDefaultTraceReaderServerClockTests.cs
@@ -31,10 +31,16 @@ public sealed class DarlingDefaultTraceReaderServerClockTests
 {
     /* Column 0 is the server-local event time; 4 is database_name, which carries a label so a test can say
        which event came back. The other columns stay NULL. */
-    private static DataTable NewTable() => Table(15, 0);
+    private static DataTable NewTable()
+    {
+        var table = Table(15, 0);
+        table.Columns.Add("c15", typeof(long));   /* event_sequence */
+        return table;
+    }
 
-    private static void AddEvent(DataTable table, DateTime? local, string label) =>
-        AddRow(table, (0, local.HasValue ? (object)local.Value : DBNull.Value), (1, "Data File Auto Grow"), (4, label));
+    private static void AddEvent(DataTable table, DateTime? local, string label, long? eventSequence = null) =>
+        AddRow(table, (0, local.HasValue ? (object)local.Value : DBNull.Value), (1, "Data File Auto Grow"), (4, label),
+            (15, eventSequence.HasValue ? (object)eventSequence.Value : DBNull.Value));
 
     private static async Task> ReadAsync(
         DataTable table, ServerClock clock, DateTime startUtc, DateTime endUtc)
@@ -81,6 +87,24 @@ public async Task TheSameEventStoredSeveralTimes_ComesBackOnce_AndADifferentEven
         Assert.Equal(new[] { "same", "other" }, rows.Select(r => r.DatabaseName).ToArray());
     }
 
+    [Fact]
+    public async Task TwoRealEventsWithIdenticalColumns_AreTwoRows_AndStoredCopiesOfOneStayOne()
+    {
+        /* #5644 H4b: two identical DBCC or audit events from one spid in one 3.33 ms tick match on every column
+           and differ only in event_sequence, so both show. A stored COPY repeats the sequence too and collapses.
+           Rows stored before the column existed (NULL sequence) keep the column-only key. */
+        var table = NewTable();
+        AddEvent(table, Naive(2026, 10, 12, 10, 49), "real", eventSequence: 100);
+        AddEvent(table, Naive(2026, 10, 12, 10, 49), "real", eventSequence: 101);
+        AddEvent(table, Naive(2026, 10, 12, 10, 49), "real", eventSequence: 101);   /* a copy of the second */
+        AddEvent(table, Naive(2026, 10, 12, 10, 49), "old");
+        AddEvent(table, Naive(2026, 10, 12, 10, 49), "old");                        /* NULL sequence: a copy */
+
+        var rows = await ReadAsync(table, Eastern(), Naive(2026, 1, 1, 0), Naive(2026, 12, 31, 0));
+
+        Assert.Equal(new[] { "real", "real", "old" }, rows.Select(r => r.DatabaseName).ToArray());
+    }
+
     [Fact]
     public async Task TwoAutoGrowsForDifferentFilesInOneTick_AreTwoEvents_AsInTheViewerAndLite()
     {
@@ -92,6 +116,8 @@ public async Task TwoAutoGrowsForDifferentFilesInOneTick_AreTwoEvents_AsInTheVie
             table.Columns.Add("c" + i, i == 0 ? typeof(DateTime) : i == 2 ? typeof(int) : typeof(string));
         }
 
+        table.Columns.Add("c15", typeof(long));   /* event_sequence */
+
         foreach (var (file, cls) in new[] { ("a.mdf", 92), ("b.mdf", 92), ("b.mdf", 93), ("a.mdf", 92) })
         {
             AddRow(table, (0, Naive(2026, 3, 9, 8)), (1, "Data File Auto Grow"), (2, cls), (4, "grow"), (9, file));
diff --git a/Darling/Darling.Tests/StoredServerClockRowTextTests.cs b/Darling/Darling.Tests/StoredServerClockRowTextTests.cs
index 5df4f3c0cb..1e70719b81 100644
--- a/Darling/Darling.Tests/StoredServerClockRowTextTests.cs
+++ b/Darling/Darling.Tests/StoredServerClockRowTextTests.cs
@@ -198,6 +198,7 @@ private static DataTable NewDefaultTraceTable()
         table.Columns.Add("text_data", typeof(string));
         table.Columns.Add("filename", typeof(string));
         table.Columns.Add("event_class", typeof(int));
+        table.Columns.Add("event_sequence", typeof(long));
         return table;
     }
 
@@ -212,11 +213,11 @@ public async Task ADefaultTraceEventReadFromTheStoredWallClock_PrintsNoOffset_In
         foreach (var label in new[] { "first-pass", "second-pass" })
         {
             table.Rows.Add(Naive(2026, 11, 1, 1, 30), "Data File Auto Grow", label, DBNull.Value, DBNull.Value, DBNull.Value,
-                DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value);
+                DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value);
         }
 
         table.Rows.Add(Naive(2026, 11, 1, 2, 30), "Data File Auto Grow", "after", DBNull.Value, DBNull.Value, DBNull.Value,
-            DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value);
+            DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value);
 
         List rows;
         using (var reader = table.CreateDataReader())
diff --git a/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs b/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs
index 04d4b1789c..8925023a6b 100644
--- a/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs
+++ b/Darling/Darling.Tests/ViewerServerClockEventReadsTests.cs
@@ -56,13 +56,15 @@ private static DataTable NewTable()
         table.Columns.Add("text_data", typeof(string));
         table.Columns.Add("filename", typeof(string));
         table.Columns.Add("event_class", typeof(int));
+        table.Columns.Add("event_sequence", typeof(long));
         return table;
     }
 
     /// One stored event.  rides in database_name so a test can say which
     /// event came back; the default name is a Default Trace category that is significant as collected.
     private static void AddEvent(DataTable table, DateTime? local, string label,
-        string eventName = "Data File Auto Grow", int? severity = null, string? filename = null, int? eventClass = null)
+        string eventName = "Data File Auto Grow", int? severity = null, string? filename = null, int? eventClass = null,
+        long? eventSequence = null)
     {
         table.Rows.Add(
             local.HasValue ? (object)local.Value : DBNull.Value,
@@ -71,7 +73,8 @@ private static void AddEvent(DataTable table, DateTime? local, string label,
             severity.HasValue ? (object)severity.Value : DBNull.Value,
             DBNull.Value, DBNull.Value,
             filename is null ? DBNull.Value : filename,
-            eventClass.HasValue ? (object)eventClass.Value : DBNull.Value);
+            eventClass.HasValue ? (object)eventClass.Value : DBNull.Value,
+            eventSequence.HasValue ? (object)eventSequence.Value : DBNull.Value);
     }
 
     private static async Task> ReadAsync(DataTable table, ServerClock clock, DateTime startUtc, DateTime endUtc)
@@ -141,6 +144,28 @@ public async Task TwoAutoGrowsForDifferentFilesInOneTick_AreTwoEvents_AndTrueCop
         Assert.Equal(3, rows.Count);
     }
 
+    /// 
+    /// #5644 H4b: two real events with identical columns (the same DBCC or audit event from one spid in one 3.33 ms
+    /// tick) differ only in event_sequence, so both show; a stored copy repeats the sequence and collapses; rows
+    /// stored before the column existed (NULL sequence) keep the column-only key.
+    /// 
+    [Fact]
+    public async Task TwoRealEventsWithIdenticalColumns_AreTwoRows_AndStoredCopiesOfOneStayOne()
+    {
+        var table = NewTable();
+        AddEvent(table, Naive(2026, 3, 9, 8), "real", eventSequence: 100);
+        AddEvent(table, Naive(2026, 3, 9, 8), "real", eventSequence: 101);
+        AddEvent(table, Naive(2026, 3, 9, 8), "real", eventSequence: 101);   /* a copy of the second */
+        AddEvent(table, Naive(2026, 3, 9, 8), "old");
+        AddEvent(table, Naive(2026, 3, 9, 8), "old");                        /* NULL sequence: a copy */
+
+        var rows = await ReadAsync(table, Eastern(), Naive(2026, 1, 1, 0), Naive(2026, 12, 31, 0));
+
+        Assert.Equal(3, rows.Count);
+        Assert.Equal(2, rows.Count(r => r.DatabaseName == "real"));
+        Assert.Single(rows, r => r.DatabaseName == "old");
+    }
+
     [Fact]
     public async Task ADefaultTraceEventInASkippedOrRepeatedLocalHour_DoesNotThrow()
     {
diff --git a/Darling/Darling.Tests/ViewerSystemEventsDataStartTests.cs b/Darling/Darling.Tests/ViewerSystemEventsDataStartTests.cs
index 944c21d8d7..0547253d2f 100644
--- a/Darling/Darling.Tests/ViewerSystemEventsDataStartTests.cs
+++ b/Darling/Darling.Tests/ViewerSystemEventsDataStartTests.cs
@@ -159,9 +159,10 @@ private static DateTime Naive(int year, int month, int day, int hour, int minute
         table.Columns.Add("text_data", typeof(string));
         table.Columns.Add("filename", typeof(string));
         table.Columns.Add("event_class", typeof(int));
+        table.Columns.Add("event_sequence", typeof(long));
         foreach (var local in storedLocal)
         {
-            table.Rows.Add(local, "Data File Auto Grow", "db", DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value);
+            table.Rows.Add(local, "Data File Auto Grow", "db", DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, 55, 1_500_000L, 256L, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value);
         }
 
         using var reader = table.CreateDataReader();
diff --git a/Darling/Darling.Tests/WebChartGapBehaviourTests.cs b/Darling/Darling.Tests/WebChartGapBehaviourTests.cs
index 7219783d60..b94ad21cf3 100644
--- a/Darling/Darling.Tests/WebChartGapBehaviourTests.cs
+++ b/Darling/Darling.Tests/WebChartGapBehaviourTests.cs
@@ -161,6 +161,28 @@ public void TheGapThreshold_IsPerSeries_ACoarseSeriesOverFineBucketsIsOneLineUnl
         Assert.Equal(0, N(r, "coarsePrimary", "dots"));
     }
 
+    [Fact]
+    public void ASeriesOfEvents_HasNoCadenceToBeSilentAgainst_SoItDrawsOneLineAndNoLoneDots()
+    {
+        // #5644 H4b: clusters of events with minutes to hours between them have a tiny median interval; the 5 minute floor
+        // used to cut that into mostly lone dots.
+        var r = Run();
+        Assert.Equal(1, N(r, "eventsLine", "lines"));
+        Assert.Equal(0, N(r, "eventsLine", "dots"));
+        Assert.Equal(1, N(r, "eventsArea", "lines"));
+        Assert.Equal(1, N(r, "eventsArea", "areas"));
+        Assert.Equal(0, N(r, "eventsArea", "dots"));
+        Assert.Equal("Infinity", r.GetProperty("eventsThreshold").GetString());
+    }
+
+    [Fact]
+    public void ASampledSeriesMissingSomeSamples_StillBreaksAtALongSilence()
+    {
+        var r = Run();
+        Assert.Equal(2, N(r, "jitterGap", "lines"));
+        Assert.Equal(0, N(r, "jitterGap", "dots"));
+    }
+
     [Fact]
     public void TheThreshold_HasNothingToCompareUnderThreeBuckets_AndNeverFallsUnderTheFloor()
     {
diff --git a/Darling/Darling.Tests/web-chart-gap-harness.mjs b/Darling/Darling.Tests/web-chart-gap-harness.mjs
index 624d459621..258dcd2e40 100644
--- a/Darling/Darling.Tests/web-chart-gap-harness.mjs
+++ b/Darling/Darling.Tests/web-chart-gap-harness.mjs
@@ -161,6 +161,16 @@ out.coarsePrimary = marks(charts.renderLineChart({
   series: [{ key: "a", label: "A", color: "#f00" }, { key: "c", label: "C", color: "#0f0" }],
 }));
 
+/* #5644 H4b: a series of EVENTS has no cadence. Clusters of rows with minutes to hours between them have a tiny median
+   interval, so the 5 minute floor used to cut them into lone dots. They draw one line; a sampled series that went silent
+   (above) still breaks. */
+const events = [0, 1, 2, 9, 10, 31, 70, 150, 151, 152, 400, 405, 410, 600, 640, 660];
+out.eventsLine = marks(draw(events, { series: [{ key: "a", label: "A", color: "#f00" }] }));
+out.eventsArea = marks(draw(events, { mode: "area", series: [{ key: "a", label: "A", color: "#f00" }] }));
+out.eventsThreshold = String(charts.gapThresholdMs(events.map((m) => m * MIN)));
+/* a 1 minute series that is missing a quarter of its samples at random is still sampled: it breaks at the 7 hour silence */
+out.jitterGap = marks(draw(withGap.filter((m, i) => i % 5 !== 3), { series: [{ key: "a", label: "A", color: "#f00" }] }));
+
 out.threshold = {
   empty: String(charts.gapThresholdMs([])),
   one: String(charts.gapThresholdMs([0])),
diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDefaultTraceReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDefaultTraceReader.cs
index c549a56fc4..5548eab3fb 100644
--- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDefaultTraceReader.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDefaultTraceReader.cs
@@ -127,7 +127,8 @@ ORDER BY sp.collection_time DESC
             dte.error_number,
             dte.severity,
             dte.duration_us,
-            dte.integer_data
+            dte.integer_data,
+            dte.event_sequence
         FROM default_trace_events AS dte, svr
         WHERE dte.server_id = $1
         AND   dte.event_time - make_interval(mins => svr.offset_minutes) >= $2 - interval '1 hour'
@@ -172,6 +173,7 @@ internal static async Task> ReadEventRowsAsync(
         DbDataReader reader, ServerClock clock, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken)
     {
         var rows = new List();
+        var seen = new HashSet<(DefaultTraceEventRow Row, long? EventSequence)>();
 
         while (await reader.ReadAsync(cancellationToken))
         {
@@ -181,7 +183,7 @@ internal static async Task> ReadEventRowsAsync(
                 continue;
             }
 
-            rows.Add(new DefaultTraceEventRow(
+            var row = new DefaultTraceEventRow(
                 eventTimeUtc,
                 reader.IsDBNull(1) ? null : reader.GetString(1),
                 reader.IsDBNull(2) ? null : reader.GetInt32(2),
@@ -196,14 +198,24 @@ internal static async Task> ReadEventRowsAsync(
                 reader.IsDBNull(11) ? null : reader.GetInt32(11),
                 reader.IsDBNull(12) ? null : reader.GetInt32(12),
                 reader.IsDBNull(13) ? null : reader.GetInt64(13),
-                reader.IsDBNull(14) ? null : reader.GetInt64(14)));
+                reader.IsDBNull(14) ? null : reader.GetInt64(14));
+
+            /* #5644 W6: a stored COPY of one event repeats the row and its event_sequence, so the first of each in
+               reader order is kept. Two real events with identical columns (the same DBCC or audit event from one spid
+               in one 3.33 ms tick) carry different event_sequence values, so both stay. A row stored before the
+               column existed has a NULL sequence and keeps the column-only key. The viewer and Lite key the same
+               way. */
+            if (seen.Add((row, reader.IsDBNull(15) ? (long?)null : reader.GetInt64(15))))
+            {
+                rows.Add(row);
+            }
         }
 
         /* OrderByDescending is a stable sort (List.Sort is not), so ties keep the reader's order. */
         /* #5644 W6: the store holds rows that are the same event twice (an older build kept both the Begin and the
            Commit of every DDL, and re-stored an event whose datetime StartTime did not round-trip through the
-           watermark parameter). The record's value equality makes Distinct drop the copies, keeping the first of each in
-           reader order, so they stop filling the page limit. */
-        return rows.Distinct().OrderByDescending(static r => r.EventTimeUtc).ToList();
+           watermark parameter). The loop above dropped the copies, keeping the first of each in reader order, so they stop
+           filling the page limit. */
+        return rows.OrderByDescending(static r => r.EventTimeUtc).ToList();
     }
 }
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js
index 27f57a88e4..cf3590bead 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/charts.js
@@ -189,11 +189,16 @@ function watchPlotWidth(host, draw, key, drawnW) {
    point to point would run a straight diagonal across it and read as data. A gap is a jump between two neighbouring
    timestamps of more than GAP_FACTOR times the series' own median interval, never less than GAP_FLOOR_MS (a collector can
    miss a cycle without that being worth a break). Fewer than three points have no normal interval to compare against, so
-   they never break. Every time-series line, area and stacked chart goes through this, so no caller inserts null points. */
+   they never break. Every time-series line, area and stacked chart goes through this, so no caller inserts null points.
+   A break is for a SAMPLED series that went silent, so it needs a cadence to be silent against. A series of events
+   (clusters of rows with hours between them) has none: its spacing runs from seconds to hours, the median is tiny, and
+   the floor would cut it into lone dots. A series has a cadence when its interquartile spacing is tight (the 75th
+   percentile of the intervals is within GAP_FACTOR times the 25th); one with no cadence never breaks (#5644 H4b). */
 const GAP_FACTOR = 3;
 const GAP_FLOOR_MS = 5 * 60 * 1000;
 
-/* The break threshold, in ms, for ascending timestamps (epoch ms); Infinity when there is nothing to compare against. */
+/* The break threshold, in ms, for ascending timestamps (epoch ms); Infinity when there is nothing to compare against or
+   the series has no regular cadence (events). */
 export function gapThresholdMs(times) {
   const deltas = [];
   for (let i = 1; i < times.length; i++) {
@@ -203,6 +208,8 @@ export function gapThresholdMs(times) {
   if (deltas.length < 2) return Infinity;
   deltas.sort((a, b) => a - b);
   const mid = deltas.length >> 1;
+  const quantile = (p) => deltas[Math.round((deltas.length - 1) * p)];
+  if (quantile(0.75) > GAP_FACTOR * quantile(0.25)) return Infinity;
   const median = deltas.length % 2 ? deltas[mid] : (deltas[mid - 1] + deltas[mid]) / 2;
   return Math.max(GAP_FACTOR * median, GAP_FLOOR_MS);
 }
diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs
index 530a1e35f1..51505e1124 100644
--- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs
+++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs
@@ -649,7 +649,8 @@ ORDER BY sp.collection_time DESC
             dte.error_number,
             dte.text_data,
             dte.filename,
-            dte.event_class
+            dte.event_class,
+            dte.event_sequence
         FROM default_trace_events AS dte, svr
         WHERE dte.server_id = $1
         AND   dte.event_time - make_interval(mins => svr.offset_minutes) >= $2 - interval '1 hour'
@@ -697,7 +698,7 @@ internal static async Task> ReadDefaultTraceEventsAsy
         DbDataReader reader, ServerClock clock, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken)
     {
         var rows = new List();
-        var seen = new HashSet<(DateTime?, string?, string?, string?, string?, string?, string?, int?, long?, long?, int?, int?, string?, string?, int?)>();
+        var seen = new HashSet<(DateTime?, string?, string?, string?, string?, string?, string?, int?, long?, long?, int?, int?, string?, string?, int?, long?)>();
 
         while (await reader.ReadAsync(cancellationToken))
         {
@@ -723,14 +724,17 @@ internal static async Task> ReadDefaultTraceEventsAsy
             var textData = reader.IsDBNull(12) ? null : reader.GetString(12);
             var filename = reader.IsDBNull(13) ? null : reader.GetString(13);
             var eventClass = reader.IsDBNull(14) ? (int?)null : reader.GetInt32(14);
+            var eventSequence = reader.IsDBNull(15) ? (long?)null : reader.GetInt64(15);
 
             /* #5644 W6: the store holds rows that are the same event twice (an older build kept both the Begin and
                the Commit of every DDL, and re-stored an event whose datetime StartTime did not round-trip through
-               the watermark parameter). They match on every column read here and only the event_sequence the
-               collector stores tells them apart, so the first of each in reader order is kept, as the MCP reader
-               and Lite's do, and the copies stop filling the grid and the page limit. */
+               the watermark parameter). A copy repeats every column read here AND the event_sequence the collector
+               stores, so the first of each in reader order is kept, as the MCP reader and Lite's do, and the copies
+               stop filling the grid and the page limit. Two real events with identical columns (the same DBCC or
+               audit event from one spid in one 3.33 ms tick) carry different sequences, so both stay; a row stored
+               before the column existed has a NULL sequence and keeps the column-only key. */
             if (!seen.Add((eventTimeUtc, eventName, databaseName, objectName, loginName, hostName, applicationName,
-                    spid, durationUs, integerData, severity, errorNumber, textData, filename, eventClass)))
+                    spid, durationUs, integerData, severity, errorNumber, textData, filename, eventClass, eventSequence)))
                 continue;
 
             rows.Add(new DefaultTraceEventRow(
diff --git a/Lite.Tests/DefaultTraceServerClockTests.cs b/Lite.Tests/DefaultTraceServerClockTests.cs
index cbc49df646..bf6bc84183 100644
--- a/Lite.Tests/DefaultTraceServerClockTests.cs
+++ b/Lite.Tests/DefaultTraceServerClockTests.cs
@@ -78,7 +78,7 @@ INSERT INTO default_trace_events
         await cmd.ExecuteNonQueryAsync();
     }
 
-    private async Task SeedAutoGrowAsync(DateTime eventTimeServerLocal, string filename, int eventClass)
+    private async Task SeedAutoGrowAsync(DateTime eventTimeServerLocal, string filename, int eventClass, long? eventSequence = null)
     {
         using var readLock = _duckDb.AcquireReadLock();
         var conn = await SeedConnectionAsync();
@@ -86,14 +86,15 @@ private async Task SeedAutoGrowAsync(DateTime eventTimeServerLocal, string filen
         cmd.CommandText = @"
 INSERT INTO default_trace_events
     (default_trace_event_id, collection_time, server_id, server_name, event_time, event_name, event_class,
-     database_name, filename, duration_us, integer_data, severity, error_number, text_data)
-VALUES ($1, $2, $3, 'TestSrv', $4, 'Data File Auto Grow', $5, 'grow', $6, 1500000, 256, NULL, NULL, NULL)";
+     database_name, filename, duration_us, integer_data, severity, error_number, text_data, event_sequence)
+VALUES ($1, $2, $3, 'TestSrv', $4, 'Data File Auto Grow', $5, 'grow', $6, 1500000, 256, NULL, NULL, NULL, $7)";
         cmd.Parameters.Add(new DuckDBParameter { Value = _nextId++ });
         cmd.Parameters.Add(new DuckDBParameter { Value = DateTime.UtcNow });
         cmd.Parameters.Add(new DuckDBParameter { Value = ServerId });
         cmd.Parameters.Add(new DuckDBParameter { Value = eventTimeServerLocal });
         cmd.Parameters.Add(new DuckDBParameter { Value = eventClass });
         cmd.Parameters.Add(new DuckDBParameter { Value = filename });
+        cmd.Parameters.Add(new DuckDBParameter { Value = (object?)eventSequence ?? DBNull.Value });
         await cmd.ExecuteNonQueryAsync();
     }
 
@@ -114,6 +115,26 @@ public async Task DefaultTrace_AutoGrowsForDifferentFilesInOneTick_AreTwoEvents_
         Assert.Equal(3, rows.Count(r => r.EventName == "Data File Auto Grow"));
     }
 
+    [Fact]
+    public async Task DefaultTrace_TwoRealEventsWithIdenticalColumns_AreTwoRows_AndStoredCopiesOfOneStayOne()
+    {
+        /* #5644 H4b: two identical DBCC or audit events from one spid in one tick differ only in event_sequence, so
+           both show; a stored copy repeats the sequence and collapses; NULL-sequence rows (stored before the column
+           existed) keep the column-only key. */
+        var service = new LocalDataService(_duckDb);
+        var tick = At(2026, 3, 9, 8, 0);
+        await SeedAutoGrowAsync(tick, "a.mdf", 92, eventSequence: 100);
+        await SeedAutoGrowAsync(tick, "a.mdf", 92, eventSequence: 101);
+        await SeedAutoGrowAsync(tick, "a.mdf", 92, eventSequence: 101);   /* a copy of the second */
+        await SeedAutoGrowAsync(tick, "old.mdf", 92);
+        await SeedAutoGrowAsync(tick, "old.mdf", 92);                    /* NULL sequence: a copy */
+
+        var rows = await service.GetDefaultTraceEventsAsync(
+            ServerId, fromDate: At(2026, 3, 9, 0, 0), toDate: At(2026, 3, 10, 0, 0), serverClock: Eastern());
+
+        Assert.Equal(3, rows.Count(r => r.EventName == "Data File Auto Grow"));
+    }
+
     // ── System Events (Default Trace): server-local event_time converted per row ──
 
     [Fact]
diff --git a/Lite/Services/LocalDataService.SystemEvents.cs b/Lite/Services/LocalDataService.SystemEvents.cs
index b10e2fe94d..c049778b34 100644
--- a/Lite/Services/LocalDataService.SystemEvents.cs
+++ b/Lite/Services/LocalDataService.SystemEvents.cs
@@ -839,8 +839,10 @@ the same clock and held to the exact window below (#4766). Bind db filter params
         var dbClause = BuildDbInClause(databaseNames, "database_name", 4, out var dbValues);
 
         /* #5644 W6: DISTINCT drops the stored copies of one event (the Begin and Commit rows of a DDL that an older
-           build kept, and an event re-stored by the datetime watermark round trip): they match on every column
-           selected here, and only the event_sequence the collector stores tells them apart. filename and event_class are
+           build kept, and an event re-stored by the datetime watermark round trip): a copy repeats every column
+           selected here INCLUDING event_sequence. Two real events with identical columns (the same DBCC or audit event
+           from one spid in one 3.33 ms tick) carry different sequences and both stay; a row stored before the column
+           existed has a NULL sequence, which DISTINCT keeps as the column-only key. filename and event_class are
            selected for the same reason as the MCP reader and the viewer key on them (the grid does not show them): two
            Data File Auto Grow rows for different files in one tick are two events. */
         command.CommandText = @"
@@ -859,7 +861,8 @@ SELECT DISTINCT
     error_number,
     text_data,
     filename,
-    event_class
+    event_class,
+    event_sequence
 FROM v_default_trace_events
 WHERE server_id = $1
 AND   event_time >= $2

From 2cb8a9370c0c3078f7f25ef91ec73c7bb49f1762 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Mon, 12 Oct 2026 22:56:51 -0400
Subject: [PATCH 32/42] Web UI: finish the live-check wording fixes (#5644)

Deadlock empty states take the page sentence only on a true empty; the blocking trend chart words itself by its collector; alert history detail localizes its time lines; plural rule handles y nouns.
---
 .../WebBlockingSourceBehaviourTests.cs        | 46 +++++++++++++++++--
 .../WebClampCellBehaviourTests.cs             | 17 +++++++
 Darling/Darling.Tests/WebPlainTextTests.cs    |  3 ++
 .../web-blocking-source-harness.mjs           | 12 ++++-
 .../Darling.Tests/web-clamp-cell-harness.mjs  | 21 +++++++++
 .../wwwroot/js/pages/server-tabs.js           | 10 ++++
 .../wwwroot/js/panels.js                      | 34 +++++++++++++-
 .../wwwroot/js/plain-text.js                  |  4 +-
 .../wwwroot/js/read-fields.js                 |  2 +-
 .../wwwroot/js/view-templates.js              |  1 +
 10 files changed, 138 insertions(+), 12 deletions(-)

diff --git a/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs b/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs
index acfb462840..80001c5b53 100644
--- a/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs
+++ b/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs
@@ -69,14 +69,14 @@ private static string[][] Lines(JsonElement r, string title) =>
             .ToArray();
 
     [Theory]
-    [InlineData("xe", "Source: blocked-process-report")]
-    [InlineData("dmv", "Source: DMV snapshot")]
-    public void BlockingEventsAndSeverity_NameTheCollectorThatAnswered(string scenario, string expected)
+    [InlineData("xe", "Source: blocked-process-report", "Blocked process report trend")]
+    [InlineData("dmv", "Source: DMV snapshot", "Blocking snapshot trend")]
+    public void BlockingEventsAndSeverity_NameTheCollectorThatAnswered(string scenario, string expected, string trendTitle)
     {
         var r = Run(scenario);
 
         /* Blocking Events is on the Overview tab and the Blocking tab; Blocking Severity only on the Blocking tab. */
-        var events = Lines(r, "Blocked process report trend");
+        var events = Lines(r, trendTitle);
         Assert.Equal(2, events.Length);
         Assert.All(events, lines => Assert.Equal(new[] { expected }, lines));
         var severity = Lines(r, "Blocking Severity");
@@ -142,6 +142,42 @@ public void TheBlockingTrendChart_SaysItCountsBlockedProcessReports()
         Assert.Contains("const BLOCKING_REPORT_SERIES = [{ key: \"count\", label: \"Reports\" }];", tabs);
         Assert.DoesNotContain("No blocking events in this window — an empty trend", tabs);
         Assert.Contains("label: \"Reports\"", templates);
+        /* The axis caption says it too: both tab charts and the starter dashboard panel carry unit "reports". */
+        Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(tabs, "unit: \"reports\",").Count);
+        Assert.Contains("unit: \"reports\",", templates);
+    }
+
+    /// #5644 W16 review: only a true negative (status "empty") takes the page's one sentence. A not_collected or unavailable
+    /// deadlock read keeps its own message; "No deadlocks in this window." over an Azure SQL Database target would be a false all-clear.
+    [Fact]
+    public void ANotCollectedOrUnavailableDeadlockRead_KeepsItsOwnMessage_NotTheAllClear()
+    {
+        var r = Run("deadlockNotCollected");
+        const string One = "No deadlocks in this window.";
+        var trend = Assert.Single(Empties(r, "blocking", "Deadlock trend"));
+        var table = Assert.Single(Empties(r, "blocking", "Deadlocks"));
+        var severity = Assert.Single(Empties(r, "blocking", "Deadlock Severity"));
+        Assert.All(new[] { trend, table, severity }, e => { Assert.Single(e); Assert.NotEqual(One, e[0]); Assert.False(string.IsNullOrWhiteSpace(e[0])); });
+        Assert.Equal("Deadlock data is unavailable right now: the collector has not run.", table[0]);
+    }
+
+    /// #5644 W16 review: when get_blocking_trend falls back to the DMV snapshot its points are snapshot rows, not blocked process
+    /// reports, so the title, legend and axis caption follow the answer's collector tag.
+    [Theory]
+    [InlineData("xe", "Blocked process report trend", "Reports", "reports")]
+    [InlineData("dmv", "Blocking snapshot trend", "Snapshot rows", "snapshot rows")]
+    public void TheBlockingTrendChart_WordsItselfByTheCollectorThatAnswered(string scenario, string title, string label, string unit)
+    {
+        var r = Run(scenario);
+        var charts = r.GetProperty("charts").EnumerateArray()
+            .Where(c => c.GetProperty("labels").EnumerateArray().Any(l => l.GetString() == label))
+            .ToList();
+        Assert.Equal(2, charts.Count);
+        Assert.All(charts, c =>
+        {
+            Assert.Equal(title, c.GetProperty("title").GetString());
+            Assert.Equal(unit, c.GetProperty("unit").GetString());
+        });
     }
 
     [Fact]
@@ -153,6 +189,6 @@ public void OnlyTheBlockingPanelsGetOne()
             .Where(p => p.GetProperty("strips").GetArrayLength() > 0)
             .Select(p => p.GetProperty("title").GetString())
             .ToArray();
-        Assert.Equal(new[] { "Blocked process report trend", "Blocked process report trend", "Blocking Severity" }, titles);
+        Assert.Equal(new[] { "Blocking snapshot trend", "Blocking snapshot trend", "Blocking Severity" }, titles);
     }
 }
diff --git a/Darling/Darling.Tests/WebClampCellBehaviourTests.cs b/Darling/Darling.Tests/WebClampCellBehaviourTests.cs
index 6e604b9cdb..ab6a20752a 100644
--- a/Darling/Darling.Tests/WebClampCellBehaviourTests.cs
+++ b/Darling/Darling.Tests/WebClampCellBehaviourTests.cs
@@ -150,6 +150,23 @@ public void TheCollectionHealthMessageColumns_AreOptedIn_AndTheCustomViewAndDefa
         }
     }
 
+    [Fact]
+    public void TheAlertHistoryDetailColumn_LocalizesItsTimeLines_AndNeverTheSqlLines()
+    {
+        // #5644 W20 (H1): get_alert_history's detail_text opts in with localizeAlertDetail; localizeText is never used on a column of SQL.
+        var d = Run().GetProperty("alertDetail");
+        var raw = d.GetProperty("raw").GetString()!;
+        Assert.Equal(raw, d.GetProperty("notOpted").GetString());
+        var opted = d.GetProperty("opted").GetString()!;
+        Assert.Contains(d.GetProperty("sinceLocal").GetString()!, opted);
+        Assert.Contains(d.GetProperty("sampleLocal").GetString()!, opted);
+        Assert.Contains(d.GetProperty("sqlLine").GetString()!, opted);
+        Assert.DoesNotContain("23:13:00Z", opted);
+
+        var fields = Wwwroot("js", "read-fields.js");
+        Assert.Contains("{ key: \"detail_text\", label: \"Detail\", wrap: true, localizeAlertDetail: true },", fields, StringComparison.Ordinal);
+    }
+
     [Fact]
     public void EnterOpensTheCell_AndOtherKeysDoNot()
     {
diff --git a/Darling/Darling.Tests/WebPlainTextTests.cs b/Darling/Darling.Tests/WebPlainTextTests.cs
index 16750687f2..3592ee4803 100644
--- a/Darling/Darling.Tests/WebPlainTextTests.cs
+++ b/Darling/Darling.Tests/WebPlainTextTests.cs
@@ -110,6 +110,7 @@ public void TrendNotes_NameTheTimeRange_TheColumnLabels_AndAgreeInNumber()
             "The session counts are levels, averaged, never summed; top_application_name, top_application_connections, top_host_name and top_host_connections are the newest collection's own values in the bucket.",
             "No query-stats group on SRV in the last 24 hour(s) has a cached plan.",
             "No blocking was recorded in the last 1 hour(s). 288 collector run(s) DID execute; 1 collector run(s) failed.",
+            "3 query(s) failed, 1 query(s) retried, 2 policy(s) skipped.",
         ], out _);
         Assert.DoesNotContain("narrow the bucket", o[0]);
         Assert.Contains("narrow the time range to see it", o[0]);
@@ -120,6 +121,8 @@ public void TrendNotes_NameTheTimeRange_TheColumnLabels_AndAgreeInNumber()
         Assert.Contains("the top application, its connection count, the top host and its connection count are", o[3]);
         Assert.Equal("No query-stats group on SRV in the last 24 hours has a cached plan.", o[4]);
         Assert.Equal("No blocking was recorded in the last 1 hour. 288 collector runs DID execute; 1 collector run failed.", o[5]);
+        /* a noun ending in a consonant plus y takes "ies", not a bare "s" */
+        Assert.Equal("3 queries failed, 1 query retried, 2 policies skipped.", o[6]);
     }
 
     /* #5644 W8: the clauses that point an MCP caller at a parameter the page has no control for. */
diff --git a/Darling/Darling.Tests/web-blocking-source-harness.mjs b/Darling/Darling.Tests/web-blocking-source-harness.mjs
index 44a35f4923..e8497c2409 100644
--- a/Darling/Darling.Tests/web-blocking-source-harness.mjs
+++ b/Darling/Darling.Tests/web-blocking-source-harness.mjs
@@ -28,6 +28,7 @@ class FakeNode {
     return this.children[0] || null;
   }
   appendChild(child) {
+    child.parentNode = this;
     this.children.push(child);
     return child;
   }
@@ -81,7 +82,7 @@ try {
       "export const CATEGORICAL_COLORS = SERIES_COLORS;\n" +
       "export function normalizeColor(color) { return color; }\n" +
       "export function renderLineChart() { return document.createElement('div'); }\n" +
-      "export function zoomableLineChart() { return document.createElement('div'); }\n" +
+      "export function zoomableLineChart(spec) { (globalThis.__charts = globalThis.__charts || []).push({ title: spec.title, labels: (spec.series || []).map((x) => x.label), unit: spec.unit }); return document.createElement('div'); }\n" +
       "export function chartZoomScope() { return ''; }\n"
   );
   const load = (rel) => import(pathToFileURL(path.join(scratch, rel)).href);
@@ -118,6 +119,13 @@ const BODIES = {
   },
   /* #5644 W16: every deadlock read answers the {status, message} envelope with ITS OWN wording (the trend's carries the collector-run
      count, the table's says "specified time range"); the stats read, which draws Deadlock Severity, says a third thing. */
+  /* #5644 W16: the engine has no deadlock data source (not_collected), or it cannot be read right now (unavailable): neither is a quiet window. */
+  deadlockNotCollected: {
+    trend: { source: null, trend: [] },
+    stats: { status: "not_collected", message: "This is a permanent engine capability gap: Azure SQL Database has no system_health session." },
+    deadlockTrend: { status: "not_collected", message: "This is a permanent engine capability gap: Azure SQL Database has no system_health session." },
+    deadlocks: { status: "unavailable", message: "Deadlock data is unavailable right now: the collector has not run." },
+  },
   deadlockEmpty: {
     trend: { source: null, trend: [] },
     stats: { status: "empty", message: "No blocking or deadlock data in the specified time range." },
@@ -175,4 +183,4 @@ for (const tab of tabs.SERVER_TABS) {
   await settle();
   for (const p of panelsOf(holder)) out.push({ tab: tab.id, ...p });
 }
-console.log(JSON.stringify({ panels: out, rejections }));
+console.log(JSON.stringify({ panels: out, charts: globalThis.__charts || [], rejections }));
diff --git a/Darling/Darling.Tests/web-clamp-cell-harness.mjs b/Darling/Darling.Tests/web-clamp-cell-harness.mjs
index ba3d4ef985..0c427a7ee4 100644
--- a/Darling/Darling.Tests/web-clamp-cell-harness.mjs
+++ b/Darling/Darling.Tests/web-clamp-cell-harness.mjs
@@ -135,4 +135,25 @@ out.shapes = l2rows.map((tr, i) => {
 });
 out.sqlZ = sqlZ;
 
+/* #5644 W20 (H1): get_alert_history's detail_text opts in with localizeAlertDetail: the labelled time lines shift, the sample-age line
+   shifts from the row's newest_sample, and the SQL lines (and a not-opted-in copy of the same text) print as written. */
+const detail = "Incident Since: 2026-10-12 23:13:00Z\nNewest blocked process sample: 23:12:45 UTC, 5m 20s old\nBlocked Query: SELECT * FROM t WHERE d = '2026-01-01 00:00:00Z'";
+const l3 = VIZ.table({ rows: [{ detail_text: detail, newest_sample: { instant_utc: "2026-10-12T23:12:45Z" } }] }, {
+  rowsKey: "rows", title: "Alert detail", tools: false,
+  columns: [
+    { key: "detail_text", label: "Detail", wrap: true },
+    { key: "detail_text", label: "Detail (opted in)", wrap: true, localizeAlertDetail: true },
+  ],
+});
+const l3tr = l3.findAll((n) => n.tag === "tr").filter((n) => n.parent && n.parent.tag === "tbody")[0];
+const l3t = l3tr.children.map((c) => c.textContent);
+out.alertDetail = {
+  notOpted: l3t[0],
+  raw: detail,
+  opted: l3t[1],
+  sinceLocal: "Incident Since: " + new Date("2026-10-12T23:13:00Z").toLocaleString(),
+  sampleLocal: "Newest blocked process sample: " + new Date("2026-10-12T23:12:45Z").toLocaleString() + ", 5m 20s old",
+  sqlLine: "Blocked Query: SELECT * FROM t WHERE d = '2026-01-01 00:00:00Z'",
+};
+
 console.log(JSON.stringify(out));
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js
index 8f79a37c1e..9d99028b62 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js
@@ -1058,6 +1058,8 @@ function line(title, read, params, rowsKey, xKey, series, opts = {}) {
     emptyWins: opts.emptyWins === true,
     /* #5244: the answer's field naming the collector that answered (get_blocking_trend's `source`); absent on every other line. */
     sourceKey: opts.sourceKey,
+    /* #5644 W16: wording that follows the collector that answered (see BLOCKING_BY_SOURCE); absent on every other line. */
+    bySource: opts.bySource,
     /* The panel is not drawn when the server answers "not collected" (it does not apply to this kind of server). */
     hideWhenNotCollected: opts.hideWhenNotCollected === true,
     span: opts.span ?? 1,
@@ -1100,8 +1102,10 @@ export const SERVER_TABS = [
       line("Blocked process report trend", "get_blocking_trend", { server, hours: ctx.hours }, "trend", "time", BLOCKING_REPORT_SERIES, {
         subtitle: ctx.label,
         format: "int",
+        unit: "reports",
         atTimeItem: "blocking",
         sourceKey: "source",
+        bySource: BLOCKING_BY_SOURCE,
         emptyText: "No blocked process reports in this window — an empty trend here means none were captured, not that nothing was collected.",
       }),
       line("Deadlocks", "get_deadlock_trend", { server, hours: ctx.hours }, "trend", "time", COUNT_SERIES, {
@@ -1321,8 +1325,10 @@ export const SERVER_TABS = [
       line("Blocked process report trend", "get_blocking_trend", { server, hours: ctx.hours }, "trend", "time", BLOCKING_REPORT_SERIES, {
         subtitle: ctx.label,
         format: "int",
+        unit: "reports",
         atTimeItem: "blocking",
         sourceKey: "source",
+        bySource: BLOCKING_BY_SOURCE,
         emptyText: "No blocked process reports in this window — an empty trend here means none were captured, not that nothing was collected.",
       }),
       line("Deadlock trend", "get_deadlock_trend", { server, hours: ctx.hours }, "trend", "time", COUNT_SERIES, {
@@ -3179,6 +3185,10 @@ const COUNT_SERIES = [{ key: "count", label: "Events" }];
 /* #5644 W16: get_blocking_trend counts one point per blocked-process REPORT (a single wait can produce several), so the
    chart says "reports" where the tile beside it counts one event per wait. Same read and same count as COUNT_SERIES. */
 const BLOCKING_REPORT_SERIES = [{ key: "count", label: "Reports" }];
+/* When get_blocking_trend has no blocked process reports for the window and falls back to the DMV blocking snapshot (AWS RDS and any
+   server without the blocked process report event), a point counts snapshot rows, not reports: title, legend and axis follow the
+   answer's `source` (panels.js applies it), so the chart never calls snapshot rows reports. */
+const BLOCKING_BY_SOURCE = { "DMV snapshot": { title: "Blocking snapshot trend", label: "Snapshot rows", unit: "snapshot rows" } };
 
 /* #2484: the aggregate lock-wait rate. One numeric key, per the same reasoning as the Current Waits series
    below — the LCK wait type is the grouping the read applied, not a second axis. */
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js
index 72d4c89953..519530e7d7 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js
@@ -47,6 +47,7 @@ import {
   clampText,
   markPartialSubtitle,
   localizeIsoText,
+  localizeAlertDetail,
 } from "./util.js";
 import { zoomableLineChart, chartZoomScope, SERIES_COLORS } from "./charts.js";
 import { plainText, plainTextNamed } from "./plain-text.js";
@@ -94,7 +95,30 @@ export function getPanelSignal() {
  * replaces the read's own message; every other panel keeps the server's message.
  */
 export function emptyHeadline(desc, res) {
-  return desc && desc.emptyWins && desc.emptyText ? desc.emptyText : res.message;
+  /* Only a TRUE negative (status "empty") takes the panel's sentence. A "not_collected" (an Azure SQL Database has no system_health
+     session), "unavailable" or "precondition" envelope is a different statement, and "No deadlocks" over it would be a false
+     all-clear that also drops the server's own engine-gate line; those keep the read's message. */
+  return desc && desc.emptyWins && desc.emptyText && res.status === "empty" ? desc.emptyText : res.message;
+}
+
+/**
+ * A panel whose wording depends on which collector answered (#5644 W16, the blocking trend: blocked process reports, or the DMV
+ * snapshot rows it falls back to). `desc.bySource` maps the answer's `desc.sourceKey` value to `{ title, label, unit }`; the
+ * returned descriptor carries the series label and axis unit, and the heading's title text is replaced in place. A descriptor
+ * without `bySource`, or an answer whose source has no entry, comes back as given.
+ */
+export function applySourceWording(desc, data, panel) {
+  const alt = desc.bySource && desc.sourceKey ? desc.bySource[getPath(data, desc.sourceKey)] : null;
+  if (!alt) return desc;
+  const head = panel ? Array.from(panel.children || [])[0] : null;
+  const titleNode = head ? head.firstChild : null;
+  if (titleNode && alt.title) titleNode.textContent = alt.title;
+  return {
+    ...desc,
+    title: alt.title || desc.title,
+    unit: alt.unit ?? desc.unit,
+    series: Array.isArray(desc.series) ? desc.series.map((x) => ({ ...x, label: alt.label || x.label })) : desc.series,
+  };
 }
 
 /**
@@ -209,6 +233,7 @@ async function loadPanelBody(desc, body, signal) {
     /* A narrowed read draws its chart over the hours it answered for, not the Range it was asked for (#2802).
        A copy, so the caller's descriptor keeps the window it asked for. */
     if (res.keptHours) desc = { ...desc, windowHours: res.keptHours };
+    desc = applySourceWording(desc, res.data, body.parentNode);
     const rendered = render(res.data, desc);
 
     mount(body, [
@@ -1263,7 +1288,12 @@ function cell(row, c) {
   /* #5644 W20: a column marked `localizeText` (a message or note: collection-health notes, collector errors) shows any ISO
      instant in its sentence in local time, like the grids beside it. Opt-in: query text, statement text, details and
      identifiers are data, and a literal such as '2026-01-01T00:00:00' inside them must show as written. */
-  const text = c.localizeText && !c.format ? localizeIsoText(shown) : shown;
+  /* #5644 W20: `localizeAlertDetail` is the same opt-in for an alert's stored body (get_alert_history's detail_text): only the
+     labelled time lines ("Incident Since:", "Started:", "From:", "To:", "Window:", ...) and the sample-age line shift, because the
+     same body carries query and SQL lines that must print as written. Never use `localizeText` on a column that holds SQL. */
+  const text = c.localizeAlertDetail && !c.format
+    ? localizeAlertDetail(shown, getPath(row, "newest_sample"))
+    : c.localizeText && !c.format ? localizeIsoText(shown) : shown;
   if (c.clamp) return el("td", { class: cls.join(" ") || null }, [clampText(text)]);
   return el("td", { class: cls.join(" ") || null, text });
 }
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js
index 2a478f9744..b68df21df5 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/plain-text.js
@@ -195,8 +195,8 @@ export function plainText(text) {
     /\btop_application_name, top_application_connections, top_host_name and top_host_connections are\b/g,
     "the top application, its connection count, the top host and its connection count are",
   );
-  /* "24 hour(s)", "288 collector run(s)": the count is known, so the word agrees with it ("1 hour", "24 hours"). */
-  t = t.replace(/(? n + " " + word + (n === "1" ? "" : "s"));
+  /* "24 hour(s)", "288 collector run(s)": the count is known, so the word agrees with it ("1 hour", "24 hours", "2 queries"). */
+  t = t.replace(/(? n + " " + (n === "1" ? word : /[^aeiou]y$/.test(word) ? word.slice(0, -1) + "ies" : word + "s"));
 
   /* "raise limit": the pages have no row-limit control. "...filter - raise limit to see more." and "...returned; raise limit
      (now 50) for more." lose the clause; ", or raise limit," and ", and raise limit if none ... shows one" lose theirs. */
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/read-fields.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/read-fields.js
index 9a5f090b3e..4b8b3b7bf5 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/read-fields.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/read-fields.js
@@ -582,7 +582,7 @@ export const READ_FIELDS = {
         { key: "threshold_value", label: "Threshold", format: "num2" },
         { key: "notification_type", label: "Notification" },
         { key: "muted", label: "Muted", format: "bool" },
-        { key: "detail_text", label: "Detail", wrap: true },
+        { key: "detail_text", label: "Detail", wrap: true, localizeAlertDetail: true },
       ],
     },
   },
diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js
index b1f7090754..ab190fdd98 100644
--- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js
+++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js
@@ -192,6 +192,7 @@ export const DASHBOARD_TEMPLATES = [
             emptyText: "No blocked process reports in this window — an empty trend here means none were captured, not that nothing was collected.",
             series: [{ key: "count", label: "Reports" }],
             format: "int",
+            unit: "reports",
           },
           {
             title: "Deadlocks",

From c51c72782aabad6b2a1d18018c3a8735cb6c7a55 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Mon, 12 Oct 2026 22:57:04 -0400
Subject: [PATCH 33/42] Count blocking waits, not report rows, on Lite's server
 tab badge and in the daily summary (#5644)

---
 .../DailySummaryBlockingWaitsLiveTests.cs     | 134 ++++++++++++++++++
 .../Mcp/DarlingHealthReader.cs                |   2 +-
 .../DailySummarySql.cs                        |  31 +++-
 .../ViewerDataService.DailySummary.cs         |   2 +-
 .../AzureMasterDailySummaryScopeTests.cs      |   8 +-
 Lite.Tests/BlockingWaitDuckDbTests.cs         |  36 +++++
 Lite.Tests/PerformanceCalendarDataTests.cs    |   8 +-
 Lite/Services/LocalDataService.Blocking.cs    |  17 ++-
 .../Services/LocalDataService.DailySummary.cs |  15 +-
 9 files changed, 232 insertions(+), 21 deletions(-)
 create mode 100644 Darling/Darling.Tests/DailySummaryBlockingWaitsLiveTests.cs

diff --git a/Darling/Darling.Tests/DailySummaryBlockingWaitsLiveTests.cs b/Darling/Darling.Tests/DailySummaryBlockingWaitsLiveTests.cs
new file mode 100644
index 0000000000..5eb1e7fa15
--- /dev/null
+++ b/Darling/Darling.Tests/DailySummaryBlockingWaitsLiveTests.cs
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Globalization;
+using System.Threading;
+using System.Threading.Tasks;
+using Npgsql;
+using NpgsqlTypes;
+using PerformanceMonitor.Collectors;
+using PerformanceMonitor.Common;
+using PerformanceMonitor.Darling.Storage;
+using Xunit;
+
+namespace Darling.Tests;
+
+/// 
+/// #5644 final review: the daily summary's blocking_events counts blocking WAITS, as the alert value, the fleet
+/// tile and the MCP counts do, not blocked-process report rows. SQL Server writes a report for a still-blocked session
+/// every threshold seconds, so a nine-report wait and a six-report wait are 2 events, not 15. Gated on DARLING_TEST_PG;
+/// read through the product's own statement (), on the blocked-process arm, the
+/// always-on DMV snapshot fallback, and the Azure master scope.
+/// 
+/* #1776 own-store: every fact plants its rows under one dedicated server id and deletes them in cleanup. */
+[Collection("live-postgres")]
+public sealed class DailySummaryBlockingWaitsLiveTests
+{
+    private const string ServerName = "darling-daily-blocking-waits";
+    private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName);
+
+    private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+
+    private static async Task ExecAsync(NpgsqlConnection c, string sql, CancellationToken ct, params object[] p)
+    {
+        using var cmd = new NpgsqlCommand(sql, c);
+        foreach (var v in p)
+            cmd.Parameters.AddWithValue(v);
+        await cmd.ExecuteNonQueryAsync(ct);
+    }
+
+    /* One wait as SQL Server reports it: a report every 5 s while the session stays blocked, the k-th carrying
+       wait_time_ms = k * 5 s. */
+    private static async Task PlantWaitAsync(
+        NpgsqlConnection c, DateTime start, int blocked, int blocker, int reports, string database, bool dmv, CancellationToken ct)
+    {
+        for (var k = 1; k <= reports; k++)
+        {
+            var at = start.AddSeconds(5 * k);
+            if (dmv)
+                await ExecAsync(c, "INSERT INTO dmv_blocking_snapshots (collection_id, collection_time, server_id, server_name, event_time, database_name, blocked_spid, blocking_spid, wait_time_ms, blocking_status) VALUES ($1,$2,$3,$4,$2,$5,$6,$7,$8,'suspended')",
+                    ct, CollectionIdGenerator.Next(), at, ServerId, ServerName, database, blocked, blocker, (long)(5000 * k));
+            else
+                await ExecAsync(c, "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, wait_time_ms, blocking_spid, blocked_spid, blocking_status, database_name) VALUES ($1,$2,$3,$4,$2,$5,$6,$7,'suspended',$8)",
+                    ct, CollectionIdGenerator.Next(), at, ServerId, ServerName, (long)(5000 * k), blocker, blocked, database);
+        }
+    }
+
+    /// Plants nine reports of one wait (database master) and six of another (GP), 15 rows and
+    /// 2 waits, on a quiet UTC day (as DMV snapshots when ), then reads that day's
+    /// blocking_events through the statement, or the master-scoped one with the sibling list at $5 when
+    /// .
+    private static async Task DayBlockingEventsAsync(bool dmv, bool scoped)
+    {
+        var cs = ConnectionString;
+        Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live test.");
+        var ct = TestContext.Current.CancellationToken;
+        using var connection = new NpgsqlConnection(cs);
+        await connection.OpenAsync(ct);
+        await PgMigrations.MigrateAsync(connection, ct);
+        await DeleteRowsAsync(connection, ct);
+        var bodySucceeded = false;
+        try
+        {
+            await ExecAsync(connection, @"
+INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_major_version, created_date, modified_date)
+VALUES ($1, $2, $2, TRUE, 16, now()::timestamp, now()::timestamp)
+ON CONFLICT (server_id) DO UPDATE SET is_enabled = TRUE", ct, ServerId, ServerName);
+
+            var day = DateTime.SpecifyKind(DateTime.UtcNow.Date.AddDays(-3), DateTimeKind.Unspecified);
+            await PlantWaitAsync(connection, day.AddHours(2), blocked: 52, blocker: 51, reports: 9, "master", dmv, ct);
+            await PlantWaitAsync(connection, day.AddHours(3), blocked: 53, blocker: 51, reports: 6, "GP", dmv, ct);
+
+            var sql = scoped ? DailySummaryAzureMasterScope.Scope(DailySummarySql.RangeSql) : DailySummarySql.RangeSql;
+            using var read = new NpgsqlCommand("SELECT blocking_events FROM (" + sql + ") AS calendar", connection);
+            read.Parameters.Add(new NpgsqlParameter { TypedValue = ServerId });
+            read.Parameters.Add(new NpgsqlParameter { TypedValue = day });
+            read.Parameters.Add(new NpgsqlParameter { TypedValue = day.AddDays(1) });
+            read.Parameters.Add(new NpgsqlParameter { TypedValue = EventWindowFloor.For(day) });
+            if (scoped)
+                read.Parameters.Add(new NpgsqlParameter { Value = new[] { "GP" }, NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Text });
+            var count = Convert.ToInt64(await read.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture);
+            bodySucceeded = true;
+            return count;
+        }
+        finally
+        {
+            await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) =>
+                await DeleteRowsAsync(cleanup, cleanupCt));
+        }
+    }
+
+    [Fact]
+    public async Task TheDay_CountsBlockedProcessWaits_NotReports()
+    {
+        Assert.Equal(2, await DayBlockingEventsAsync(dmv: false, scoped: false));
+    }
+
+    [Fact]
+    public async Task TheDay_CountsDmvSnapshotWaits_NotSnapshots_WhenThereAreNoReports()
+    {
+        Assert.Equal(2, await DayBlockingEventsAsync(dmv: true, scoped: false));
+    }
+
+    [Fact]
+    public async Task TheAzureMasterScope_StillSkipsTheSiblingDatabase_AndCountsWaits()
+    {
+        /* The GP wait belongs to a database monitored as its own target: the master's day keeps the one master wait. */
+        Assert.Equal(1, await DayBlockingEventsAsync(dmv: false, scoped: true));
+    }
+
+    private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct)
+    {
+        using var cleanup = new NpgsqlCommand(
+            $"DELETE FROM blocked_process_reports WHERE server_id = {ServerId}; " +
+            $"DELETE FROM dmv_blocking_snapshots WHERE server_id = {ServerId}; " +
+            $"DELETE FROM servers WHERE server_id = {ServerId};", connection);
+        await cleanup.ExecuteNonQueryAsync(ct);
+    }
+}
diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs
index 56d4f0f241..5f12dac077 100644
--- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs
@@ -341,7 +341,7 @@ have not happened yet (review finding on #3525). The fleet sweep does NOT read t
     /// viewer's Performance Calendar. This was a hand-copied literal described as "verbatim" with nothing
     /// enforcing it, so the calendar and this MCP tool could silently answer the same day differently (#1661).
     ///
- public const string DailySummaryRangeSql = DailySummarySql.RangeSql; + public static readonly string DailySummaryRangeSql = DailySummarySql.RangeSql; /// The range read's rows plus the horizon they were judged against (#3541 A9). /// One row per day the spine holds, oldest first, each stamped with its . diff --git a/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs b/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs index f937785c53..4d4ac16bc6 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs @@ -8,6 +8,7 @@ using System; using System.Linq; +using PerformanceMonitor.Common; namespace PerformanceMonitor.Darling.Storage; @@ -38,7 +39,7 @@ public static class DailySummarySql /// per-stored-name aggregation this read always ran; wait_per_type merges its groups, so the trim runs /// once per group rather than once per row. /// - public const string RangeSql = """ + private const string RangeSqlTemplate = """ WITH wait_per_spelling AS ( SELECT date_trunc('day', collection_time) AS d, wait_type, SUM(delta_wait_time_ms) AS ms FROM v_wait_stats @@ -87,17 +88,17 @@ FROM v_deadlocks AND collection_time >= $4 GROUP BY 1 ), + /*blocking-waits*/ bpr AS ( + /* #5644: counts WAITS, not report rows (BlockingWaits.WaitsCteSql, the one definition the alert, the + fleet tile and the MCP counts share), bucketed on the day of the wait's longest report. */ SELECT date_trunc('day', event_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms - FROM v_blocked_process_reports - WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 - AND collection_time >= $4 + FROM bpr_waits GROUP BY 1 ), dmv AS ( SELECT date_trunc('day', collection_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms - FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 + FROM dmv_waits GROUP BY 1 ), cpu AS ( @@ -221,6 +222,24 @@ FROM day_spine s ORDER BY s.d """; + /// + /// with the two blocking-wait chains spliced in (#5644): the day's + /// blocking_events counts blocking WAITS, as the alert value, the fleet "Blocking" tile and the MCP counts + /// do (), not blocked-process report rows (SQL Server writes about eleven + /// per one-minute wait). The blocked-process side carries database_name for + /// 's filter; the DMV side keeps its collected-day bucket. + /// + public static readonly string RangeSql = RangeSqlTemplate.Replace( + "/*blocking-waits*/", + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "bpr_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms, database_name FROM v_blocked_process_reports + WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $4") + "," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "dmv_waits", + @"SELECT blocked_spid, blocking_spid, event_time, collection_time, wait_time_ms FROM v_dmv_blocking_snapshots + WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3") + ",", + StringComparison.Ordinal); + /// /// The queries CTE exactly as it appears in — the ONE part of the /// daily summary that reads a table subject to the 4-day raw drop. Every other source here (wait_stats, diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs index e48845f6dc..dffa0cbb2d 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs @@ -36,7 +36,7 @@ public sealed partial class ViewerDataService /// service's DarlingHealthReader (the get_daily_health MCP tool) — the two used to keep /// hand-copied literals with nothing enforcing the copy (#1661). /// - public const string DailySummaryRangeSql = DailySummarySql.RangeSql; + public static readonly string DailySummaryRangeSql = DailySummarySql.RangeSql; /// The tier-routed form. See . public static string DailySummaryRangeSqlFor(RetentionTier tier) => DailySummarySql.RangeSqlFor(tier); diff --git a/Lite.Tests/AzureMasterDailySummaryScopeTests.cs b/Lite.Tests/AzureMasterDailySummaryScopeTests.cs index 15387fbd8f..e8c4d6385e 100644 --- a/Lite.Tests/AzureMasterDailySummaryScopeTests.cs +++ b/Lite.Tests/AzureMasterDailySummaryScopeTests.cs @@ -50,12 +50,14 @@ private async Task ExecAsync(string sql, params object?[] args) await cmd.ExecuteNonQueryAsync(); } - private async Task SeedBprAsync(int count, string database) + /* #5644: the day counts WAITS; reports with the same time and wait time (and, here, no session ids) are one + wait, so callers seeding two databases in one day give each its own start minute. */ + private async Task SeedBprAsync(int count, string database, int startMinute = 0) { for (var i = 0; i < count; i++) await ExecAsync( "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, event_time, server_id, server_name, database_name, wait_time_ms) VALUES ($1,$2,$2,$3,'TestServer',$4,1000)", - _nextId--, Day.AddHours(1).AddMinutes(i), ServerId, database); + _nextId--, Day.AddHours(1).AddMinutes(startMinute + i), ServerId, database); } private async Task SeedDmvAsync(int count, string database) @@ -138,7 +140,7 @@ public async Task MasterWithSiblings_DmvArmCountsOnlyMasterRows() public async Task SqlServerTarget_AndSiblinglessMaster_CountEverything(bool emptyList) { await SeedBprAsync(4, "GP"); - await SeedBprAsync(3, "master"); + await SeedBprAsync(3, "master", startMinute: 30); await SeedDeadlocksAsync(5, "GP"); await SeedDeadlocksAsync(3, "master"); AnalysisService.SeparatelyMonitoredDatabasesProvider = _ => emptyList ? Array.Empty() : null; diff --git a/Lite.Tests/BlockingWaitDuckDbTests.cs b/Lite.Tests/BlockingWaitDuckDbTests.cs index 48007ccf98..9664afd12a 100644 --- a/Lite.Tests/BlockingWaitDuckDbTests.cs +++ b/Lite.Tests/BlockingWaitDuckDbTests.cs @@ -183,6 +183,42 @@ public async Task TheDmvSnapshotFallback_CountsWaitsToo() Assert.Equal(2, await OverviewCountAsync()); } + /* ── #5644 final review: the server tab badge and the daily summary count waits too ── */ + + private async Task BadgeBlockingCountAsync() => + (await new LocalDataService(_duckDb).GetAlertCountsAsync(ServerId, hoursBack: 2, fromDate: null, toDate: null)).blockingCount; + + /* The day buckets are UTC days of the wait's longest report; a run across midnight is two buckets, so sum them. */ + private async Task DailySummaryBlockingEventsAsync() + { + var today = DateTime.UtcNow.Date; + var rows = await new LocalDataService(_duckDb).GetDailySummaryRangeAsync(ServerId, today.AddDays(-1), today.AddDays(2)); + return rows.Sum(r => r.BlockingEvents); + } + + [Fact] + public async Task TheServerTabBadgeAndTheDailySummary_CountWaitsNotReports() + { + /* Nine reports of one wait, then six of another: 15 report rows, 2 waits. */ + await SeedWaitAsync(52, 51, Ago(30), reports: 9); + await SeedWaitAsync(53, 51, Ago(20), reports: 6); + + Assert.Equal(2, CSharpWaitCount()); + Assert.Equal(2, await BadgeBlockingCountAsync()); + Assert.Equal(2, await DailySummaryBlockingEventsAsync()); + Assert.Equal(2, await OverviewCountAsync()); + } + + [Fact] + public async Task TheServerTabBadgeAndTheDailySummary_CountWaitsOnTheDmvFallbackToo() + { + await SeedWaitAsync(52, 51, Ago(30), reports: 11, dmv: true); + await SeedWaitAsync(53, 51, Ago(20), reports: 6, dmv: true); + + Assert.Equal(2, await BadgeBlockingCountAsync()); + Assert.Equal(2, await DailySummaryBlockingEventsAsync()); + } + /* ── #5644 wave-1 review: sub-millisecond starts and NULL times ── */ private sealed record Edge(int Blocked, int Blocker, DateTime? Time, long? Wait); diff --git a/Lite.Tests/PerformanceCalendarDataTests.cs b/Lite.Tests/PerformanceCalendarDataTests.cs index 706a5b9986..2b0403cdf1 100644 --- a/Lite.Tests/PerformanceCalendarDataTests.cs +++ b/Lite.Tests/PerformanceCalendarDataTests.cs @@ -104,10 +104,12 @@ private Task SeedDmvBlockingAsync(DateTime day) => "INSERT INTO dmv_blocking_snapshots (collection_id, collection_time, server_id, server_name, monitor_loop) VALUES ($1,$2,$3,$4,1)", _nextId--, day.AddHours(1), ServerId, ServerName); - private Task SeedBprAsync(DateTime day, long waitTimeMs) => + /* #5644: the day's blocking count is WAITS, and reports with the same time and wait time are one wait, so a + caller that wants several distinct events gives each its own minute. */ + private Task SeedBprAsync(DateTime day, long waitTimeMs, int minute = 0) => ExecAsync( "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, wait_time_ms, event_time) VALUES ($1,$2,$3,$4,$5,$2)", - _nextId--, day.AddHours(1), ServerId, ServerName, waitTimeMs); + _nextId--, day.AddHours(1).AddMinutes(minute), ServerId, ServerName, waitTimeMs); private Task SeedMemoryAsync(DateTime day, int process, int system) => ExecAsync( @@ -176,7 +178,7 @@ public async Task GetDailySummaryRange_BucketsEachDay_AndBandsViaSharedCalculato // exactly (#3539 A3) — each under the 10 s wait bar so the count arm is what decides. await SeedCollectionRunAsync(Day(11)); for (var i = 0; i < 120; i++) - await SeedBprAsync(Day(11), 6_000); + await SeedBprAsync(Day(11), 6_000, minute: i); // 07-12 Healthy: collected, quiet. await SeedCollectionRunAsync(Day(12)); diff --git a/Lite/Services/LocalDataService.Blocking.cs b/Lite/Services/LocalDataService.Blocking.cs index 4ac6e22d2d..777a6608e4 100644 --- a/Lite/Services/LocalDataService.Blocking.cs +++ b/Lite/Services/LocalDataService.Blocking.cs @@ -581,7 +581,7 @@ FROM population p /// /// Gets lightweight blocking + deadlock counts and latest event time for alert badge updates. - /// Much cheaper than fetching full rows with XML — just COUNT(*) and MAX(time). + /// Much cheaper than fetching full rows with XML — a count of waits and MAX(time). /// /// /// / are the tab's custom range as naive-UTC instants @@ -601,11 +601,20 @@ FROM population p /* blocking_count prefers the blocked-process-report; falls back to the always-on DMV snapshot when BPR captured nothing (AWS RDS). latest_event_time includes DMV blocking recency too. */ + /* #5644: the count is WAITS, not report rows (BlockingWaits.WaitsCteSql, the definition the Overview tile, the + alert and the analysis fact share): SQL Server writes a report for a still-blocked session every threshold + seconds, so one one-minute wait put about eleven on this badge beside a tile that said 1. */ command.CommandText = @" +WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.DuckDb, "xe_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms + FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3") + @" AS ev") + "," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.DuckDb, "dmv_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms + FROM v_dmv_blocking_snapshots + WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3") + @" SELECT - COALESCE(NULLIF((SELECT COUNT(*) FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3") + @" AS ev), 0), - (SELECT COUNT(*) FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3)) AS blocking_count, + COALESCE(NULLIF((SELECT COUNT(*) FROM xe_waits), 0), (SELECT COUNT(*) FROM dmv_waits)) AS blocking_count, (SELECT " + StoredEventCopies.DeadlockDistinctCount + @" FROM v_deadlocks AS dl WHERE server_id = $1 AND deadlock_time >= $2 AND deadlock_time <= $3) AS deadlock_count, (SELECT MAX(t) FROM ( SELECT MAX(event_time) AS t FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3") + @" AS ev diff --git a/Lite/Services/LocalDataService.DailySummary.cs b/Lite/Services/LocalDataService.DailySummary.cs index 2f966d106d..caede84489 100644 --- a/Lite/Services/LocalDataService.DailySummary.cs +++ b/Lite/Services/LocalDataService.DailySummary.cs @@ -52,15 +52,24 @@ FROM v_deadlocks AS dl WHERE server_id = $1 AND deadlock_time >= $2 AND deadlock_time < $3 GROUP BY 1 ), +" + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.DuckDb, "bpr_waits", + @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms + FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3/*scope*/") + @" AS ev") + "," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.DuckDb, "dmv_waits", + @"SELECT blocked_spid, blocking_spid, event_time, collection_time, wait_time_ms + FROM v_dmv_blocking_snapshots + WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3/*scope*/") + @", bpr AS ( + /* #5644: counts WAITS, not report rows (BlockingWaits.WaitsCteSql, the definition the alert, the Overview tile + and the analysis fact share), bucketed on the day of the wait's longest report. */ SELECT date_trunc('day', event_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms - FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3/*scope*/") + @" AS ev + FROM bpr_waits GROUP BY 1 ), dmv AS ( SELECT date_trunc('day', collection_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms - FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3/*scope*/ + FROM dmv_waits GROUP BY 1 ), cpu AS ( From 5ba497c15f9e9eec92bc5d351b1a2227e4ba5fd3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:28:30 -0400 Subject: [PATCH 34/42] Starter dashboard blocking trend panel is titled for reports or snapshot rows (#5644) --- Darling/Darling.Tests/ChartIntegerTicksPinTests.cs | 2 +- Darling/Darling.Tests/ViewTemplatesTests.cs | 6 +++++- Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs | 7 +++++-- .../wwwroot/js/view-templates.js | 8 +++++--- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Darling/Darling.Tests/ChartIntegerTicksPinTests.cs b/Darling/Darling.Tests/ChartIntegerTicksPinTests.cs index e67c2f5b3f..025bc85852 100644 --- a/Darling/Darling.Tests/ChartIntegerTicksPinTests.cs +++ b/Darling/Darling.Tests/ChartIntegerTicksPinTests.cs @@ -85,7 +85,7 @@ public void EveryCountChartOnTheServerPage_DeclaresWholeNumbers_AndVizLinePasses public void TheStarterDashboardAndComposedCountCharts_AskForWholeNumberTicks() { var templates = Js("view-templates.js"); - Assert.Matches(@"series: \[\{ key: ""count"", label: ""Reports"" \}\],\n\s+format: ""int"",", templates); + Assert.Matches(@"series: \[\{ key: ""count"", label: ""Reports or snapshot rows"" \}\],\n\s+format: ""int"",", templates); Assert.Contains("...READ_FIELDS.get_deadlock_trend.line,", templates, StringComparison.Ordinal); Assert.Matches(@"get_deadlock_trend: \{[\s\S]*?line: \{[^}]*?format: ""int"",[^}]*?label: ""Deadlocks""", Js("read-fields.js")); diff --git a/Darling/Darling.Tests/ViewTemplatesTests.cs b/Darling/Darling.Tests/ViewTemplatesTests.cs index 9410d1f4c4..c66d84e2d2 100644 --- a/Darling/Darling.Tests/ViewTemplatesTests.cs +++ b/Darling/Darling.Tests/ViewTemplatesTests.cs @@ -306,7 +306,11 @@ public void EveryTemplateDataPanel_ExplainsItsOwnEmptyState() /* The two reads whose EMPTY ARRAY means "it did not happen" say exactly that, rather than inheriting a sentence about collection. Absence and non-occurrence are the distinction this whole codebase keeps closing, and a starter dashboard is the worst place to blur it. */ - Assert.Contains("an empty trend here means none happened, not that nothing was collected", js, StringComparison.Ordinal); + /* The blocking trend counts blocked process REPORTS (or DMV snapshot rows), so its sentence says "captured" (#5644 W16); + the deadlock trend panel spreads READ_FIELDS and keeps "happened". */ + Assert.Contains("an empty trend here means none were captured, not that nothing was collected", js, StringComparison.Ordinal); + Assert.Contains("an empty trend here means none happened, not that nothing was collected", + ReadRepoFileLf(Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "wwwroot", "js", "read-fields.js")), StringComparison.Ordinal); /* And the two reads that CANNOT be honest on day one are deliberately absent, not merely unused: the analysis pass writes nothing for 24 hours, and a target with Query Store off has nothing ever. */ diff --git a/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs b/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs index 80001c5b53..de1b2b75eb 100644 --- a/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs +++ b/Darling/Darling.Tests/WebBlockingSourceBehaviourTests.cs @@ -141,10 +141,13 @@ public void TheBlockingTrendChart_SaysItCountsBlockedProcessReports() Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(tabs, "line\\(\"Blocked process report trend\", \"get_blocking_trend\"").Count); Assert.Contains("const BLOCKING_REPORT_SERIES = [{ key: \"count\", label: \"Reports\" }];", tabs); Assert.DoesNotContain("No blocking events in this window — an empty trend", tabs); - Assert.Contains("label: \"Reports\"", templates); + /* The starter dashboard panel has no collector tag to follow, so it words itself for both sources. */ + Assert.Contains("title: \"Blocking trend (reports or snapshot rows)\"", templates); + Assert.Contains("label: \"Reports or snapshot rows\"", templates); + Assert.DoesNotContain("title: \"Blocked process report trend\"", templates); /* The axis caption says it too: both tab charts and the starter dashboard panel carry unit "reports". */ Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(tabs, "unit: \"reports\",").Count); - Assert.Contains("unit: \"reports\",", templates); + Assert.Contains("unit: \"reports or snapshot rows\",", templates); } /// #5644 W16 review: only a true negative (status "empty") takes the page's one sentence. A not_collected or unavailable diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js index ab190fdd98..7af79e6036 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/view-templates.js @@ -183,16 +183,18 @@ export const DASHBOARD_TEMPLATES = [ definition: { panels: [ { - title: "Blocked process report trend", + /* #5644 W16: a Custom View panel has no collector tag, and get_blocking_trend falls back from blocked process + reports to DMV snapshot rows, so the title, legend and axis caption are true for both. */ + title: "Blocking trend (reports or snapshot rows)", read: "get_blocking_trend", params: { server, hours: 24 }, viz: "line", rowsKey: "trend", xKey: "time", emptyText: "No blocked process reports in this window — an empty trend here means none were captured, not that nothing was collected.", - series: [{ key: "count", label: "Reports" }], + series: [{ key: "count", label: "Reports or snapshot rows" }], format: "int", - unit: "reports", + unit: "reports or snapshot rows", }, { title: "Deadlocks", From b69234eae6b0ba57a623a3655cd9081d242ac2f2 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:15:59 -0400 Subject: [PATCH 35/42] Overview tab Deadlocks chart uses the same empty sentence as the Blocking tab (#5644) --- Darling/Darling.Tests/WebAuditLaneDLabelTests.cs | 2 ++ .../wwwroot/js/pages/server-tabs.js | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs b/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs index 1daec931d3..2d7a54e07a 100644 --- a/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs +++ b/Darling/Darling.Tests/WebAuditLaneDLabelTests.cs @@ -48,6 +48,8 @@ public void W16_TheDeadlockTrendChartAndTheDeadlocksTableAreNamedApartAndEmptyTh Assert.Contains("line(\"Deadlock trend\", \"get_deadlock_trend\", { server, hours: ctx.hours }, \"trend\", \"time\", COUNT_SERIES, {\n subtitle: ctx.label,\n format: \"int\",\n atTimeItem: \"deadlocks\",\n emptyText: DEADLOCKS_EMPTY_TEXT,", tabs, StringComparison.Ordinal); Assert.Contains("const DEADLOCKS_EMPTY_TEXT = \"No deadlocks in this window.\";", tabs, StringComparison.Ordinal); Assert.Contains(" ctx.label,\n DEADLOCKS_EMPTY_TEXT,\n 2,\n \"separately_monitored_note\"", tabs, StringComparison.Ordinal); + Assert.Contains("line(\"Deadlocks\", \"get_deadlock_trend\", { server, hours: ctx.hours }, \"trend\", \"time\", COUNT_SERIES, {\n subtitle: ctx.label,\n format: \"int\",\n atTimeItem: \"deadlocks\",\n emptyText: DEADLOCKS_EMPTY_TEXT,\n emptyWins: true,", tabs, StringComparison.Ordinal); + Assert.DoesNotContain("an empty trend here means none happened", tabs, StringComparison.Ordinal); } [Fact] diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js index 9d99028b62..74d5f3f337 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js @@ -1112,7 +1112,8 @@ export const SERVER_TABS = [ subtitle: ctx.label, format: "int", atTimeItem: "deadlocks", - emptyText: "No deadlocks in this window — an empty trend here means none happened, not that nothing was collected.", + emptyText: DEADLOCKS_EMPTY_TEXT, + emptyWins: true, }), fileIoPanel(server, ctx), table( From 3c4fc7058cb5b84189a4da8222e72417c73406cf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:00:59 -0400 Subject: [PATCH 36/42] Tests follow the web UI audit: Alert History harness finds Metric by header, localized detail pin, plain-text.js import, emptyWins signature (#5644) --- .../Darling.Tests/AlertNotebookRenderClientTests.cs | 4 +++- .../Darling.Tests/FinOpsTabIndexAnalysisPageTests.cs | 2 +- .../Darling.Tests/FinOpsTabUtilizationRawPageTests.cs | 2 +- Darling/Darling.Tests/ServerPageTabsTests.cs | 4 +++- Darling/Darling.Tests/alert-history-range-harness.mjs | 10 +++++++++- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Darling/Darling.Tests/AlertNotebookRenderClientTests.cs b/Darling/Darling.Tests/AlertNotebookRenderClientTests.cs index d7de36724b..5289e7697b 100644 --- a/Darling/Darling.Tests/AlertNotebookRenderClientTests.cs +++ b/Darling/Darling.Tests/AlertNotebookRenderClientTests.cs @@ -126,7 +126,9 @@ public void Views_AlertHeaderCell_RendersValueThresholdAndDetailText_ThroughElsT Assert.Contains("a.detail_text", views, StringComparison.Ordinal); // The detail_text line must go through el()'s { text: ... } prop (textContent), never innerHTML. - Assert.Contains("el(\"pre\", { class: \"code\", text: a.detail_text })", views, StringComparison.Ordinal); + // W20 (#5644) shows the stored text with its UTC lines converted to the viewer's zone (localizeAlertDetail returns + // a string), so the text prop now holds that call; it is still el()'s text path, not markup. + Assert.Contains("el(\"pre\", { class: \"code\", text: localizeAlertDetail(a.detail_text, a.newest_sample) })", views, StringComparison.Ordinal); // Scope the innerHTML ban to renderAlertHeaderCell's own body — the file's header comment (R4) // says "never innerHTML" in prose, which the whole-file assertion this replaces was tripping on. diff --git a/Darling/Darling.Tests/FinOpsTabIndexAnalysisPageTests.cs b/Darling/Darling.Tests/FinOpsTabIndexAnalysisPageTests.cs index 8b5d004855..d4794fc4d1 100644 --- a/Darling/Darling.Tests/FinOpsTabIndexAnalysisPageTests.cs +++ b/Darling/Darling.Tests/FinOpsTabIndexAnalysisPageTests.cs @@ -358,7 +358,7 @@ public void TheTabImportsOnlyTheSharedHelpers() { var imports = Regex.Matches(Tab(), "from \"([^\"]+)\";").Select(m => m.Groups[1].Value).ToList(); Assert.NotEmpty(imports); - Assert.All(imports, i => Assert.Contains(i, new[] { "./database-box.js", "../../panels.js", "../../util.js", "./gate.js" })); + Assert.All(imports, i => Assert.Contains(i, new[] { "./database-box.js", "../../panels.js", "../../util.js", "../../plain-text.js", "./gate.js" })); Assert.Contains("./gate.js", imports); } diff --git a/Darling/Darling.Tests/FinOpsTabUtilizationRawPageTests.cs b/Darling/Darling.Tests/FinOpsTabUtilizationRawPageTests.cs index 83c15dd8eb..4c3b56f865 100644 --- a/Darling/Darling.Tests/FinOpsTabUtilizationRawPageTests.cs +++ b/Darling/Darling.Tests/FinOpsTabUtilizationRawPageTests.cs @@ -126,7 +126,7 @@ public void TheTab_ImportsOnlyFromTheSharedModules() { var imports = Regex.Matches(Tab(), "from \"([^\"]+)\"").Select(m => m.Groups[1].Value).ToList(); Assert.NotEmpty(imports); - Assert.All(imports, i => Assert.Contains(i, new[] { "../../panels.js", "../../charts.js", "../../util.js", "../../read-fields.js", "./gate.js" })); + Assert.All(imports, i => Assert.Contains(i, new[] { "../../panels.js", "../../charts.js", "../../util.js", "../../read-fields.js", "../../plain-text.js", "./gate.js" })); Assert.Contains("./gate.js", imports); } diff --git a/Darling/Darling.Tests/ServerPageTabsTests.cs b/Darling/Darling.Tests/ServerPageTabsTests.cs index 01f75b2ea5..ef9e5f5050 100644 --- a/Darling/Darling.Tests/ServerPageTabsTests.cs +++ b/Darling/Darling.Tests/ServerPageTabsTests.cs @@ -937,9 +937,11 @@ public void EveryDataPanel_ExplainsItsOwnEmptyState() rendered the same way), and #4843 appended `columnGroups = null` (the opt-in column groups of a wide grid), and #5226 appended `control = null` (a node drawn under the title: the ranking selector of the Top Queries / Top Procedures cards), and #5245 appended `dbScope = null` (the panel's own database-scope chip, for a panel whose scope differs from its read's class; unrelated to this guard)., and the release walk appended `extensionMissingLine = null` (the plain line a precondition panel shows when its optional extension is not installed; unrelated to this guard). The literal is spelled out here rather than truncated at emptyText because a prefix match would stop noticing a parameter inserted BEFORE it. */ + /* #5644 W16 appended `emptyWins = false` after those (the panel's one agreed sentence replaces the read's own empty message + on an empty envelope; unrelated to this guard, and the throw below still fires without an emptyText). */ Assert.Contains( "function table(title, read, params, rowsKey, columns, subtitle, emptyText, span = 2, " - + "noteKey = null, moreNoteKeys = null, columnGroups = null, control = null, dbScope = null, extensionMissingLine = null)", + + "noteKey = null, moreNoteKeys = null, columnGroups = null, control = null, dbScope = null, extensionMissingLine = null, emptyWins = false)", js, StringComparison.Ordinal); Assert.Contains( diff --git a/Darling/Darling.Tests/alert-history-range-harness.mjs b/Darling/Darling.Tests/alert-history-range-harness.mjs index 628e8c76d1..853f2cde18 100644 --- a/Darling/Darling.Tests/alert-history-range-harness.mjs +++ b/Darling/Darling.Tests/alert-history-range-harness.mjs @@ -91,7 +91,15 @@ try { const checkbox = (main) => main.all("input").find((i) => i.attrs.type === "checkbox"); const pick = async (sel, value) => { sel.value = String(value); sel.fire("change"); await new Promise((r) => setTimeout(r, 20)); }; const bodyRows = (main) => { const t = main.all("table")[0]; return t ? t.children[1].children : []; }; - const metrics = (main) => bodyRows(main).map((tr) => tr.children[3].textContent); + // The Metric column is found by its header, not a fixed index: W14 (#5644) put Triage and Mute before Time, so the + // column position moves with the grid's leading columns (and with whether the seat can edit). + const metricIndex = (main) => { + const t = main.all("table")[0]; + const i = t ? t.children[0].children[0].children.findIndex((th) => th.textContent.trim().startsWith("Metric")) : -1; + if (i < 0) throw new Error("the Alert History grid has no Metric header"); + return i; + }; + const metrics = (main) => { const i = metricIndex(main); return bodyRows(main).map((tr) => tr.children[i].textContent); }; const settle = () => new Promise((r) => setTimeout(r, 20)); const out = {}; From 168ffe347d4b6a5c45a4484aa6c60d92093baebf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:05:12 -0400 Subject: [PATCH 37/42] Update four test pins the #5644 web UI audit outdated: wait-counting seeds, Default Trace columns, AvgCpuMs golden, Regression column spec --- .../DailySummaryAzureMasterScopeLiveTests.cs | 15 +++++++++------ ...WindowedReadsAreBoundedLivePostgresTests.cs | 5 ++++- .../FinOpsDatabaseResources/golden.json | 18 ++++++++++++------ .../ProductiveZeroBandingTests.cs | 4 +++- ...werDailySummaryAzureMasterScopeLiveTests.cs | 12 +++++++----- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/Darling/Darling.Tests/DailySummaryAzureMasterScopeLiveTests.cs b/Darling/Darling.Tests/DailySummaryAzureMasterScopeLiveTests.cs index e06cc241d7..27fafab461 100644 --- a/Darling/Darling.Tests/DailySummaryAzureMasterScopeLiveTests.cs +++ b/Darling/Darling.Tests/DailySummaryAzureMasterScopeLiveTests.cs @@ -81,9 +81,9 @@ await Exec(connection, "INSERT INTO collection_log (log_id, server_id, server_na await Server(LoneId, Base + "-lone", 5); await Server(PlainId, Base + "-plain", 3); - async Task Bpr(int id, string name, DateTime day, int waitMs, string db) => + async Task Bpr(int id, string name, DateTime day, int waitMs, string db, int minute = 0) => await Exec(connection, "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, wait_time_ms, blocking_spid, blocked_spid, blocking_status, database_name) VALUES ($1,$2,$3,$4,$2,$5,60,70,'suspended',$6)", - ct, CollectionIdGenerator.Next(), day.AddHours(12), id, name, waitMs, db); + ct, CollectionIdGenerator.Next(), day.AddHours(12).AddMinutes(minute), id, name, waitMs, db); async Task Dmv(int id, string name, DateTime day, int waitMs) => await Exec(connection, "INSERT INTO dmv_blocking_snapshots (collection_id, collection_time, server_id, server_name, event_time, database_name, blocked_spid, blocking_spid, wait_time_ms, blocking_status) VALUES ($1,$2,$3,$4,$2,'master',70,60,$5,'suspended')", ct, CollectionIdGenerator.Next(), day.AddHours(13), id, name, waitMs); @@ -93,10 +93,13 @@ await Exec(connection, "INSERT INTO deadlocks (deadlock_id, collection_time, ser var master = Base + "-master"; /* Day 1: three GP reports waiting 600 s, one of the master's own waiting 2 s; day 2: only GP's, so the - scoped day falls back to the DMV snapshots. Unequal waits, so a peak taken from the wrong set shows. */ - for (var i = 0; i < 3; i++) await Bpr(MasterId, master, Day1, 600_000, "GP"); - await Bpr(MasterId, master, Day1, 2_000, "master"); - for (var i = 0; i < 2; i++) await Bpr(MasterId, master, Day2, 400_000, "GP"); + scoped day falls back to the DMV snapshots. Unequal waits, so a peak taken from the wrong set shows. + #5644: the daily summary counts blocking WAITS (BlockingWaits.WaitsCteSql: one per blocked session, + blocker and derived start), not report rows, so each intended event gets its own minute and is its + own wait; reports of one pair at one instant would be a single wait. */ + for (var i = 0; i < 3; i++) await Bpr(MasterId, master, Day1, 600_000, "GP", minute: i); + await Bpr(MasterId, master, Day1, 2_000, "master", minute: 3); + for (var i = 0; i < 2; i++) await Bpr(MasterId, master, Day2, 400_000, "GP", minute: i); await Dmv(MasterId, master, Day2, 7_000); await Dmv(MasterId, master, Day2, 3_000); /* Day 1 deadlocks: an own-database row; a master-stamped graph wholly in GP; a master-stamped mixed diff --git a/Darling/Darling.Tests/EventWindowedReadsAreBoundedLivePostgresTests.cs b/Darling/Darling.Tests/EventWindowedReadsAreBoundedLivePostgresTests.cs index 02e0d49d8e..926b59c06a 100644 --- a/Darling/Darling.Tests/EventWindowedReadsAreBoundedLivePostgresTests.cs +++ b/Darling/Darling.Tests/EventWindowedReadsAreBoundedLivePostgresTests.cs @@ -107,7 +107,10 @@ ORDER BY sp.collection_time DESC dte.integer_data, dte.severity, dte.error_number, - dte.text_data + dte.text_data, + dte.filename, + dte.event_class, + dte.event_sequence FROM default_trace_events AS dte, svr WHERE dte.server_id = $1 AND dte.event_time - make_interval(mins => svr.offset_minutes) >= $2 diff --git a/Darling/Darling.Tests/Fixtures/FinOpsDatabaseResources/golden.json b/Darling/Darling.Tests/Fixtures/FinOpsDatabaseResources/golden.json index 175cfe3fc4..ddef0e5b74 100644 --- a/Darling/Darling.Tests/Fixtures/FinOpsDatabaseResources/golden.json +++ b/Darling/Darling.Tests/Fixtures/FinOpsDatabaseResources/golden.json @@ -106,7 +106,8 @@ "PctCpu": 36.51, "PctIo": 5.56, "TotalCpuTimeMs": 0, - "AvgIoMb": 0 + "AvgIoMb": 0, + "AvgCpuMs": 0 }, { "DatabaseName": "Bravo", @@ -116,7 +117,8 @@ "PctCpu": 15.87, "PctIo": 0.88, "TotalCpuTimeMs": 0, - "AvgIoMb": 0 + "AvgIoMb": 0, + "AvgCpuMs": 0 }, { "DatabaseName": "Charlie", @@ -126,7 +128,8 @@ "PctCpu": 15.87, "PctIo": 0.00, "TotalCpuTimeMs": 0, - "AvgIoMb": 0 + "AvgIoMb": 0, + "AvgCpuMs": 0 } ], "byAvg": [ @@ -138,7 +141,8 @@ "PctCpu": 0, "PctIo": 0, "TotalCpuTimeMs": 5000, - "AvgIoMb": 0.2000 + "AvgIoMb": 0.2000, + "AvgCpuMs": 500.00 }, { "DatabaseName": "Charlie", @@ -148,7 +152,8 @@ "PctCpu": 0, "PctIo": 0, "TotalCpuTimeMs": 5000, - "AvgIoMb": 0.0000 + "AvgIoMb": 0.0000, + "AvgCpuMs": 500.00 }, { "DatabaseName": "", @@ -158,7 +163,8 @@ "PctCpu": 0, "PctIo": 0, "TotalCpuTimeMs": 4000, - "AvgIoMb": 0.0000 + "AvgIoMb": 0.0000, + "AvgCpuMs": 500.00 } ] }, diff --git a/Darling/Darling.Tests/ProductiveZeroBandingTests.cs b/Darling/Darling.Tests/ProductiveZeroBandingTests.cs index 39a64ce747..091b11214b 100644 --- a/Darling/Darling.Tests/ProductiveZeroBandingTests.cs +++ b/Darling/Darling.Tests/ProductiveZeroBandingTests.cs @@ -398,11 +398,13 @@ stopped PRODUCING reaches the same number an install countersign already reads. } } + /* #5644 W11: the column's long text now clips to one line (clamp) and prints its ISO times local + (localizeText) instead of wrapping, so the pin follows that spec; the column itself must still exist. */ /* And the web table has a column for it: the viewer reads the full shape (full_detail: true), which always carried the sentence, but no column rendered it, so a regressed row read WARNING with every cell beside it blank. */ Assert.Contains( - "{ key: \"regression_finding\", label: \"Regression\", wrap: true, plain: true }", + "{ key: \"regression_finding\", label: \"Regression\", clamp: true, localizeText: true, plain: true }", ReadRepoFile("Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js"), StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/ViewerDailySummaryAzureMasterScopeLiveTests.cs b/Darling/Darling.Tests/ViewerDailySummaryAzureMasterScopeLiveTests.cs index 0bc8719c21..e189913de8 100644 --- a/Darling/Darling.Tests/ViewerDailySummaryAzureMasterScopeLiveTests.cs +++ b/Darling/Darling.Tests/ViewerDailySummaryAzureMasterScopeLiveTests.cs @@ -63,9 +63,11 @@ private static async Task WithStoreAsync(Func + private static Task BprAsync(NpgsqlConnection connection, int id, DateTime day, int waitMs, string db, CancellationToken ct, int minute = 0) => DarlingMcpTestData.ExecAsync(connection, ct, "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, wait_time_ms, blocking_spid, blocked_spid, blocking_status, database_name) VALUES ($1, $2, $3, 's', $2, $4, 60, 70, 'suspended', $5)", - CollectionIdGenerator.Next(), day.AddHours(12), id, waitMs, db); + CollectionIdGenerator.Next(), day.AddHours(12).AddMinutes(minute), id, waitMs, db); private static Task DmvAsync(NpgsqlConnection connection, int id, DateTime day, int waitMs, CancellationToken ct) => DarlingMcpTestData.ExecAsync(connection, ct, From 54909f3eebc0c000d9e7923bc996c2f6651df036 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:07:39 -0400 Subject: [PATCH 38/42] CI fixes: four pins follow the #5644 wording and panel changes; darling filter reaches the linked watermark test --- .github/workflows/build.yml | 2 ++ Darling/Darling.Tests/PgIndexBloatCoverageTests.cs | 2 +- Darling/Darling.Tests/WebDataStartNoteTests.cs | 2 +- Darling/Darling.Tests/WebServerTrendsBehaviourTests.cs | 2 +- Darling/Darling.Tests/WebWaitsActivityTrendsBehaviourTests.cs | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ba6ffdf6a8..7f9d36aa1e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -735,6 +735,8 @@ jobs: - 'Lite.Tests/LiteTimeRangeGapTests.cs' # #5641: Darling.Tests compiles this Lite.Tests file in (Linked\AlertLegendParityTests.cs), so an edit to it has to reach the Darling suite. - 'Lite.Tests/AlertLegendParityTests.cs' + # #5644: Darling.Tests compiles this Lite.Tests file in (Linked\DefaultTraceEventSequenceWatermarkTests.cs), so an edit to it has to reach the Darling suite. + - 'Lite.Tests/DefaultTraceEventSequenceWatermarkTests.cs' # #5596: Darling.Tests' WpfExitSwitchPinTests reads this project file for the WPF exit switch, so an edit to it has to reach the Darling suite. - 'Lite.Tests/Lite.Tests.csproj' # #5616: the two shard scripts are run by the shard jobs and parsed by Darling.Tests pins. diff --git a/Darling/Darling.Tests/PgIndexBloatCoverageTests.cs b/Darling/Darling.Tests/PgIndexBloatCoverageTests.cs index 3b3e315b0e..0756f02956 100644 --- a/Darling/Darling.Tests/PgIndexBloatCoverageTests.cs +++ b/Darling/Darling.Tests/PgIndexBloatCoverageTests.cs @@ -1024,7 +1024,7 @@ above them would never reach a rendered body - and that is precisely the half th /* The helper forwards it to the descriptor, or the panel below configures something nothing reads. */ Assert.Contains("noteKey = null", tabs, StringComparison.Ordinal); - Assert.Contains("span, noteKey, dbScope, extensionMissingLine }", tabs, StringComparison.Ordinal); + Assert.Contains("span, noteKey, dbScope, extensionMissingLine, emptyWins }", tabs, StringComparison.Ordinal); /* And THIS panel asks for it. Scoped to a window after its own title so a noteKey on some other panel cannot satisfy the claim. */ diff --git a/Darling/Darling.Tests/WebDataStartNoteTests.cs b/Darling/Darling.Tests/WebDataStartNoteTests.cs index 42db38c900..e470ccc33c 100644 --- a/Darling/Darling.Tests/WebDataStartNoteTests.cs +++ b/Darling/Darling.Tests/WebDataStartNoteTests.cs @@ -841,7 +841,7 @@ public void TheNote_IsDrawnByEveryGridPath_AndByNoChartPath() /* #4966: the empty envelope carries the note too, drawn above its message, and a read's answer is composed in the browser's zone for the envelope as it is for rows. */ - Assert.Contains("mount(body, [kept, windowFloorStrip(res.data, desc), emptyStrip(res.message)]);", panels, StringComparison.Ordinal); + Assert.Contains("mount(body, [kept, windowFloorStrip(res.data, desc), emptyStrip(emptyHeadline(desc, res))]);", panels, StringComparison.Ordinal); Assert.Contains("res.kind !== \"data\" && res.kind !== \"empty\"", util, StringComparison.Ordinal); Assert.Contains("windowFloorStrip(res.data, spec)", tabs, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/WebServerTrendsBehaviourTests.cs b/Darling/Darling.Tests/WebServerTrendsBehaviourTests.cs index 289af76fc3..8cd2320f8a 100644 --- a/Darling/Darling.Tests/WebServerTrendsBehaviourTests.cs +++ b/Darling/Darling.Tests/WebServerTrendsBehaviourTests.cs @@ -150,7 +150,7 @@ public void AnEmptyAnswer_ShowsTheReadsMessage_NotAChart() { var r = Run(scenario); Assert.Equal(JsonValueKind.Null, r.GetProperty("chart").ValueKind); - Assert.Contains("recorded in the last 24 hour(s)", Strings(r.GetProperty("empties")).Single()); + Assert.Contains("recorded in the last 24 hours", Strings(r.GetProperty("empties")).Single()); } } diff --git a/Darling/Darling.Tests/WebWaitsActivityTrendsBehaviourTests.cs b/Darling/Darling.Tests/WebWaitsActivityTrendsBehaviourTests.cs index f2943ce0f5..6c30ab8c55 100644 --- a/Darling/Darling.Tests/WebWaitsActivityTrendsBehaviourTests.cs +++ b/Darling/Darling.Tests/WebWaitsActivityTrendsBehaviourTests.cs @@ -134,7 +134,7 @@ public void AnEmptyAnswer_ShowsTheReadsMessage_NotAChart() { var r = Run(scenario); Assert.Equal(JsonValueKind.Null, r.GetProperty("chart").ValueKind); - Assert.Contains("recorded in the last 24 hour(s)", Strings(r.GetProperty("empties")).Single()); + Assert.Contains("recorded in the last 24 hours", Strings(r.GetProperty("empties")).Single()); } } From 0785efaaeb08484c3042ca7c3768150b66b655c7 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:31:21 -0400 Subject: [PATCH 39/42] Band blocking on report counts again; /api/fleet refuses out-of-range hours and worst_count (#5644) W1 made the displayed blocking counts WAITS, but the health band's 5/20 per-hour tiers were measured in blocked-process reports, so a long wait stopped banding by count. The fleet card and the viewer summary keep waits for display and hand the band the report count. /api/fleet refuses hours outside 1..168 and a negative worst_count with the invalid envelope instead of clamping. --- .../BlockingBandReportUnitLiveTests.cs | 165 ++++++++++++++++++ .../Darling.Tests/DarlingWebEndpointsTests.cs | 65 +++++++ .../PgFactCollector.Waits.cs | 5 +- .../DarlingWebEndpoints.cs | 34 +++- .../Mcp/DarlingFleetReader.cs | 113 +++++++++--- .../ViewerDataService.AzureMasterScope.cs | 7 +- .../ViewerDataService.Overview.cs | 28 ++- 7 files changed, 375 insertions(+), 42 deletions(-) create mode 100644 Darling/Darling.Tests/BlockingBandReportUnitLiveTests.cs diff --git a/Darling/Darling.Tests/BlockingBandReportUnitLiveTests.cs b/Darling/Darling.Tests/BlockingBandReportUnitLiveTests.cs new file mode 100644 index 0000000000..e4fb072b97 --- /dev/null +++ b/Darling/Darling.Tests/BlockingBandReportUnitLiveTests.cs @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// #5644 review of W1: the displayed blocking count is WAITS, but the health band's per-hour count tiers +/// (ServerHealthThresholds.BlockingWarnPerHour 5 / BlockingCriticalPerHour 20) were measured in +/// blocked-process REPORTS per server-hour, so the band reads the report count. One wait that SQL Server reported +/// 22 times in the last ten minutes (short waits, so the 10 s / 60 s wait arms stay out of it) is ONE wait on the +/// card and 22 reports for the band: Critical by count, as it was before W1. Read through the fleet reader and the +/// viewer's per-server summary. Gated on DARLING_TEST_PG. +/// +/* #1776 own-store: every fact plants its rows under one dedicated server id and deletes them in cleanup. */ +[Collection("live-postgres")] +public sealed class BlockingBandReportUnitLiveTests +{ + private const string ServerName = "darling-blocking-band-report-unit"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + + private const int Reports = 22; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private static async Task ExecAsync(NpgsqlConnection c, string sql, CancellationToken ct, params object[] p) + { + using var cmd = new NpgsqlCommand(sql, c); + foreach (var v in p) + cmd.Parameters.AddWithValue(v); + await cmd.ExecuteNonQueryAsync(ct); + } + + /* One wait reported 22 times, 100 ms apart: the k-th report carries wait_time_ms = k * 100 and is written at + start + wait, so every report derives the same wait start (the rule BlockingWaits collapses on). The longest + wait is 2.2 s: under the 10 s wait arm, so only the count arm can band it. */ + private static async Task PlantAsync(NpgsqlConnection c, bool dmv, CancellationToken ct) + { + var start = DateTime.SpecifyKind(DateTime.UtcNow.AddMinutes(-10), DateTimeKind.Unspecified); + for (var k = 1; k <= Reports; k++) + { + var at = start.AddMilliseconds(100 * k); + if (dmv) + await ExecAsync(c, "INSERT INTO dmv_blocking_snapshots (collection_id, collection_time, server_id, server_name, event_time, database_name, blocked_spid, blocking_spid, wait_time_ms, blocking_status) VALUES ($1,$2,$3,$4,$2,'db',52,51,$5,'suspended')", + ct, CollectionIdGenerator.Next(), at, ServerId, ServerName, (long)(100 * k)); + else + await ExecAsync(c, "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, wait_time_ms, blocking_spid, blocked_spid, blocking_status, database_name) VALUES ($1,$2,$3,$4,$2,$5,51,52,'suspended','db')", + ct, CollectionIdGenerator.Next(), at, ServerId, ServerName, (long)(100 * k)); + } + } + + private static async Task OpenSeededAsync(bool dmv, CancellationToken ct) + { + var connection = new NpgsqlConnection(ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await ExecAsync(connection, @" +INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_major_version, created_date, modified_date) +VALUES ($1, $2, $2, TRUE, 16, now()::timestamp, now()::timestamp) +ON CONFLICT (server_id) DO UPDATE SET is_enabled = TRUE", ct, ServerId, ServerName); + await PlantAsync(connection, dmv, ct); + return connection; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task TheFleetCard_ShowsOneWait_AndBandsOnTheReports(bool dmv) + { + Assert.SkipWhen(string.IsNullOrEmpty(ConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the live test."); + var ct = TestContext.Current.CancellationToken; + var bodySucceeded = false; + try + { + await using var connection = await OpenSeededAsync(dmv, ct); + await using var postgres = NpgsqlDataSource.Create(ConnectionString!); + var now = DateTime.UtcNow; + + var result = await DarlingFleetReader.GetFleetOverviewAsync( + postgres, now.AddHours(-1), now, now, cancellationToken: ct); + + var card = result.Cards.Single(c => c.ServerId == ServerId); + Assert.Equal(1, card.BlockingCount); // displayed: ONE wait + Assert.Equal(Reports, card.BlockingReportCount); // the band's unit + Assert.Equal((double)Reports, card.BlockingRatePerHour); // reports per hour + Assert.Equal(HealthSeverity.Critical, card.BlockingSeverity); + Assert.True(card.MaxBlockingWaitMs < 10_000, "the wait arm must not be what bands this card"); + /* The re-band the fleet ranking runs reads the same report count. */ + Assert.Equal(Reports, card.ToHealthMetrics().BlockingCount); + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(ConnectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task TheViewerSummary_ShowsOneWait_AndBandsOnTheReports(bool dmv) + { + Assert.SkipWhen(string.IsNullOrEmpty(ConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the live test."); + var ct = TestContext.Current.CancellationToken; + var bodySucceeded = false; + try + { + await using var connection = await OpenSeededAsync(dmv, ct); + await using var viewer = new ViewerDataService(ConnectionString!); + + var summary = await viewer.GetServerSummaryAsync(ServerId, ServerName, null, ct); + + Assert.Equal(1, summary.BlockingCount); + Assert.Equal(Reports, summary.BlockingReportCount); + Assert.Equal(Reports, summary.BlockingCountForBand); + Assert.Equal((double)Reports, summary.BlockingRatePerHour); + Assert.Equal(HealthSeverity.Critical, summary.BlockingSeverity); + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(ConnectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + /// A card or summary built without a report count (the pre-W1 shape) bands on its own count. + [Fact] + public void ACardBuiltWithoutAReportCount_BandsOnItsCount() + { + var summary = new ServerSummaryItem { BlockingCount = 7, BlockingWindow = TimeSpan.FromHours(1) }; + Assert.Equal(7, summary.BlockingCountForBand); + + var withReports = new ServerSummaryItem { BlockingCount = 1, BlockingReportCount = 30, BlockingWindow = TimeSpan.FromHours(1) }; + Assert.Equal(30, withReports.BlockingCountForBand); + Assert.Equal(HealthSeverity.Critical, withReports.BlockingSeverity); + } + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var cleanup = new NpgsqlCommand( + $"DELETE FROM blocked_process_reports WHERE server_id = {ServerId}; " + + $"DELETE FROM dmv_blocking_snapshots WHERE server_id = {ServerId}; " + + $"DELETE FROM servers WHERE server_id = {ServerId};", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/DarlingWebEndpointsTests.cs b/Darling/Darling.Tests/DarlingWebEndpointsTests.cs index 8e1381ccf2..daa7e2e206 100644 --- a/Darling/Darling.Tests/DarlingWebEndpointsTests.cs +++ b/Darling/Darling.Tests/DarlingWebEndpointsTests.cs @@ -877,6 +877,71 @@ public void MapAll_BuildsTheAnalysisServiceWithTheHostsLogger() Assert.Contains("logger: logger", line, StringComparison.Ordinal); } + /// + /// #5644 (review of W22): /api/fleet refuses an out-of-range window or a negative worst_count with the + /// {"status":"invalid"} envelope, naming the spelling the caller sent, and still answers the in-range requests + /// (the bare route, hours=24, the edges 1 and 168, worst_count=0) with no refusal. It used to clamp + /// (hours=500 answered 200 over 168 hours, worst_count=-3 over zero). + /// + [Theory] + [InlineData("hours", 500, "hours")] + [InlineData("hours", 0, "hours")] + [InlineData("hours", -5, "hours")] + [InlineData("hours", 169, "hours")] + [InlineData("hours_back", 500, "hours_back")] + [InlineData("hours_back", 0, "hours_back")] + public void FleetParamRefusal_RefusesAnOutOfRangeWindow_NamingTheParameterSent(string key, int value, string expectedParameter) + { + var refusal = DarlingWebEndpoints.FleetParamRefusal(value, 3, k => k == key ? value.ToString() : null); + + Assert.NotNull(refusal); + Assert.True(McpHelpers.IsRefusalEnvelope(refusal)); + var envelope = JsonNode.Parse(refusal)!; + Assert.Equal("invalid", (string?)envelope["status"]); + Assert.Equal(expectedParameter, (string?)envelope["hints"]!["parameter"]); + } + + [Theory] + [InlineData(-1)] + [InlineData(-3)] + public void FleetParamRefusal_RefusesANegativeWorstCount(int worstCount) + { + var refusal = DarlingWebEndpoints.FleetParamRefusal(24, worstCount, k => k == "worst_count" ? worstCount.ToString() : null); + + Assert.NotNull(refusal); + var envelope = JsonNode.Parse(refusal)!; + Assert.Equal("invalid", (string?)envelope["status"]); + Assert.Equal("worst_count", (string?)envelope["hints"]!["parameter"]); + } + + [Theory] + [InlineData(1, 0)] + [InlineData(1, 5)] + [InlineData(24, 5)] + [InlineData(168, 5)] + public void FleetParamRefusal_AnswersInRangeRequests(int hours, int worstCount) + { + Assert.Null(DarlingWebEndpoints.FleetParamRefusal(hours, worstCount, k => k == "hours" ? hours.ToString() : null)); + } + + /// The /api/fleet route runs that refusal before it reads, and no longer clamps the window or the count. + [Fact] + public void FleetRoute_RefusesBeforeReading_AndDoesNotClamp() + { + var source = ReadSource(WebEndpointsSourcePath); + var start = source.IndexOf("app.MapGet(\"/api/fleet\"", StringComparison.Ordinal); + Assert.True(start >= 0, "the /api/fleet route has moved; update this pin."); + var end = source.IndexOf("app.MapGet(", start + 10, StringComparison.Ordinal); + var route = source[start..end]; + + Assert.Contains("FleetParamRefusal(", route, StringComparison.Ordinal); + Assert.DoesNotContain("Math.Clamp", route, StringComparison.Ordinal); + Assert.DoesNotContain("Math.Max", route, StringComparison.Ordinal); + Assert.True( + route.IndexOf("FleetParamRefusal(", StringComparison.Ordinal) < route.IndexOf("GetFleetOverviewAsync", StringComparison.Ordinal), + "the refusal must come before the read"); + } + private static string ReadSource(string relative) { var path = Path.Combine(RepoRoot(), relative); diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs index d65281769d..1d8b39c357 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs @@ -299,7 +299,10 @@ GROUP BY bucket_start MAX(wait_time_ms) AS max_wait_time_ms, COUNT(DISTINCT blocking_spid) AS distinct_head_blockers, COUNT(CASE WHEN blocking_status = 'sleeping' THEN 1 END) AS sleeping_blocker_count, - (SELECT COALESCE(MAX(bucket_event_count), 0) FROM buckets) AS peak_4h_event_count + (SELECT COALESCE(MAX(bucket_event_count), 0) FROM buckets) AS peak_4h_event_count, + /* #5644: the report rows before they collapse to waits, the unit the health band's per-hour count tiers were + measured in. Appended last so every ordinal above is where it was; the fleet tile and the viewer band read it. */ + (SELECT COUNT(*) FROM waits_marked) AS report_count FROM reports"; public static string BlockingSql => BuildBlockingSql(@" diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs index f71257aa29..3ed9f58464 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs @@ -349,8 +349,17 @@ makes it the only one that can answer when the store IS the problem — so it re return ToHttpResult(fleetRefusal, DarlingWebFailureLog.RouteOf(context.Request), logger, 0, context.RequestAborted); } - var hours = Math.Clamp(QueryInt(context, "hours_back", "hours", DefaultFleetHours), 1, 168); - var worstCount = Math.Max(0, QueryInt(context, "worst_count", null, DarlingFleetReader.DefaultWorstCount)); + /* #5644 (review of W22): an out-of-range window or count is refused with the {"status":"invalid"} 400 + the read loop answers, never clamped to a different window and answered 200 (REVIEW-TRAPS: counts + are refused, not clamped). The refusal names the spelling the caller sent (hours or hours_back). + The in-product callers send neither (apiGetFleet asks for the bare /api/fleet, the one-hour default). */ + var hours = QueryInt(context, "hours_back", "hours", DefaultFleetHours); + var worstCount = QueryInt(context, "worst_count", null, DarlingFleetReader.DefaultWorstCount); + if (FleetParamRefusal(hours, worstCount, key => First(context, key)) is { } fleetRangeRefusal) + { + return ToHttpResult(fleetRangeRefusal, DarlingWebFailureLog.RouteOf(context.Request), logger, 0, context.RequestAborted); + } + var now = DateTime.UtcNow; var result = await DarlingFleetReader.GetFleetOverviewAsync( postgres, now.AddHours(-hours), now, now, worstCount, @@ -6508,6 +6517,25 @@ internal static string NameWindowParameterTheCallerSent(string result, Func + /// #5644: the refusal envelope for an out-of-range /api/fleet window ( outside 1..168) or a + /// negative , else null. Same envelope and sentences as the MCP tool's + /// , with the window parameter named as the caller sent it + /// (). Pure over the lookup, so a test pins it without a server. + /// + internal static string? FleetParamRefusal(int hours, int worstCount, Func lookup) + { + var window = McpHelpers.ValidateHoursBack(hours); + if (window is not null) + { + return NameWindowParameterTheCallerSent(window, lookup); + } + + return worstCount < 0 + ? McpHelpers.Refusal("worst_count", $"Invalid worst_count value '{worstCount}'. Must be zero or a positive integer.") + : null; + } + private static string? NonIntegerParamRefusal(HttpContext context) => NonIntegerParamRefusal(key => First(context, key)); @@ -6517,7 +6545,7 @@ private static int QueryInt(HttpContext context, string key, string? aliasKey, i /// /// A row-count knob (?limit= / ?top=), clamped to [1, ] at the /// dispatch layer so an authenticated/loopback caller can't request an unbounded result set from a reader - /// that binds the value straight into LIMIT $N (the /api/fleet window is clamped the same way). + /// that binds the value straight into LIMIT $N (the /api/fleet window is refused out of range instead, #5644). /// private static int Rows(HttpContext context, string key, int def) => ClampRows(QueryInt(context, key, null, def)); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index b5156f1cf5..6813a5d6c1 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -334,49 +334,65 @@ WHERE s.is_enabled AND {SqlServerCollectedTargetSql}"; /// Blocking in the window per server from BOTH sources — XE blocked-process reports and the always-on - /// DMV snapshot, each counted per server_id with its worst wait, lined up by a FULL OUTER JOIN so a + /// DMV snapshot, each counted per server_id as WAITS with its worst wait (and, for the health band, as + /// the raw report rows: xe_reports / dmv_reports, #5644), lined up by a FULL OUTER JOIN so a /// server present in only one source still appears. The caller applies Lite's XE-preferred / DMV-fallback per /// server. $1 window start, $2 window end, $3 the for $1 (all naive UTC). /// /// $3 is the partition-column bound the event window cannot supply (#3895): without it both scans /// open every retained chunk to count the last hour. It cannot drop a row the window wants — /// says why. - /* #5644 W1: each side counts WAITS (a blocked session, its blocker and the wait's derived start), not report - rows, with the longest wait time as its max: SQL Server writes a report for a still-blocked session every - threshold seconds, so one one-minute wait was about eleven tile counts. BlockingWaits.WaitsCteSql is the - one definition the alert engine and the analysis fact share. The DMV side applies the same rule to its - snapshot rows (a wait seen in several snapshots is one wait). */ + /* #5644 W1: xe_count / dmv_count (and the max waits) count WAITS (a blocked session, its blocker and the wait's + derived start), not report rows: SQL Server writes a report for a still-blocked session every threshold + seconds, so one one-minute wait was about eleven tile counts. BlockingWaits.WaitsCteSql is the one definition + the alert engine and the analysis fact share. The DMV side applies the same rule to its snapshot rows (a wait + seen in several snapshots is one wait). + + #5644 (review): xe_reports / dmv_reports keep the raw ROW count of the same two windowed reads. The health + band's count tiers (5 / 20 per hour) were measured in blocked-process REPORTS per server-hour, so the band + reads those; the tile, the alert value and the MCP counts stay waits. Each source is read once (the + *_reports CTE, shared by the report count and the waits chain), so the window and the floor are still + applied once per source. */ public static readonly string FleetBlockingSql = @" -WITH " + BlockingWaits.WaitsCteSql( - BlockingWaits.SqlDialect.Postgres, "xe_waits", - @"SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms +WITH xe_reports AS ( + SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms FROM v_blocked_process_reports WHERE event_time >= $1 AND event_time <= $2 - AND collection_time >= $3", new[] { "server_id" }) + "," + BlockingWaits.WaitsCteSql( - BlockingWaits.SqlDialect.Postgres, "dmv_waits", - @"SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms + AND collection_time >= $3 +), +dmv_reports AS ( + SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms FROM v_dmv_blocking_snapshots WHERE event_time >= $1 AND event_time <= $2 - AND collection_time >= $3", new[] { "server_id" }) + @" + AND collection_time >= $3 +)," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "xe_waits", + "SELECT * FROM xe_reports", new[] { "server_id" }) + "," + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "dmv_waits", + "SELECT * FROM dmv_reports", new[] { "server_id" }) + @" SELECT COALESCE(xe.server_id, dmv.server_id) AS server_id, COALESCE(xe.cnt, 0) AS xe_count, COALESCE(xe.max_wait, 0) AS xe_max_wait, COALESCE(dmv.cnt, 0) AS dmv_count, - COALESCE(dmv.max_wait, 0) AS dmv_max_wait + COALESCE(dmv.max_wait, 0) AS dmv_max_wait, + COALESCE(xe.reports, 0) AS xe_reports, + COALESCE(dmv.reports, 0) AS dmv_reports FROM ( - SELECT server_id, COUNT(*) AS cnt, MAX(wait_time_ms) AS max_wait - FROM xe_waits - GROUP BY server_id + SELECT r.server_id, r.reports, w.cnt, w.max_wait + FROM (SELECT server_id, COUNT(*) AS reports FROM xe_reports GROUP BY server_id) AS r + LEFT JOIN (SELECT server_id, COUNT(*) AS cnt, MAX(wait_time_ms) AS max_wait FROM xe_waits GROUP BY server_id) AS w + ON r.server_id = w.server_id ) AS xe FULL OUTER JOIN ( - SELECT server_id, COUNT(*) AS cnt, MAX(wait_time_ms) AS max_wait - FROM dmv_waits - GROUP BY server_id + SELECT r.server_id, r.reports, w.cnt, w.max_wait + FROM (SELECT server_id, COUNT(*) AS reports FROM dmv_reports GROUP BY server_id) AS r + LEFT JOIN (SELECT server_id, COUNT(*) AS cnt, MAX(wait_time_ms) AS max_wait FROM dmv_waits GROUP BY server_id) AS w + ON r.server_id = w.server_id ) AS dmv ON xe.server_id = dmv.server_id"; /// Deadlocks in the window per server — count and newest deadlock instant (for each card's @@ -986,6 +1002,14 @@ internal static FleetServerCard BuildCard( snapshot — both count and worst-wait come from whichever source wins. */ var blockingCount = blocking.XeCount > 0 ? blocking.XeCount : blocking.DmvCount; var maxBlockingWaitMs = blocking.XeCount > 0 ? blocking.XeMaxWait : blocking.DmvMaxWait; + /* #5644 (review of W1): the count the BAND reads is the winning source's REPORT count, not blockingCount + (waits, which the card shows). ServerHealthThresholds.BlockingWarnPerHour / BlockingCriticalPerHour + were measured in blocked-process reports per server-hour; a long wait is many reports and one wait, + and handing the waits to those tiers stopped it banding by count. A row built without a report count + (the pre-W1 shape a test constructs) reads its own count, which is what it always banded on. */ + var blockingReportCount = blocking.XeCount > 0 + ? blocking.XeReports ?? blocking.XeCount + : blocking.DmvReports ?? blocking.DmvCount; var cpuPercent = cpu.SqlCpu; var otherCpu = cpu.OtherCpu; @@ -1042,7 +1066,7 @@ deadlock_source and the coverage block disclosing the gap. The PostgreSQL arm is computed from an empty set. That is also what keeps #3539 A6's card - an online PostgreSQL target nothing has collected from - measuring nothing. */ var memoryPressureForBand = ServerMetricSources.DmvSourced(hasMemoryPressure, isPostgres); - var blockingForBand = ServerMetricSources.DmvSourced(blockingCount, isPostgres); + var blockingForBand = ServerMetricSources.DmvSourced(blockingReportCount, isPostgres); int? deadlocksForBand = deadlockMeasured ? deadlockCount : null; var metrics = new ServerHealthMetrics @@ -1139,7 +1163,9 @@ deliberately not surfaced. */ MaxBlockingWaitMs = maxBlockingWaitMs, /* blockingForBand, not the raw count, for the reason DeadlockRatePerHour below gives: a PostgreSQL target's raw count is a structural zero, and a rate derived from it would publish - 0.0/hr against a severity that reads Unknown (#3539 A3). */ + 0.0/hr against a severity that reads Unknown (#3539 A3). It is REPORTS per hour, the unit the + band's tiers were measured in (#5644), beside a count that is waits. */ + BlockingReportCount = blockingReportCount, BlockingRatePerHour = blockingForBand.HasValue ? ServerHealthClassifier.BlockingRatePerHour(blockingForBand.Value, deadlockWindow) : null, @@ -1632,7 +1658,12 @@ private static async Task ScopeAzureMasterRowsAsync( postgres, server.ServerId, startUtc, endUtc, separate, cancellationToken); blocking.TryGetValue(server.ServerId, out var current); - blocking[server.ServerId] = current with { XeCount = scoped.XeCount, XeMaxWait = scoped.XeMaxWait }; + blocking[server.ServerId] = current with + { + XeCount = scoped.XeCount, + XeMaxWait = scoped.XeMaxWait, + XeReports = scoped.XeReports, + }; deadlocks.TryGetValue(server.ServerId, out var currentDeadlock); deadlocks[server.ServerId] = currentDeadlock with { @@ -1653,8 +1684,10 @@ private static async Task ScopeAzureMasterRowsAsync( /// .., skipping the events of the databases in /// . is the newest of the deadlocks that count, or /// null when none does. + /// counts waits; is the report-row count of the same + /// scoped read, the band's input (#5644). internal readonly record struct AzureMasterScopedCounts( - int XeCount, long XeMaxWait, int DeadlockCount, DateTime? DeadlockLastSeen); + int XeCount, long XeMaxWait, int DeadlockCount, DateTime? DeadlockLastSeen, int XeReports = 0); /// The scoped reads behind , shared with the per-server summary /// (get_server_summary) so the two surfaces count a master's events with the same statements. @@ -1669,6 +1702,7 @@ internal static async Task ReadAzureMasterScopedCountsA await using var connection = await postgres.OpenConnectionAsync(cancellationToken); var xeCount = 0; + var xeReports = 0; long xeMaxWait = 0; await using (var command = new NpgsqlCommand(PgFactCollector.BlockingSqlSkippingSeparate, connection) { CommandTimeout = McpCommandDeadlines.ReadSeconds }) @@ -1683,12 +1717,13 @@ internal static async Task ReadAzureMasterScopedCountsA { xeCount = reader.IsDBNull(0) ? 0 : (int)Math.Min(Convert.ToInt64(reader.GetValue(0)), int.MaxValue); xeMaxWait = reader.IsDBNull(2) ? 0 : Convert.ToInt64(reader.GetValue(2)); + xeReports = reader.IsDBNull(6) ? 0 : (int)Math.Min(Convert.ToInt64(reader.GetValue(6)), int.MaxValue); } } var (deadlockCount, deadlockLastSeen) = await PgFactCollector.CountAndNewestDeadlocksSkippingSeparateAsync( connection, serverId, startUtc, endUtc, separate, cancellationToken, McpCommandDeadlines.ReadSeconds); - return new AzureMasterScopedCounts(xeCount, xeMaxWait, (int)Math.Min(deadlockCount, int.MaxValue), deadlockLastSeen); + return new AzureMasterScopedCounts(xeCount, xeMaxWait, (int)Math.Min(deadlockCount, int.MaxValue), deadlockLastSeen, xeReports); } private static async Task> ReadBlockingAsync( @@ -1707,7 +1742,9 @@ private static async Task> ReadBlockingAsync( reader.IsDBNull(1) ? 0 : Convert.ToInt32(reader.GetValue(1)), reader.IsDBNull(2) ? 0 : Convert.ToInt64(reader.GetValue(2)), reader.IsDBNull(3) ? 0 : Convert.ToInt32(reader.GetValue(3)), - reader.IsDBNull(4) ? 0 : Convert.ToInt64(reader.GetValue(4))); + reader.IsDBNull(4) ? 0 : Convert.ToInt64(reader.GetValue(4)), + reader.IsDBNull(5) ? 0 : Convert.ToInt32(reader.GetValue(5)), + reader.IsDBNull(6) ? 0 : Convert.ToInt32(reader.GetValue(6))); } return map; @@ -2099,7 +2136,11 @@ internal readonly record struct PgCpuRow( internal readonly record struct MemoryRow(double? MemoryMb, double? BufferPoolMb); internal readonly record struct MemoryPressureRow(long WaiterCount, long TimeoutCount, long ForcedCount, double? GrantedMemoryMb); internal readonly record struct ThreadsRow(int? TotalThreads, int? CurrentWorkers, int RunnableTasks, long WorkQueue); - internal readonly record struct BlockingRow(int XeCount, long XeMaxWait, int DmvCount, long DmvMaxWait); + /// One server's blocking in the window: XeCount / DmvCount are WAITS (what the card shows), + /// XeReports / DmvReports the raw report / snapshot rows the band's per-hour tiers were measured in + /// (#5644). Null reports mean "not read separately": the band reads the count. + internal readonly record struct BlockingRow( + int XeCount, long XeMaxWait, int DmvCount, long DmvMaxWait, int? XeReports = null, int? DmvReports = null); internal readonly record struct DeadlockRow(int Count, DateTime? LastSeen); /// One server's row (#3935): its newest collection inside the @@ -2501,7 +2542,20 @@ public sealed class FleetServerCard [JsonPropertyName("has_memory_pressure")] public bool HasMemoryPressure { get; init; } [JsonPropertyName("memory_severity")] public HealthSeverity MemorySeverity { get; init; } - [JsonPropertyName("blocking_count")] public int BlockingCount { get; init; } + /// Blocking WAITS in the window (#5644 W1: a blocked session, its blocker and the wait's start): what the + /// card, the totals and the MCP counts show. The band does NOT read this; it reads + /// , the unit its per-hour tiers were measured in. + [JsonPropertyName("blocking_count")] + public int BlockingCount { get; init; } + + /// The winning source's blocked-process REPORT (or DMV snapshot) rows in the window, before they are + /// collapsed to waits (#5644): the count blocking_severity and blocking_rate_per_hour band on, + /// because ServerHealthThresholds.BlockingWarnPerHour / BlockingCriticalPerHour were measured in + /// reports per server-hour. Not published (the contract is the wait count above); null on a card built + /// without one, which re-bands on . + [JsonIgnore] + public int? BlockingReportCount { get; init; } + [JsonPropertyName("max_blocking_wait_ms")] public long MaxBlockingWaitMs { get; init; } /// Blocking events per HOUR over the card's window — the figure the count arm of /// blocking_severity banded on (#3539 A3), or null when the window was too short to normalise. @@ -2651,7 +2705,8 @@ Insights arm and the source is what decides which. Omitting them here would leav deliberately left as zeros (#3017) — reading them here would hand the ranking a measurement the card's own severity says it does not have. */ HasMemoryPressure = ServerMetricSources.DmvSourced(HasMemoryPressure, IsPostgres), - BlockingCount = ServerMetricSources.DmvSourced(BlockingCount, IsPostgres), + /* The REPORT count the card banded on (#5644), not the wait count it shows. */ + BlockingCount = ServerMetricSources.DmvSourced(BlockingReportCount ?? BlockingCount, IsPostgres), MaxBlockedSeconds = MaxBlockingWaitMs / 1000.0, /* #3539 A3: the count's denominator, for the same reason the deadlock window travels below. */ BlockingWindow = BlockingWindow, diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AzureMasterScope.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AzureMasterScope.cs index 9b64ec35ff..6c723e68db 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AzureMasterScope.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AzureMasterScope.cs @@ -150,7 +150,7 @@ private static IReadOnlyList SeparatelyMonitoredFrom( /// The master's windowed XE blocking count and worst wait without the separately monitored databases' /// events (, the read the service's analysis uses). /// - private async Task<(int Count, long MaxWaitMs)> ReadScopedBlockingAsync( + private async Task<(int Count, long MaxWaitMs, int Reports)> ReadScopedBlockingAsync( int serverId, DateTime start, DateTime end, IReadOnlyList separate, CancellationToken cancellationToken) { await ScopeReadStageAsync("blocking"); @@ -164,12 +164,13 @@ private static IReadOnlyList SeparatelyMonitoredFrom( await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (!await reader.ReadAsync(cancellationToken)) { - return (0, 0); + return (0, 0, 0); } return ( reader.IsDBNull(0) ? 0 : Convert.ToInt32(reader.GetValue(0)), - reader.IsDBNull(2) ? 0L : Convert.ToInt64(reader.GetValue(2))); + reader.IsDBNull(2) ? 0L : Convert.ToInt64(reader.GetValue(2)), + reader.IsDBNull(6) ? 0 : Convert.ToInt32(reader.GetValue(6))); } /// The master's windowed deadlock count without the deadlocks wholly in the separately monitored databases. diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs index b3e6ebbb16..b28b0dfbf1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs @@ -159,7 +159,9 @@ WITH newest AS /// and probe the window's chunks instead of every retained one. The two "ever" reads keep no bound: /// they ask for the newest event in all of history, which a floor would change the answer to. /// - /* #5644 W1: the two counts are WAITS (BlockingWaits.WaitsCteSql), not report rows, as the fleet tile's are. */ + /* #5644 W1: the two counts are WAITS (BlockingWaits.WaitsCteSql), not report rows, as the fleet tile's are. + The last two columns are the raw report / snapshot rows (the CTE's *_marked relation holds every source row): + the unit the health band's per-hour count tiers were measured in, so the band reads them (#5644 review). */ public static readonly string ServerSummaryBlockingSql = @" WITH " + BlockingWaits.WaitsCteSql( BlockingWaits.SqlDialect.Postgres, "xe_waits", @@ -174,7 +176,9 @@ WITH newest AS (SELECT COUNT(*) FROM dmv_waits), (SELECT MAX(wait_time_ms) FROM dmv_waits), (SELECT MAX(event_time) FROM v_blocked_process_reports WHERE server_id = $1), - (SELECT MAX(event_time) FROM v_dmv_blocking_snapshots WHERE server_id = $1)"; + (SELECT MAX(event_time) FROM v_dmv_blocking_snapshots WHERE server_id = $1), + (SELECT COUNT(*) FROM xe_waits_marked), + (SELECT COUNT(*) FROM dmv_waits_marked)"; /// /// Deadlock count in the window plus the newest deadlock ever — the windowed count for the card value, @@ -287,6 +291,7 @@ expressions of "one hour" is how a denominator drifts away from its numerator. * double? memoryMb = null; double? bufferPoolMb = null; var blockingCount = 0; + var blockingReportCount = 0; long maxBlockingWaitMs = 0; int? lastBlockingMinutesAgo = null; var deadlockCount = 0; @@ -403,13 +408,15 @@ Unknown rather than Healthy. */ { var xeCount = reader.IsDBNull(0) ? 0 : Convert.ToInt32(reader.GetValue(0)); var xeMaxWait = reader.IsDBNull(1) ? 0L : Convert.ToInt64(reader.GetValue(1)); + var xeReports = reader.IsDBNull(6) ? 0 : Convert.ToInt32(reader.GetValue(6)); + var dmvReports = reader.IsDBNull(7) ? 0 : Convert.ToInt32(reader.GetValue(7)); if (separate.Count > 0) { /* The XE arm without the separately monitored databases' reports; the DMV arm and the XE-then-DMV fallback below are unchanged. */ try { - (xeCount, xeMaxWait) = await ReadScopedBlockingAsync(serverId, windowStart, scopeEnd, separate, cancellationToken); + (xeCount, xeMaxWait, xeReports) = await ReadScopedBlockingAsync(serverId, windowStart, scopeEnd, separate, cancellationToken); } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -426,11 +433,13 @@ is better than a card with no blocking figure. */ if (xeCount > 0) { blockingCount = xeCount; + blockingReportCount = xeReports; maxBlockingWaitMs = xeMaxWait; } else { blockingCount = dmvCount; + blockingReportCount = dmvReports; maxBlockingWaitMs = dmvMaxWait; } @@ -544,6 +553,7 @@ know the engine (the loader stamps IsPostgres afterwards), and one arm is always MemoryTimeoutCount = memoryTimeoutCount, MemoryForcedCount = memoryForcedCount, BlockingCount = blockingCount, + BlockingReportCount = blockingReportCount, MaxBlockingWaitMs = maxBlockingWaitMs, LastBlockingMinutesAgo = lastBlockingMinutesAgo, DeadlockCount = deadlockCount, @@ -1186,9 +1196,15 @@ decision the service's fleet card uses so the two cannot disagree about whether /// semaphore to read (every PostgreSQL target). public bool? MemoryPressureForBand => ServerMetricSources.DmvSourced(HasMemoryPressure, IsPostgres); - /// Blocking events as a BANDABLE reading — null when this card reads no blocking source for - /// this engine. - public int? BlockingCountForBand => ServerMetricSources.DmvSourced(BlockingCount, IsPostgres); + /// Blocking as a BANDABLE reading — null when this card reads no blocking source for this engine. + /// It is the REPORT count (), not the wait count the card shows + /// (, #5644 W1): the band's per-hour count tiers (5 / 20) were measured in blocked-process + /// reports per server-hour, and a long wait is many reports and one wait. + public int? BlockingCountForBand => ServerMetricSources.DmvSourced(BlockingReportCount ?? BlockingCount, IsPostgres); + + /// The winning source's blocked-process report (or DMV snapshot) rows in the window, before they collapse + /// to waits (#5644). Set by the read; null on a card built without one, which bands on . + public int? BlockingReportCount { get; set; } /// Deadlocks as a BANDABLE reading (#3539). On a SQL Server card it is the count itself — a /// graph count is an observation even at zero (#3272's engine-not-collector rule). On a PostgreSQL card From d47ce50b54a1894035e79e147713683b7a1ea108 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:54:38 -0400 Subject: [PATCH 40/42] Band the daily summary and the fleet sweep on blocked-process report counts; displayed counts stay waits (#5644) --- .../DailySummaryBlockingWaitsLiveTests.cs | 94 ++++++++++++++++--- .../Darling.Tests/FleetSweepEngineTests.cs | 32 +++++++ .../ViewerCalendarRetentionPortTests.cs | 5 +- .../Darling.Tests/ViewerDailyHealthTests.cs | 2 +- .../FleetSweepEngine.cs | 18 +++- .../Mcp/DarlingHealthReader.cs | 9 ++ .../DailySummaryAzureMasterScope.cs | 20 ++++ .../DailySummarySql.cs | 18 +++- .../ViewerDataService.DailySummary.cs | 11 +++ Lite.Tests/BlockingWaitDuckDbTests.cs | 42 ++++++++- Lite.Tests/PerformanceCalendarDataTests.cs | 4 +- .../Services/LocalDataService.DailySummary.cs | 25 ++++- PerformanceMonitor.Common/BlockingWaits.cs | 17 +++- PerformanceMonitor.Common/DailyHealthBand.cs | 31 ++++-- 14 files changed, 283 insertions(+), 45 deletions(-) diff --git a/Darling/Darling.Tests/DailySummaryBlockingWaitsLiveTests.cs b/Darling/Darling.Tests/DailySummaryBlockingWaitsLiveTests.cs index 5eb1e7fa15..0975bc04fa 100644 --- a/Darling/Darling.Tests/DailySummaryBlockingWaitsLiveTests.cs +++ b/Darling/Darling.Tests/DailySummaryBlockingWaitsLiveTests.cs @@ -10,10 +10,13 @@ using System.Globalization; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; using Npgsql; using NpgsqlTypes; using PerformanceMonitor.Collectors; using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; using PerformanceMonitor.Darling.Storage; using Xunit; @@ -46,17 +49,17 @@ private static async Task ExecAsync(NpgsqlConnection c, string sql, Cancellation /* One wait as SQL Server reports it: a report every 5 s while the session stays blocked, the k-th carrying wait_time_ms = k * 5 s. */ private static async Task PlantWaitAsync( - NpgsqlConnection c, DateTime start, int blocked, int blocker, int reports, string database, bool dmv, CancellationToken ct) + NpgsqlConnection c, DateTime start, int blocked, int blocker, int reports, string database, bool dmv, CancellationToken ct, int spacingMs = 5000) { for (var k = 1; k <= reports; k++) { - var at = start.AddSeconds(5 * k); + var at = start.AddMilliseconds(spacingMs * k); if (dmv) await ExecAsync(c, "INSERT INTO dmv_blocking_snapshots (collection_id, collection_time, server_id, server_name, event_time, database_name, blocked_spid, blocking_spid, wait_time_ms, blocking_status) VALUES ($1,$2,$3,$4,$2,$5,$6,$7,$8,'suspended')", - ct, CollectionIdGenerator.Next(), at, ServerId, ServerName, database, blocked, blocker, (long)(5000 * k)); + ct, CollectionIdGenerator.Next(), at, ServerId, ServerName, database, blocked, blocker, (long)(spacingMs * k)); else await ExecAsync(c, "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, wait_time_ms, blocking_spid, blocked_spid, blocking_status, database_name) VALUES ($1,$2,$3,$4,$2,$5,$6,$7,'suspended',$8)", - ct, CollectionIdGenerator.Next(), at, ServerId, ServerName, (long)(5000 * k), blocker, blocked, database); + ct, CollectionIdGenerator.Next(), at, ServerId, ServerName, (long)(spacingMs * k), blocker, blocked, database); } } @@ -64,7 +67,7 @@ await ExecAsync(c, "INSERT INTO blocked_process_reports (blocked_report_id, coll /// 2 waits, on a quiet UTC day (as DMV snapshots when ), then reads that day's /// blocking_events through the statement, or the master-scoped one with the sibling list at $5 when /// . - private static async Task DayBlockingEventsAsync(bool dmv, bool scoped) + private static async Task<(long Events, long Reports)> DayBlockingCountsAsync(bool dmv, bool scoped) { var cs = ConnectionString; Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live test."); @@ -86,16 +89,19 @@ INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_major await PlantWaitAsync(connection, day.AddHours(3), blocked: 53, blocker: 51, reports: 6, "GP", dmv, ct); var sql = scoped ? DailySummaryAzureMasterScope.Scope(DailySummarySql.RangeSql) : DailySummarySql.RangeSql; - using var read = new NpgsqlCommand("SELECT blocking_events FROM (" + sql + ") AS calendar", connection); + using var read = new NpgsqlCommand("SELECT blocking_events, blocking_reports FROM (" + sql + ") AS calendar", connection); read.Parameters.Add(new NpgsqlParameter { TypedValue = ServerId }); read.Parameters.Add(new NpgsqlParameter { TypedValue = day }); read.Parameters.Add(new NpgsqlParameter { TypedValue = day.AddDays(1) }); read.Parameters.Add(new NpgsqlParameter { TypedValue = EventWindowFloor.For(day) }); if (scoped) read.Parameters.Add(new NpgsqlParameter { Value = new[] { "GP" }, NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Text }); - var count = Convert.ToInt64(await read.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); + using var reader = await read.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct)); + var counts = (Convert.ToInt64(reader.GetValue(0), CultureInfo.InvariantCulture), + Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); bodySucceeded = true; - return count; + return counts; } finally { @@ -107,20 +113,85 @@ await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) = [Fact] public async Task TheDay_CountsBlockedProcessWaits_NotReports() { - Assert.Equal(2, await DayBlockingEventsAsync(dmv: false, scoped: false)); + var (events, reports) = await DayBlockingCountsAsync(dmv: false, scoped: false); + Assert.Equal(2, events); + Assert.Equal(15, reports); // #5644: the band's input stays the report rows } [Fact] public async Task TheDay_CountsDmvSnapshotWaits_NotSnapshots_WhenThereAreNoReports() { - Assert.Equal(2, await DayBlockingEventsAsync(dmv: true, scoped: false)); + var (events, reports) = await DayBlockingCountsAsync(dmv: true, scoped: false); + Assert.Equal(2, events); + Assert.Equal(15, reports); // the DMV fallback's rows, as the band counted them before waits } [Fact] public async Task TheAzureMasterScope_StillSkipsTheSiblingDatabase_AndCountsWaits() { /* The GP wait belongs to a database monitored as its own target: the master's day keeps the one master wait. */ - Assert.Equal(1, await DayBlockingEventsAsync(dmv: false, scoped: true)); + var (events, reports) = await DayBlockingCountsAsync(dmv: false, scoped: true); + Assert.Equal(1, events); + Assert.Equal(9, reports); // the same scope filter reaches the report count + } + + /// #5644 review: one long wait written as 130 reports (a report every 50 ms, so the longest block is 6.5 s and + /// the wait arm stays out of it) is ONE blocking event, which every display and MCP count shows, but the band's count + /// tiers (5 and 20 per server-hour) were measured in reports, so the day bands Warning on 130 / 24 h = 5.4 per hour. + /// Read through the reader the calendar and the MCP tools use. On the waits-banded head the day read 1 event over 24 h + /// and came out Healthy. + [Fact] + public async Task TheBand_ReadsTheReportCount_WhileTheDisplayedCountStaysWaits() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live test."); + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(DarlingStoreConnection.PinSessionTimeZoneUtc(cs!)); + var bodySucceeded = false; + try + { + await ExecAsync(connection, @" +INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_major_version, created_date, modified_date) +VALUES ($1, $2, $2, TRUE, 16, now()::timestamp, now()::timestamp) +ON CONFLICT (server_id) DO UPDATE SET is_enabled = TRUE", ct, ServerId, ServerName); + + var day = DateTime.SpecifyKind(DateTime.UtcNow.Date.AddDays(-3), DateTimeKind.Unspecified); + await ExecAsync(connection, "INSERT INTO collection_log (log_id, server_id, server_name, collector_name, collection_time, status) VALUES ($1,$2,$3,'wait_stats',$4,'SUCCESS')", + ct, CollectionIdGenerator.Next(), ServerId, ServerName, day.AddHours(1)); + await PlantWaitAsync(connection, day.AddHours(2), blocked: 52, blocker: 51, reports: 130, "master", dmv: false, ct, spacingMs: 50); + + DarlingHealthReader.ResetRangeCacheForTests(); + var rows = (await DarlingHealthReader.GetDailySummaryRangeAsync( + postgres, ServerId, day, day.AddDays(1), separatelyMonitored: null, logger: NullLogger.Instance, cancellationToken: ct)).Rows; + var row = Assert.Single(rows); + + Assert.Equal(1, row.BlockingEvents); + Assert.Equal(130, row.BlockingReports); + Assert.Equal(6_500, row.MaxBlockDurationMs); + Assert.Equal(DailyHealthBand.Warning, row.HealthBand); + Assert.Equal(1, row.ToSignals().BlockingEvents); + Assert.Equal(130, row.ToSignals().BlockingReports); + + /* The fleet sweep reads the same rows (#5644): its signals carry the report count its blocking band reads, + and the wait count it shows. */ + var sweep = await FleetSweepEngine.ReadServerSignalsAsync(postgres, ServerId, ServerName, day, day.AddDays(1), ct); + Assert.Null(sweep.ReadFault); + Assert.Equal(1, sweep.Signals.BlockingEvents); + Assert.Equal(130, sweep.Signals.BlockingReports); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity( + sweep.Signals.BlockingReports, sweep.Signals.PeakBlockWaitMs / 1000.0, sweep.Signals.Window)); + bodySucceeded = true; + } + finally + { + DarlingHealthReader.ResetRangeCacheForTests(); + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } } private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) @@ -128,6 +199,7 @@ private static async Task DeleteRowsAsync(NpgsqlConnection connection, Cancellat using var cleanup = new NpgsqlCommand( $"DELETE FROM blocked_process_reports WHERE server_id = {ServerId}; " + $"DELETE FROM dmv_blocking_snapshots WHERE server_id = {ServerId}; " + + $"DELETE FROM collection_log WHERE server_id = {ServerId}; " + $"DELETE FROM servers WHERE server_id = {ServerId};", connection); await cleanup.ExecuteNonQueryAsync(ct); } diff --git a/Darling/Darling.Tests/FleetSweepEngineTests.cs b/Darling/Darling.Tests/FleetSweepEngineTests.cs index 4c5afe24e7..074680767b 100644 --- a/Darling/Darling.Tests/FleetSweepEngineTests.cs +++ b/Darling/Darling.Tests/FleetSweepEngineTests.cs @@ -613,6 +613,38 @@ public void TheBlockingFamily_FiresOnTheRateOrTheWaitArm_AndNamesWhich() Assert.Empty(dayComposition.WouldHavePaged); } + /// + /// #5644 review: the blocking family bands on the span's blocked-process REPORT count (the unit its 20/hr tier was + /// measured in), while the evidence's blocking_count stays the count of WAITS. One long wait written as 20 + /// reports in an hour pages as it did before waits were counted; the same wait with no report count, as 1 event, + /// does not. + /// + [Fact] + public void TheBlockingFamily_BandsOnReports_AndShowsWaits() + { + var oneLongWait = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + BlockingEvents = 1, + BlockingReports = 20, + Window = FixtureSpan, + }, 0, null); + var row = ComposeSimple(new[] { oneLongWait }, alertsEnabled: false) + .WouldHavePaged.Single(w => w.AlertFamily == FleetSweepEngine.FamilyBlocking); + using var evidence = JsonDocument.Parse(row.EvidenceJson); + Assert.Equal("blocked-process reports per hour over the sweep span", evidence.RootElement.GetProperty("trigger").GetString()); + Assert.Equal(20.0, evidence.RootElement.GetProperty("value").GetDouble()); + Assert.Equal(1, evidence.RootElement.GetProperty("blocking_count").GetInt64()); + + var withoutReports = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + BlockingEvents = 1, + Window = FixtureSpan, + }, 0, null); + Assert.Empty(ComposeSimple(new[] { withoutReports }, alertsEnabled: false).WouldHavePaged); + } + /// /// The tiers handed to Compose are the tiers the verdicts band on (#3525) — the store's pair /// travels into the shared scorer, so a fleet whose knobs were raised sweeps on the raised pair diff --git a/Darling/Darling.Tests/ViewerCalendarRetentionPortTests.cs b/Darling/Darling.Tests/ViewerCalendarRetentionPortTests.cs index 14a9c3c65c..1d198d9298 100644 --- a/Darling/Darling.Tests/ViewerCalendarRetentionPortTests.cs +++ b/Darling/Darling.Tests/ViewerCalendarRetentionPortTests.cs @@ -251,7 +251,10 @@ public void DailySummaryRetention_IsOneDefinition_ConsumedByBothReaders() /* The viewer reads the aggregate's LAST column for presence, at the ordinal the MCP reader reads. */ Assert.Contains("SignalSourcesPresent = reader.IsDBNull(13) ? 0 : Convert.ToInt32(reader.GetValue(13)),", viewer, StringComparison.Ordinal); Assert.Contains("SignalSourcesPresent = reader.IsDBNull(13) ? 0 : Convert.ToInt32(reader.GetValue(13)),", mcp, StringComparison.Ordinal); - Assert.EndsWith("AS signal_sources_present", DailySummarySql.RangeSql.Split("FROM day_spine")[0].TrimEnd(), StringComparison.Ordinal); + /* #5644: blocking_reports (the band's report count) was appended after it, at ordinal 14 and last. */ + var projection = DailySummarySql.RangeSql.Split("FROM day_spine")[0].TrimEnd(); + Assert.Contains("AS signal_sources_present,", projection, StringComparison.Ordinal); + Assert.EndsWith("AS blocking_reports", projection, StringComparison.Ordinal); } /// diff --git a/Darling/Darling.Tests/ViewerDailyHealthTests.cs b/Darling/Darling.Tests/ViewerDailyHealthTests.cs index 92f0cd878a..074f8ce735 100644 --- a/Darling/Darling.Tests/ViewerDailyHealthTests.cs +++ b/Darling/Darling.Tests/ViewerDailyHealthTests.cs @@ -574,7 +574,7 @@ trigger this replaced. */ Assert.Equal(2, summary!.BlockingEvents); Assert.Equal(0, summary.DeadlockCount); Assert.Equal("Healthy", summary.OverallHealth); - Assert.Contains("2 blocking events (0.1/hr)", summary.SignalsTooltip, StringComparison.Ordinal); + Assert.Contains("2 blocking events (0.1 reports/hr)", summary.SignalsTooltip, StringComparison.Ordinal); bodySucceeded = true; } diff --git a/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs b/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs index 70b3d5477a..2391c0fb4d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs +++ b/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs @@ -752,17 +752,21 @@ wait row names the wait bar. */ /* The SIGNALS' peak, not the reading's: Classify sees only the signals, and the two must agree. The production read fills both from one MAX. */ var peakBlockSeconds = signals.PeakBlockWaitMs / 1000.0; - if (ServerHealthClassifier.BlockingSeverity(signals.BlockingEvents, peakBlockSeconds, signals.Window) + /* #5644: the band's count is the span's blocked-process REPORT rows (DailyHealthSignals.BlockingReports), the + unit its rate tiers were measured in; BlockingEvents counts WAITS and is what the evidence shows as the + count. Classify reads the same pair, so the ledger still cannot page on a count the verdict did not band. */ + var bandedBlocking = signals.BlockingReports ?? signals.BlockingEvents; + if (ServerHealthClassifier.BlockingSeverity(bandedBlocking, peakBlockSeconds, signals.Window) == HealthSeverity.Critical) { - yield return (FamilyBlocking, BlockingEvidence(signals.BlockingEvents, signals.Window, peakBlockSeconds)); + yield return (FamilyBlocking, BlockingEvidence(signals.BlockingEvents, bandedBlocking, signals.BlockingReports.HasValue, signals.Window, peakBlockSeconds)); } } /// The blocking family's evidence (#3539 A3): the arm that banded is the one named. A 60 s /// block is the wait arm's Critical whatever the rate; otherwise the RATE is the value, with the raw /// count as its own member for the operator reconciling against the blocking grid. - private static string BlockingEvidence(long count, TimeSpan window, double peakBlockSeconds) + private static string BlockingEvidence(long count, long bandedCount, bool bandedOnReports, TimeSpan window, double peakBlockSeconds) { if (peakBlockSeconds >= ServerHealthThresholds.BlockingCriticalWaitSeconds) { @@ -778,11 +782,13 @@ private static string BlockingEvidence(long count, TimeSpan window, double peakB /* Critical without the wait arm implies a rateable window at or past the Critical tier, so the rate is present by construction. */ - var ratePerHour = ServerHealthClassifier.BlockingRatePerHour(count, window)!.Value; + var ratePerHour = ServerHealthClassifier.BlockingRatePerHour(bandedCount, window)!.Value; return JsonSerializer.Serialize(new { derivation = "summary-scoring critical trigger under alerts_enabled: false — not an alert-engine replay", - trigger = "blocking events per hour over the sweep span", + trigger = bandedOnReports + ? "blocked-process reports per hour over the sweep span" + : "blocking events per hour over the sweep span", value = ratePerHour, threshold = ServerHealthThresholds.BlockingCriticalPerHour, blocking_count = count, @@ -979,6 +985,8 @@ internal static async Task ReadServerSignalsAsync( CollectionRuns = rows.Sum(r => r.CollectionRuns), HighCpuEvents = rows.Sum(r => r.HighCpuEvents), BlockingEvents = rows.Sum(r => r.BlockingEvents), + /* #5644: the report rows the band's count tiers were measured in; a hand-built row without them bands its events. */ + BlockingReports = rows.Sum(r => r.BlockingReports ?? r.BlockingEvents), /* #3539 A2: the longest block across the span's day buckets — a MAX, not a sum, because it is a magnitude; the blocking band's wait arm reads it. */ PeakBlockWaitMs = peakBlock, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs index 5f12dac077..e3ac56a029 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs @@ -303,6 +303,12 @@ public sealed record DailySummaryReadRow( /// from a day the purge has not reached. public int SignalSourcesPresent { get; init; } + /// The blocked-process report rows behind (#5644), from the aggregate's + /// trailing blocking_reports column. counts WAITS and is what every + /// payload shows; the band's count tiers were measured in reports, so the band reads this. null on a + /// hand-built row, which bands its . + public long? BlockingReports { get; init; } + public DailyHealthSignals ToSignals() => new() { /* #3541 A9: a purged or past-horizon day is a NoData day to the band, whatever the spine still @@ -316,6 +322,7 @@ zero IS a measurement and the band stands. */ CollectionRuns = CollectionRuns, HighCpuEvents = HighCpuEvents, BlockingEvents = BlockingEvents, + BlockingReports = BlockingReports, /* #3539 A2: the day bands blocking through the card's BlockingSeverity, whose wait arm reads the longest block — so the peak travels in the signals rather than only into the reasons line. */ PeakBlockWaitMs = MaxBlockDurationMs, @@ -731,5 +738,7 @@ positional reads above stay where they were. */ CollectionRuns = reader.IsDBNull(12) ? 0L : Convert.ToInt64(reader.GetValue(12)), /* #3541 A9: the signal-presence count, after collection_runs. */ SignalSourcesPresent = reader.IsDBNull(13) ? 0 : Convert.ToInt32(reader.GetValue(13)), + /* #5644: the report count the band reads, after signal_sources_present. */ + BlockingReports = reader.IsDBNull(14) ? 0L : Convert.ToInt64(reader.GetValue(14)), }; } diff --git a/Darling/PerformanceMonitor.Darling.Storage/DailySummaryAzureMasterScope.cs b/Darling/PerformanceMonitor.Darling.Storage/DailySummaryAzureMasterScope.cs index b910e1b09e..89e5efe498 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DailySummaryAzureMasterScope.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DailySummaryAzureMasterScope.cs @@ -49,6 +49,10 @@ public static class DailySummaryAzureMasterScope private const string DeadlockSelectLine = "SELECT date_trunc('day', deadlock_time) AS d, COUNT(*) AS c"; private const string BprSelectLine = "SELECT date_trunc('day', event_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms"; + // #5644: the report count the health band reads, the bpr CTE's next select line. It takes the same scope filter + // as the wait count beside it, or a scoped day would band on reports of waits it no longer counts. + private const string BprReportsLine = "SUM(wait_reports)::bigint AS reports"; + /// The graphs the every-process rule still has to decide, one row per stored deadlock: the row's /// database is unknown, is master, or is a listed one. $1 server_id, $2/$3 the half-open window as /// reads it, $4 the event-window floor for $2, $5 the list. @@ -74,9 +78,25 @@ public static string Scope(string routedSql) scoped = ReplaceOnce(scoped, BprSelectLine, "SELECT date_trunc('day', event_time) AS d, COUNT(*) FILTER (WHERE " + BprKeepPredicate + ") AS c, " + "MAX(wait_time_ms) FILTER (WHERE " + BprKeepPredicate + ") AS max_wait_ms"); + /* The line after it is the bpr CTE's own (the dmv CTE's reports column is the same text, so the bpr select line + comes first): replace the FIRST occurrence only, which is the bpr one. */ + scoped = ReplaceFirst(scoped, BprReportsLine, + "SUM(wait_reports) FILTER (WHERE " + BprKeepPredicate + ")::bigint AS reports"); return scoped; } + private static string ReplaceFirst(string sql, string fragment, string replacement) + { + var at = sql.IndexOf(fragment, StringComparison.Ordinal); + if (at < 0) + { + throw new InvalidOperationException( + "Daily-summary master scoping found no '" + fragment + "' to replace: DailySummarySql.RangeSql has drifted from DailySummaryAzureMasterScope (#5644)."); + } + + return sql.Remove(at, fragment.Length).Insert(at, replacement); + } + private static string ReplaceOnce(string sql, string fragment, string replacement) { var at = sql.IndexOf(fragment, StringComparison.Ordinal); diff --git a/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs b/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs index 4d4ac16bc6..c97b673ebd 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs @@ -92,12 +92,14 @@ GROUP BY 1 bpr AS ( /* #5644: counts WAITS, not report rows (BlockingWaits.WaitsCteSql, the one definition the alert, the fleet tile and the MCP counts share), bucketed on the day of the wait's longest report. */ - SELECT date_trunc('day', event_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms + SELECT date_trunc('day', event_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms, + SUM(wait_reports)::bigint AS reports FROM bpr_waits GROUP BY 1 ), dmv AS ( - SELECT date_trunc('day', collection_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms + SELECT date_trunc('day', collection_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms, + SUM(wait_reports)::bigint AS reports FROM dmv_waits GROUP BY 1 ), @@ -208,7 +210,13 @@ purged shell whose one surviving "signal" is a day the tier never carried into p + (CASE WHEN b.d IS NULL THEN 0 ELSE 1 END) + (CASE WHEN dm.d IS NULL THEN 0 ELSE 1 END) + (CASE WHEN cp.d IS NULL THEN 0 ELSE 1 END) - + (CASE WHEN m.d IS NULL THEN 0 ELSE 1 END) AS signal_sources_present + + (CASE WHEN m.d IS NULL THEN 0 ELSE 1 END) AS signal_sources_present, + /* #5644: the day's blocked-process REPORT rows (DMV snapshot rows on the fallback), from the SAME source + and day bucket as blocking_events, which counts WAITS. The health band's count tiers + (ServerHealthThresholds.BlockingWarnPerHour/CriticalPerHour) were measured in reports per server-hour, + so the band reads this column; every displayed count stays blocking_events. Appended LAST, after + signal_sources_present, so every existing ordinal read stays where it was. */ + CASE WHEN COALESCE(b.c, 0) > 0 THEN b.reports ELSE COALESCE(dm.reports, 0) END AS blocking_reports FROM day_spine s LEFT JOIN waits w ON w.d = s.d LEFT JOIN queries q ON q.d = s.d @@ -234,10 +242,10 @@ ORDER BY s.d BlockingWaits.WaitsCteSql( BlockingWaits.SqlDialect.Postgres, "bpr_waits", @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms, database_name FROM v_blocked_process_reports - WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $4") + "," + BlockingWaits.WaitsCteSql( + WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $4", reportCount: true) + "," + BlockingWaits.WaitsCteSql( BlockingWaits.SqlDialect.Postgres, "dmv_waits", @"SELECT blocked_spid, blocking_spid, event_time, collection_time, wait_time_ms FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3") + ",", + WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3", reportCount: true) + ",", StringComparison.Ordinal); /// diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs index dffa0cbb2d..d17eb7bc7c 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DailySummary.cs @@ -67,6 +67,7 @@ private sealed record RawDailySummaryDay public long? UniqueQueries { get; init; } public long DeadlockCount { get; init; } public long BlockingEvents { get; init; } + public long BlockingReports { get; init; } public long HighCpuEvents { get; init; } public long CollectionErrors { get; init; } public long MemoryPressureEvents { get; init; } @@ -290,6 +291,8 @@ skipped. The MCP reader (DarlingHealthReader) makes the same read at the same or /* #3541 A9 / #3653: the aggregate's LAST column, how many of the seven signal sources hold a row for the day — the fact that tells a purged shell from a day the purge has not reached. */ SignalSourcesPresent = reader.IsDBNull(13) ? 0 : Convert.ToInt32(reader.GetValue(13)), + /* #5644: the report count the band reads (blocking_events counts WAITS), after signal_sources_present. */ + BlockingReports = reader.IsDBNull(14) ? 0L : Convert.ToInt64(reader.GetValue(14)), }; /// #4232: the banding half of the old ReadDailySummaryRow — stamps a raw parsed day with the @@ -306,6 +309,7 @@ private static DailySummaryRow ToDailySummaryRow(RawDailySummaryDay raw, DailyHe UniqueQueries = raw.UniqueQueries, DeadlockCount = raw.DeadlockCount, BlockingEvents = raw.BlockingEvents, + BlockingReports = raw.BlockingReports, HighCpuEvents = raw.HighCpuEvents, CollectionErrors = raw.CollectionErrors, MemoryPressureEvents = raw.MemoryPressureEvents, @@ -347,6 +351,12 @@ public class DailySummaryRow public long? UniqueQueries { get; set; } public long DeadlockCount { get; set; } public long BlockingEvents { get; set; } + + /// The blocked-process report rows behind (#5644). The band's count tiers were + /// measured in reports, so carries this for the band while every display keeps + /// , the count of WAITS. null on a hand-built row, which bands its + /// . + public long? BlockingReports { get; set; } public long HighCpuEvents { get; set; } public long MemoryPressureEvents { get; set; } public long MemoryCriticalEvents { get; set; } @@ -407,6 +417,7 @@ row and the MCP reader's row make. */ CollectionRuns = CollectionRuns, HighCpuEvents = HighCpuEvents, BlockingEvents = BlockingEvents, + BlockingReports = BlockingReports, /* #3539 A2: the blocking band's wait arm reads the longest block, so the peak rides in the signals. */ PeakBlockWaitMs = MaxBlockDurationMs, MemoryPressureEvents = MemoryPressureEvents, diff --git a/Lite.Tests/BlockingWaitDuckDbTests.cs b/Lite.Tests/BlockingWaitDuckDbTests.cs index 9664afd12a..137a413cc6 100644 --- a/Lite.Tests/BlockingWaitDuckDbTests.cs +++ b/Lite.Tests/BlockingWaitDuckDbTests.cs @@ -73,12 +73,13 @@ await ExecAsync( /// moves each report's wait time off the exact value, alternately up and down, so consecutive derived starts differ by twice that. /// private async Task SeedWaitAsync( - int blocked, int blocker, DateTime start, int reports, int intervalSeconds = 5, int jitterMs = 0, bool dmv = false) + int blocked, int blocker, DateTime start, int reports, int intervalSeconds = 5, int jitterMs = 0, bool dmv = false, int intervalMs = 0) { for (var k = 0; k < reports; k++) { - var waitMs = ((k + 1) * intervalSeconds * 1000L) + (k % 2 == 0 ? jitterMs : -jitterMs); - var eventTime = start.AddSeconds((k + 1) * intervalSeconds); + var step = intervalMs > 0 ? intervalMs : intervalSeconds * 1000; + var waitMs = ((k + 1) * step * 1L) + (k % 2 == 0 ? jitterMs : -jitterMs); + var eventTime = start.AddMilliseconds((k + 1) * step); _seeded.Add(new Report(blocked, blocker, eventTime, waitMs)); if (dmv) await ExecAsync( @@ -219,6 +220,41 @@ public async Task TheServerTabBadgeAndTheDailySummary_CountWaitsOnTheDmvFallback Assert.Equal(2, await DailySummaryBlockingEventsAsync()); } + /// #5644 review: one long wait written as 130 reports (a report every 50 ms, so the longest block is 6.5 s and + /// the wait arm stays out of it) is ONE blocking event, which the overview card, the badge and the MCP counts show, + /// but the band's count tiers (5 and 20 per server-hour) were measured in reports, so the day bands Warning on + /// 130 / 24 h = 5.4 per hour. On the waits-banded head the day read 1 event over 24 h and came out Healthy. The + /// overview card's last-hour count stays one wait, and that card does not band by the tiers at all (it is elevated + /// on any blocking). + [Fact] + public async Task TheDailyBand_ReadsTheReportCount_WhileTheDisplayedCountsStayWaits() + { + var yesterday = DateTime.UtcNow.Date.AddDays(-1); + await SeedWaitAsync(52, 51, yesterday.AddHours(2), reports: 130, intervalMs: 50); + + var row = Assert.Single(await new LocalDataService(_duckDb).GetDailySummaryRangeAsync(ServerId, yesterday, yesterday.AddDays(1))); + + Assert.Equal(1, row.BlockingEvents); + Assert.Equal(130, row.BlockingReports); + Assert.Equal(6_500, row.MaxBlockDurationMs); + Assert.Equal(DailyHealthBand.Warning, row.HealthBand); + Assert.Equal(1, CSharpWaitCount()); + } + + /// The report count follows the DMV snapshot fallback too (snapshot rows are what the band counted before waits). + [Fact] + public async Task TheDailyReportCount_FollowsTheDmvFallback() + { + var yesterday = DateTime.UtcNow.Date.AddDays(-1); + await SeedWaitAsync(52, 51, yesterday.AddHours(2), reports: 11, dmv: true); + await SeedWaitAsync(53, 51, yesterday.AddHours(3), reports: 6, dmv: true); + + var row = Assert.Single(await new LocalDataService(_duckDb).GetDailySummaryRangeAsync(ServerId, yesterday, yesterday.AddDays(1))); + + Assert.Equal(2, row.BlockingEvents); + Assert.Equal(17, row.BlockingReports); + } + /* ── #5644 wave-1 review: sub-millisecond starts and NULL times ── */ private sealed record Edge(int Blocked, int Blocker, DateTime? Time, long? Wait); diff --git a/Lite.Tests/PerformanceCalendarDataTests.cs b/Lite.Tests/PerformanceCalendarDataTests.cs index 2b0403cdf1..795ee09a0a 100644 --- a/Lite.Tests/PerformanceCalendarDataTests.cs +++ b/Lite.Tests/PerformanceCalendarDataTests.cs @@ -236,14 +236,14 @@ public async Task GetDailySummaryRange_BucketsEachDay_AndBandsViaSharedCalculato Assert.Equal(DailyHealthBand.Warning, byDate[Day(9)].HealthBand); Assert.Equal(15_000, byDate[Day(9)].MaxBlockDurationMs); - Assert.Contains("1 blocking event (0.0/hr, peak block 15.0 s)", byDate[Day(9)].SignalsTooltip, StringComparison.Ordinal); + Assert.Contains("1 blocking event (0.0 reports/hr, peak block 15.0 s)", byDate[Day(9)].SignalsTooltip, StringComparison.Ordinal); Assert.Equal(DailyHealthBand.Warning, byDate[Day(10)].HealthBand); Assert.Equal(1, byDate[Day(10)].AlertCount); // dismissed + resolution excluded Assert.Equal(DailyHealthBand.Warning, byDate[Day(11)].HealthBand); Assert.Equal(120, byDate[Day(11)].BlockingEvents); - Assert.Contains("120 blocking events (5.0/hr, peak block 6.0 s)", byDate[Day(11)].SignalsTooltip, StringComparison.Ordinal); + Assert.Contains("120 blocking events (5.0 reports/hr, peak block 6.0 s)", byDate[Day(11)].SignalsTooltip, StringComparison.Ordinal); Assert.Equal(DailyHealthBand.Healthy, byDate[Day(12)].HealthBand); Assert.True(byDate[Day(12)].HasData); diff --git a/Lite/Services/LocalDataService.DailySummary.cs b/Lite/Services/LocalDataService.DailySummary.cs index caede84489..9c3b34f74a 100644 --- a/Lite/Services/LocalDataService.DailySummary.cs +++ b/Lite/Services/LocalDataService.DailySummary.cs @@ -55,20 +55,22 @@ GROUP BY 1 " + BlockingWaits.WaitsCteSql( BlockingWaits.SqlDialect.DuckDb, "bpr_waits", @"SELECT blocked_spid, blocking_spid, event_time, wait_time_ms - FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3/*scope*/") + @" AS ev") + "," + BlockingWaits.WaitsCteSql( + FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3/*scope*/") + @" AS ev", reportCount: true) + "," + BlockingWaits.WaitsCteSql( BlockingWaits.SqlDialect.DuckDb, "dmv_waits", @"SELECT blocked_spid, blocking_spid, event_time, collection_time, wait_time_ms FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3/*scope*/") + @", + WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3/*scope*/", reportCount: true) + @", bpr AS ( /* #5644: counts WAITS, not report rows (BlockingWaits.WaitsCteSql, the definition the alert, the Overview tile and the analysis fact share), bucketed on the day of the wait's longest report. */ - SELECT date_trunc('day', event_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms + SELECT date_trunc('day', event_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms, + CAST(SUM(wait_reports) AS BIGINT) AS reports FROM bpr_waits GROUP BY 1 ), dmv AS ( - SELECT date_trunc('day', collection_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms + SELECT date_trunc('day', collection_time) AS d, COUNT(*) AS c, MAX(wait_time_ms) AS max_wait_ms, + CAST(SUM(wait_reports) AS BIGINT) AS reports FROM dmv_waits GROUP BY 1 ), @@ -162,7 +164,11 @@ share. Appended after peak_block_wait_ms so every existing ordinal read stays wh + (CASE WHEN b.d IS NULL THEN 0 ELSE 1 END) + (CASE WHEN dm.d IS NULL THEN 0 ELSE 1 END) + (CASE WHEN cp.d IS NULL THEN 0 ELSE 1 END) - + (CASE WHEN m.d IS NULL THEN 0 ELSE 1 END) AS signal_sources_present + + (CASE WHEN m.d IS NULL THEN 0 ELSE 1 END) AS signal_sources_present, + /* #5644: the day's blocked-process REPORT rows (DMV snapshot rows on the fallback), from the SAME source and day + bucket as blocking_events, which counts WAITS. The health band's count tiers were measured in reports per + server-hour, so the band reads this column; every displayed count stays blocking_events. Appended LAST. */ + CASE WHEN COALESCE(b.c, 0) > 0 THEN b.reports ELSE COALESCE(dm.reports, 0) END AS blocking_reports FROM day_spine s LEFT JOIN waits w ON w.d = s.d LEFT JOIN queries q ON q.d = s.d @@ -289,6 +295,8 @@ private static DailySummaryRow ReadDailySummaryRow(System.Data.Common.DbDataRead CollectionRuns = reader.IsDBNull(12) ? 0L : Convert.ToInt64(reader.GetValue(12)), /* #3541 A9: the signal-presence count, after collection_runs. */ SignalSourcesPresent = reader.IsDBNull(13) ? 0 : Convert.ToInt32(reader.GetValue(13)), + /* #5644: the report count the band reads (blocking_events counts WAITS), after signal_sources_present. */ + BlockingReports = reader.IsDBNull(14) ? 0L : Convert.ToInt64(reader.GetValue(14)), HasData = true, }; if (scopedDeadlocks != null) @@ -317,6 +325,12 @@ public class DailySummaryRow public long UniqueQueries { get; set; } public long DeadlockCount { get; set; } public long BlockingEvents { get; set; } + + /// The blocked-process report rows behind (#5644). The band's count tiers were + /// measured in reports, so carries this for the band while every display keeps + /// , the count of WAITS. null on a hand-built row, which bands its + /// . + public long? BlockingReports { get; set; } public long HighCpuEvents { get; set; } public long MemoryPressureEvents { get; set; } public long MemoryCriticalEvents { get; set; } @@ -378,6 +392,7 @@ public class DailySummaryRow CollectionRuns = CollectionRuns, HighCpuEvents = HighCpuEvents, BlockingEvents = BlockingEvents, + BlockingReports = BlockingReports, /* #3539 A2: the blocking band's wait arm reads the longest block, so the peak rides in the signals. */ PeakBlockWaitMs = MaxBlockDurationMs, MemoryPressureEvents = MemoryPressureEvents, diff --git a/PerformanceMonitor.Common/BlockingWaits.cs b/PerformanceMonitor.Common/BlockingWaits.cs index f85c33fb04..cc3a8b0016 100644 --- a/PerformanceMonitor.Common/BlockingWaits.cs +++ b/PerformanceMonitor.Common/BlockingWaits.cs @@ -26,7 +26,9 @@ namespace PerformanceMonitor.Common; /// being read, so two reports of one (blocked, blocker) pair belong to the same wait when their derived starts are /// within of the previous one (reports sorted by derived start). A different /// wait of the same pair that was reported at all lasted at least the threshold (5 s) before its first report, -/// so its start is at least that far from the previous wait's start; the tolerance sits well below it. +/// so its start is at least that far from the previous wait's start; the tolerance sits well below it. That +/// holds for blocked-process reports only. A DMV blocking snapshot catches a wait of any length, so two separate +/// short waits of one pair can start less than 2 s apart and count as one wait; issue #5743 tracks the fix. /// /// The same rule is written twice because the counts live in two places: for rows /// already in memory (the alert engine's read), and for the counts that run in the @@ -122,12 +124,21 @@ public enum SqlDialect /// wait_time_ms is the longest wait of that wait's reports and whose other columns are that /// report's. The caller continues the WITH list after it (the text ends with the closing parenthesis, no /// trailing comma). + /// + /// adds a wait_reports column to the chain's last CTE: how many source + /// rows (blocked-process reports, or DMV snapshot rows) the wait collapsed. It is the count a health band + /// whose tiers were measured in REPORTS per server-hour reads (#5644), while the displayed count stays one per + /// wait. Off by default so every other caller's CTE keeps its shape. /// public static string WaitsCteSql( - SqlDialect dialect, string name, string source, IReadOnlyList? partitionColumns = null) + SqlDialect dialect, string name, string source, IReadOnlyList? partitionColumns = null, + bool reportCount = false) { var partition = string.Join( ", ", (partitionColumns ?? Array.Empty()).Concat(new[] { "blocked_spid", "blocking_spid" })); + var reportColumn = reportCount + ? $", COUNT(*) OVER (PARTITION BY {partition}, n.wait_no) AS wait_reports" + : string.Empty; var epochMs = dialect == SqlDialect.DuckDb ? "(CAST(epoch_us(s.event_time) AS DOUBLE) / 1000.0)" : "(date_part('epoch', s.event_time) * 1000.0)"; @@ -154,7 +165,7 @@ END AS starts_wait ), {name}_ranked AS ( SELECT n.*, - ROW_NUMBER() OVER (PARTITION BY {partition}, n.wait_no ORDER BY n.wait_time_ms DESC NULLS LAST, n.event_time DESC NULLS LAST) AS wait_rank + ROW_NUMBER() OVER (PARTITION BY {partition}, n.wait_no ORDER BY n.wait_time_ms DESC NULLS LAST, n.event_time DESC NULLS LAST) AS wait_rank{reportColumn} FROM {name}_numbered AS n ), {name} AS ( diff --git a/PerformanceMonitor.Common/DailyHealthBand.cs b/PerformanceMonitor.Common/DailyHealthBand.cs index 6d0783923e..adb9176412 100644 --- a/PerformanceMonitor.Common/DailyHealthBand.cs +++ b/PerformanceMonitor.Common/DailyHealthBand.cs @@ -103,11 +103,20 @@ public readonly record struct DailyHealthSignals /// rate — see . public long HighCpuEvents { get; init; } - /// Blocking events in the window (blocked-process reports, falling back to DMV blocking - /// snapshots). Banded as a RATE over through the card band's own + /// Blocking events in the window, counted as WAITS (#5644): one per blocked session, blocker and wait + /// start, from blocked-process reports and falling back to DMV blocking snapshots. The figure every surface + /// DISPLAYS. The band does not read it when is set. + public long BlockingEvents { get; init; } + + /// The blocked-process REPORT rows (DMV snapshot rows on the fallback) behind + /// (#5644), or null when the producer does not carry the report count, + /// in which case is banded. SQL Server writes about eleven reports for one + /// one-minute wait, and and + /// were measured in reports per server-hour, so the + /// band keeps that unit: it reads this count as a RATE over through the card band's own /// (#3539 A2/A3), together with /// . - public long BlockingEvents { get; init; } + public long? BlockingReports { get; init; } /// The longest single block in the window, in milliseconds (#3539 A2) — the blocking /// band's rate-independent wait arm: a 60-second block is Critical and a 10-second one Warning @@ -266,7 +275,7 @@ arm are documented on ServerHealthThresholds. Delegating rather than restating i undeclared span) is the fallback this classifier wants: a non-zero count reads Warning, a zero count (Unknown) stays out of the blocking trigger. */ var blockingSeverity = ServerHealthClassifier.BlockingSeverity( - signals.BlockingEvents, signals.PeakBlockWaitMs / 1000.0, signals.Window); + signals.BlockingReports ?? signals.BlockingEvents, signals.PeakBlockWaitMs / 1000.0, signals.Window); var cpuSeverity = HighCpuSeverity(signals.HighCpuEvents, signals.Window, t); var collectionSeverity = CollectionErrorSeverity(signals.CollectionErrors, signals.CollectionRuns); @@ -626,9 +635,12 @@ private static void AppendCollectionErrors(List lines, in DailyHealthSig lines.Add(line); } - /// The blocking line — the count, then the per-hour rate the band evaluated when the window - /// is rateable (#3539 A3, the deadlock line's rule), then the peak block duration when one is known, - /// e.g. "42 blocking events (1.8/hr, peak block 12.5 s)". An unrateable window prints no rate. + /// The blocking line — the count (of WAITS, #5644), then the per-hour rate the band evaluated when the + /// window is rateable (#3539 A3, the deadlock line's rule), then the peak block duration when one is known, + /// e.g. "42 blocking events (1.8/hr, peak block 12.5 s)". The band's tiers are measured in blocked-process + /// reports per hour, so when is carried the rate is that + /// count's and says so ("19.8 reports/hr"), instead of a wait count's rate the band never read. An unrateable + /// window prints no rate. private static void AppendBlocking(List lines, in DailyHealthSignals signals, long peakBlockMs) { var count = signals.BlockingEvents; @@ -639,9 +651,10 @@ private static void AppendBlocking(List lines, in DailyHealthSignals sig var line = count.ToString("N0", CultureInfo.InvariantCulture) + " " + noun; var qualifiers = new List(2); - var rate = ServerHealthClassifier.BlockingRatePerHour(count, signals.Window); + var rate = ServerHealthClassifier.BlockingRatePerHour(signals.BlockingReports ?? count, signals.Window); if (rate.HasValue) - qualifiers.Add(rate.Value.ToString("0.0", CultureInfo.InvariantCulture) + "/hr"); + qualifiers.Add(rate.Value.ToString("0.0", CultureInfo.InvariantCulture) + + (signals.BlockingReports.HasValue ? " reports/hr" : "/hr")); if (peakBlockMs > 0) qualifiers.Add("peak block " + FormatDurationMs(peakBlockMs)); if (qualifiers.Count > 0) From 6e8fa8aa89cf83c6484e030d45f8ec662ee06836 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:50:19 -0400 Subject: [PATCH 41/42] The fleet bounded-read reference query returns the band's report counts too (#5644) --- Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs b/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs index f00c433488..aee29b590a 100644 --- a/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs +++ b/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs @@ -319,7 +319,9 @@ FROM v_cpu_scheduler_stats COALESCE(xe.cnt, 0) AS xe_count, COALESCE(xe.max_wait, 0) AS xe_max_wait, COALESCE(dmv.cnt, 0) AS dmv_count, - COALESCE(dmv.max_wait, 0) AS dmv_max_wait + COALESCE(dmv.max_wait, 0) AS dmv_max_wait, + COALESCE(xe.cnt, 0) AS xe_reports, + COALESCE(dmv.cnt, 0) AS dmv_reports FROM ( SELECT server_id, COUNT(*) AS cnt, MAX(wait_time_ms) AS max_wait From 3f6f8ae778696d3f30217fb81bffde39b8c7a979 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:51:52 -0400 Subject: [PATCH 42/42] Pin that the daily summary SQL carries the skip collector's name (#5644) --- Lite.Tests/CollectionCycleSkipTests.cs | 566 +++++++++++++++++++++++++ 1 file changed, 566 insertions(+) create mode 100644 Lite.Tests/CollectionCycleSkipTests.cs diff --git a/Lite.Tests/CollectionCycleSkipTests.cs b/Lite.Tests/CollectionCycleSkipTests.cs new file mode 100644 index 0000000000..d2bd6a0577 --- /dev/null +++ b/Lite.Tests/CollectionCycleSkipTests.cs @@ -0,0 +1,566 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using DuckDB.NET.Data; +using Microsoft.Extensions.Logging; +using PerformanceMonitorLite.Database; +using PerformanceMonitorLite.Mcp; +using PerformanceMonitorLite.Models; +using PerformanceMonitorLite.Services; +using Xunit; + +namespace PerformanceMonitorLite.Tests; + +/// +/// #5643: a collection cycle that runs late skips the grid slots it missed, and used to say nothing. The loop now counts +/// the skipped slots, logs one Warning naming the slowest steps (or the wait that overran), and writes one +/// collection_log row per enabled server. The count is driven here by the production wait +/// () over a fake clock and a fake delay, the way +/// drives it. +/// +/* Collection app-logger-statics: AFailedRowWrite_... reads AppLogger's process-wide buffer, which the other classes in the + collection also drain. */ +[Collection("app-logger-statics")] +public sealed class CollectionCycleSkipTests : IDisposable +{ + private static readonly DateTime T0 = new(2026, 10, 12, 10, 0, 0, DateTimeKind.Utc); + private static readonly TimeSpan Interval = TimeSpan.FromMinutes(1); + + private readonly string _dir = Directory.CreateTempSubdirectory("pm-lite-skip-tests-").FullName; + private readonly List _initializers = []; + + public void Dispose() + { + foreach (var initializer in _initializers) + { + initializer.Dispose(); + } + + try { Directory.Delete(_dir, recursive: true); } catch { /* best effort */ } + } + + /// + /// One loop pass from slot : the cycle's work takes , the wall clock + /// steps by before the wait begins, and the wait's delay comes back + /// late (negative: the clock stepped back while it ran). + /// + private static async Task<(DateTime Next, int Skipped)> PassAsync(DateTime start, TimeSpan cycleTook, + TimeSpan stepDuringCycle = default, TimeSpan oversleepDuringWait = default) + { + var now = start + cycleTook + stepDuringCycle; + var next = await CollectionBackgroundService.WaitForNextCycleAsync(start, Interval, () => now, + (wait, _) => + { + now += wait + oversleepDuringWait; + return Task.CompletedTask; + }, + CancellationToken.None); + return (next, CollectionCycleSkip.CountSkippedSlots(start, next, Interval)); + } + + [Theory] + [InlineData(5, 0)] // an ordinary cycle + [InlineData(59, 0)] // finishes just inside the interval + [InlineData(61, 1)] // 1 second over: the slot at +60 s was missed + [InlineData(130, 2)] // slots at +60 s and +120 s were missed + [InlineData(312, 5)] // 5 minutes 12 s: slots +60 s through +300 s + public async Task ACycleThatRunsLong_SkipsTheWholeIntervalsItMissed(int cycleSeconds, int expectedSkipped) + { + var (_, skipped) = await PassAsync(T0, TimeSpan.FromSeconds(cycleSeconds)); + + Assert.Equal(expectedSkipped, skipped); + } + + [Fact] + public async Task ACycleThatEndsAHairBeforeItsSlotOrAfterIt_SkipsNothing() + { + // The wait returns a second after its slot. + var (_, late) = await PassAsync(T0, TimeSpan.FromSeconds(5), oversleepDuringWait: TimeSpan.FromSeconds(1)); + // The wait returns before its slot (an early delay): the stamp keeps its grid slot. + var (_, early) = await PassAsync(T0, TimeSpan.FromSeconds(5), oversleepDuringWait: TimeSpan.FromMilliseconds(-300)); + + Assert.Equal(0, late); + Assert.Equal(0, early); + } + + [Fact] + public async Task AWallClockThatStepsBackwards_IsNotASkippedSlot() + { + /* #4732: the step re-anchors the grid at the clock's reading, which lands BEFORE the slot that should have come + next. Counting that as a skip (or a negative count) would write a SKIPPED row for every clock correction. */ + var (stepDuringCycle, _) = await PassAsync(T0, TimeSpan.FromSeconds(5), stepDuringCycle: TimeSpan.FromMinutes(-10)); + var (stepDuringWait, _) = await PassAsync(T0, TimeSpan.FromSeconds(5), oversleepDuringWait: TimeSpan.FromMinutes(-10)); + + Assert.Equal(0, CollectionCycleSkip.CountSkippedSlots(T0, stepDuringCycle, Interval)); + Assert.Equal(0, CollectionCycleSkip.CountSkippedSlots(T0, stepDuringWait, Interval)); + Assert.Equal(0, CollectionCycleSkip.CountSkippedSlots(T0, T0 - TimeSpan.FromDays(1), Interval)); + } + + [Fact] + public async Task AWaitThatOverran_CountsTheSlotsOfTheSleep_WhileTheCycleWasShort() + { + /* A 47-minute sleep inside the wait: the cycle took 5 s, the wait ran 47 minutes past its slot. */ + var (next, skipped) = await PassAsync(T0, TimeSpan.FromSeconds(5), oversleepDuringWait: TimeSpan.FromMinutes(47)); + + Assert.Equal(47, skipped); + Assert.Equal(T0 + TimeSpan.FromMinutes(1) + TimeSpan.FromMinutes(47), next); + } + + [Fact] + public void AnIntervalThatIsNotPositive_CountsNothing() + { + Assert.Equal(0, CollectionCycleSkip.CountSkippedSlots(T0, T0.AddHours(1), TimeSpan.Zero)); + } + + // ── the report's text ──────────────────────────────────────────────────────────────────────────────── + + private static List Steps() => + [ + new("connection_check", 400), + new("collectors", 61_000), + new("archival", 240_000), + new("query_store_backfill", 0), + new("retention", 9_000), + new("analysis", 2_000), + ]; + + [Fact] + public void ALongCycle_NamesItsTotalTheSlotsAndItsThreeSlowestSteps() + { + var report = CollectionCycleSkip.Build(4, TimeSpan.FromSeconds(312), TimeSpan.FromSeconds(1), Steps(), Interval); + + Assert.NotNull(report); + Assert.Equal( + "Collection cycle took 312s, so 4 collection slot(s) were skipped. Slowest steps: archival 240s, collectors 61s, retention 9s.", + report!.Message); + Assert.Equal(4, report.SlotsSkipped); + Assert.Equal(312_000, report.CycleMs); + Assert.Equal("archival", report.SlowestStep); + Assert.Equal(240_000, report.SlowestStepMs); + } + + [Fact] + public void AStepNameWithUnderscores_IsReadableInTheLine_AndKeepsItsNameInTheRow() + { + var steps = new List { new("query_store_backfill", 90_000), new("collectors", 1_500) }; + + var report = CollectionCycleSkip.Build(1, TimeSpan.FromSeconds(95), TimeSpan.Zero, steps, Interval); + + Assert.Contains("Slowest steps: query store backfill 90s, collectors 1.5s.", report!.Message); + Assert.Equal("query_store_backfill", report.SlowestStep); + } + + [Fact] + public void AShortCycleWithAnOverrunWait_SaysTheWaitOverran_AndNamesTheWaitAsTheSlowestStep() + { + var report = CollectionCycleSkip.Build(46, TimeSpan.FromSeconds(5), TimeSpan.FromMinutes(47), Steps(), Interval); + + Assert.NotNull(report); + Assert.Equal( + "Collection wait overran: it lasted 2820s against a 60s interval, so 46 collection slot(s) were skipped. " + + "The cycle itself took 5s; the computer may have slept or hibernated, or the clock stepped forward.", + report!.Message); + Assert.Equal("wait", report.SlowestStep); + Assert.Equal(2_820_000, report.SlowestStepMs); + Assert.Equal(5_000, report.CycleMs); + } + + [Fact] + public void ALongCycleFollowedByAnOverrunWait_PrintsBothCauses_AndDoesNotBlameTheCycleAlone() + { + /* A 65 s cycle (4 s over the interval), then the computer slept: the wait lasted 2 h. The slot count is the whole gap. */ + var report = CollectionCycleSkip.Build(124, TimeSpan.FromSeconds(65), TimeSpan.FromHours(2), Steps(), Interval); + + Assert.NotNull(report); + Assert.Equal( + "Collection cycle took 65s and the wait overran, so 124 collection slot(s) were skipped across the two. " + + "Slowest steps: archival 240s, collectors 61s, retention 9s. " + + "The wait lasted 7200s against a 60s interval; the computer may have slept or hibernated, or the clock stepped forward.", + report!.Message); + Assert.Equal(124, report.SlotsSkipped); + Assert.Equal(65_000, report.CycleMs); + /* The row's slowest item is whichever is longer: here the wait. */ + Assert.Equal("wait", report.SlowestStep); + Assert.Equal(7_200_000, report.SlowestStepMs); + Assert.DoesNotContain("so 124 collection slot(s) were skipped. ", report.Message, StringComparison.Ordinal); + } + + [Fact] + public void ALongCycleWhoseWaitRanOnlyItsShortSlack_IsTheCyclesCauseAlone() + { + /* A 130 s cycle then a wait that only used up its own slack (0 s left after the cycle): no overrun, so one cause. */ + var report = CollectionCycleSkip.Build(2, TimeSpan.FromSeconds(130), TimeSpan.FromSeconds(30), Steps(), Interval); + + Assert.StartsWith("Collection cycle took 130s, so 2 collection slot(s) were skipped.", report!.Message, StringComparison.Ordinal); + Assert.DoesNotContain("overran", report.Message, StringComparison.Ordinal); + } + + [Fact] + public void TheGapSentence_NamesBothEndsOfTheGap_WhenTheStampsAreGiven() + { + var late = T0 + TimeSpan.FromHours(25); + var report = CollectionCycleSkip.Build(1499, TimeSpan.FromSeconds(5), TimeSpan.FromHours(25), Steps(), Interval, T0, late); + + Assert.EndsWith(" The gap ran from 2026-10-12 10:00:00 UTC to 2026-10-13 11:00:00 UTC.", report!.Message, StringComparison.Ordinal); + Assert.DoesNotContain("=", report.Message); + } + + [Fact] + public void NoSkippedSlots_IsNoReport() + { + Assert.Null(CollectionCycleSkip.Build(0, TimeSpan.FromSeconds(300), TimeSpan.Zero, Steps(), Interval)); + } + + [Fact] + public void TheReportText_CarriesNoEpochLabel_SoTheBaselineMarkerReadNeverPicksItUp() + { + /* BaselineDiscontinuity.MarkerRowsSql selects every collection_log row whose note carries an epoch label. */ + var report = CollectionCycleSkip.Build(3, TimeSpan.FromSeconds(200), TimeSpan.Zero, Steps(), Interval)!; + var overran = CollectionCycleSkip.Build(3, TimeSpan.FromSeconds(2), TimeSpan.FromMinutes(5), Steps(), Interval)!; + + foreach (var message in new[] { report.Message, overran.Message }) + { + Assert.DoesNotContain("=", message); + } + } + + // ── the log line and the rows ──────────────────────────────────────────────────────────────────────── + + private sealed class CapturingLogger(List<(LogLevel Level, string Message)> sink) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, + Func formatter) => sink.Add((logLevel, formatter(state, exception))); + } + + private sealed record Fixture( + DuckDbInitializer DuckDb, + CollectionBackgroundService Loop, + ServerConnection Enabled1, + ServerConnection Enabled2, + ServerConnection Disabled, + List<(LogLevel Level, string Message)> Log); + + private async Task OpenAsync(bool dropLogTable = false) + { + var duckDb = new DuckDbInitializer(Path.Combine(_dir, "pm.duckdb")); + _initializers.Add(duckDb); + await duckDb.InitializeFromTemplateAsync(); + + var servers = new ServerManager(_dir); + var one = new ServerConnection { ServerName = "skip-one", DisplayName = "skip-one" }; + var two = new ServerConnection { ServerName = "skip-two", DisplayName = "skip-two" }; + var off = new ServerConnection { ServerName = "skip-off", DisplayName = "skip-off", IsEnabled = false }; + servers.AddServer(one); + servers.AddServer(two); + servers.AddServer(off); + + if (dropLogTable) + { + using var readLock = duckDb.AcquireReadLock(); + using var connection = duckDb.CreateConnection(); + await connection.OpenAsync(TestContext.Current.CancellationToken); + using var drop = connection.CreateCommand(); + drop.CommandText = "DROP TABLE collection_log"; + await drop.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + var collector = new RemoteCollectorService(duckDb, servers, new ScheduleManager(_dir)); + var log = new List<(LogLevel, string)>(); + var loop = new CollectionBackgroundService(collector, duckDb, serverManager: servers, + logger: new CapturingLogger(log)); + return new Fixture(duckDb, loop, one, two, off, log); + } + + private sealed record LogRow(string Collector, string Status, DateTime Time, int? DurationMs, int? Rows, int? SqlMs, + int? DuckMs, int? Items, string? SlowestItem, int? SlowestItemMs, string? Message, string ServerName); + + private static async Task> RowsAsync(DuckDbInitializer duckDb) + { + var rows = new Dictionary(); + using var readLock = duckDb.AcquireReadLock(); + using var connection = duckDb.CreateConnection(); + await connection.OpenAsync(TestContext.Current.CancellationToken); + using var command = connection.CreateCommand(); + command.CommandText = @"SELECT server_id, collector_name, status, collection_time, duration_ms, rows_collected, + sql_duration_ms, duckdb_duration_ms, fanout_item_count, slowest_item, slowest_item_ms, error_message, server_name + FROM collection_log"; + using var reader = await command.ExecuteReaderAsync(TestContext.Current.CancellationToken); + while (await reader.ReadAsync(TestContext.Current.CancellationToken)) + { + int? Int(int i) => reader.IsDBNull(i) ? null : Convert.ToInt32(reader.GetValue(i)); + rows[reader.GetInt32(0)] = new LogRow(reader.GetString(1), reader.GetString(2), reader.GetDateTime(3), Int(4), Int(5), + Int(6), Int(7), Int(8), reader.IsDBNull(9) ? null : reader.GetString(9), Int(10), + reader.IsDBNull(11) ? null : reader.GetString(11), reader.GetString(12)); + } + + return rows; + } + + [Fact] + public async Task ASkippedCycle_LogsOneWarning_AndWritesOneRowPerEnabledServer_WithTheAgreedShape() + { + var f = await OpenAsync(); + var previous = T0; + var next = T0 + TimeSpan.FromMinutes(6); + + await f.Loop.ReportSkippedSlotsAsync(previous, next, TimeSpan.FromSeconds(312), TimeSpan.FromSeconds(48), Steps(), + CancellationToken.None); + + var expected = CollectionCycleSkip.Build(5, TimeSpan.FromSeconds(312), TimeSpan.FromSeconds(48), Steps(), Interval, previous, next)!; + Assert.Contains("2026-10-12 10:00:00 UTC to 2026-10-12 10:06:00 UTC", expected.Message, StringComparison.Ordinal); + var warning = Assert.Single(f.Log); + Assert.Equal(LogLevel.Warning, warning.Level); + Assert.Equal(expected.Message, warning.Message); + + var rows = await RowsAsync(f.DuckDb); + Assert.Equal(2, rows.Count); + Assert.False(rows.ContainsKey(RemoteCollectorService.GetServerId(f.Disabled))); + foreach (var server in new[] { f.Enabled1, f.Enabled2 }) + { + var row = rows[RemoteCollectorService.GetServerId(server)]; + Assert.Equal(CollectionCycleSkip.CollectorName, row.Collector); + Assert.Equal("SKIPPED", row.Status); + /* The late cycle's own start (the stamp the wait returned), not the previous cycle's: the message names that end. */ + Assert.Equal(next, row.Time); + Assert.Equal(312_000, row.DurationMs); + Assert.Equal(5, row.Items); + Assert.Equal("archival", row.SlowestItem); + Assert.Equal(240_000, row.SlowestItemMs); + Assert.Equal(expected.Message, row.Message); + Assert.Equal(0, row.Rows); + Assert.Null(row.SqlMs); + Assert.Null(row.DuckMs); + Assert.Equal(server.DisplayName, row.ServerName); + } + } + + [Fact] + public async Task ATwentyFiveHourGapThatEndedAMinuteAgo_ShowsInTheMcpBlockAndTheCollectionHealthLine() + { + /* The row is stamped at the late cycle's own start, so a gap longer than the 24-hour window is still inside it for + 24 hours after it ends. Stamped at the previous cycle (25 hours back) it would read count_24h 0 from the start. */ + var f = await OpenAsync(); + var lateStamp = DateTime.UtcNow.AddMinutes(-1); + var previous = lateStamp.AddHours(-25); + + await f.Loop.ReportSkippedSlotsAsync(previous, lateStamp, TimeSpan.FromSeconds(5), TimeSpan.FromHours(25), Steps(), + CancellationToken.None); + + var summary = await new LocalDataService(f.DuckDb) + .GetSkippedCycleSummaryAsync(RemoteCollectorService.GetServerId(f.Enabled1), cancellationToken: TestContext.Current.CancellationToken); + var block = System.Text.Json.JsonSerializer.SerializeToElement(McpHealthTools.SkippedCyclesBlock(summary)); + + Assert.Equal(1, block.GetProperty("count_24h").GetInt64()); + Assert.Equal(1, summary.Count); + Assert.StartsWith("Skipped cycles in 24 h: 1 late cycle(s), 1,499 slot(s) skipped.", summary.SummaryLine(), StringComparison.Ordinal); + Assert.Contains("The gap ran from", summary.LastMessage, StringComparison.Ordinal); + } + + [Fact] + public async Task AWaitThatOverran_WritesTheWaitAsTheSlowestItem() + { + var f = await OpenAsync(); + + await f.Loop.ReportSkippedSlotsAsync(T0, T0 + TimeSpan.FromMinutes(48), TimeSpan.FromSeconds(5), + TimeSpan.FromMinutes(47), Steps(), CancellationToken.None); + + var row = (await RowsAsync(f.DuckDb))[RemoteCollectorService.GetServerId(f.Enabled1)]; + Assert.Equal("wait", row.SlowestItem); + Assert.Equal(2_820_000, row.SlowestItemMs); + Assert.Equal(5_000, row.DurationMs); + Assert.Equal(47, row.Items); + Assert.StartsWith("Collection wait overran", row.Message); + } + + [Fact] + public async Task ACycleOnTime_AndABackwardClockStep_LogNothingAndWriteNothing() + { + var f = await OpenAsync(); + + await f.Loop.ReportSkippedSlotsAsync(T0, T0 + Interval, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(55), Steps(), CancellationToken.None); + await f.Loop.ReportSkippedSlotsAsync(T0, T0 - TimeSpan.FromMinutes(10), TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(55), Steps(), CancellationToken.None); + + Assert.Empty(f.Log); + Assert.Empty(await RowsAsync(f.DuckDb)); + } + + [Fact] + public async Task AFailedRowWrite_IsLoggedAndNeverLeavesTheLoop() + { + var f = await OpenAsync(dropLogTable: true); + AppLogger.DrainBufferedLines(); + + var ex = await Record.ExceptionAsync(() => f.Loop.ReportSkippedSlotsAsync(T0, T0 + TimeSpan.FromMinutes(6), + TimeSpan.FromSeconds(312), TimeSpan.Zero, Steps(), CancellationToken.None)); + + Assert.Null(ex); + /* The loop's Warning is written before the row, on every skip; it is not the failure. The write failure goes to + AppLogger from RemoteCollectorService.WriteCollectionLogRowAsync, once per enabled server, read here from + AppLogger's own buffer (the seam its other pins use). */ + Assert.Single(f.Log); + var failures = AppLogger.DrainBufferedLines() + .Where(l => l.Contains("[ERROR]", StringComparison.Ordinal) + && l.Contains("[Collector]", StringComparison.Ordinal) + && l.Contains("COLLECTION LOGGING FAILED", StringComparison.Ordinal)) + .ToList(); + Assert.Equal(2, failures.Count); + } + + // ── the collector name has one home ────────────────────────────────────────────────────────────────── + + [Fact] + public void NoFileUnderLite_CopiesTheCollectorNameAsASingleQuotedSqlLiteral() + { + /* The nine reader predicates interpolate CollectionCycleSkip.CollectorName; a rename must move writer and readers + together. The census has no exceptions: doc comments write the name in tags, never in single quotes. */ + var literal = "'" + CollectionCycleSkip.CollectorName + "'"; + var lite = Path.GetFullPath(Path.Combine(RepoRootFromThisFile(), "Lite")); + var offenders = Directory.EnumerateFiles(lite, "*", SearchOption.AllDirectories) + .Where(f => !f.Contains(Path.DirectorySeparatorChar + "bin" + Path.DirectorySeparatorChar, StringComparison.Ordinal) + && !f.Contains(Path.DirectorySeparatorChar + "obj" + Path.DirectorySeparatorChar, StringComparison.Ordinal) + && !string.Equals(Path.GetFileName(f), "CollectionCycleSkip.cs", StringComparison.Ordinal) + && (f.EndsWith(".cs", StringComparison.Ordinal) || f.EndsWith(".xaml", StringComparison.Ordinal) + || f.EndsWith(".sql", StringComparison.Ordinal))) + .Where(f => File.ReadAllText(f).Contains(literal, StringComparison.Ordinal)) + .Select(f => Path.GetRelativePath(lite, f)) + .ToList(); + + Assert.Empty(offenders); + } + + [Fact] + public void TheNinePredicates_InterpolateTheCollectorName() + { + var services = Path.Combine(RepoRootFromThisFile(), "Lite", "Services"); + var count = Directory.EnumerateFiles(services, "LocalDataService.*.cs") + .Sum(f => System.Text.RegularExpressions.Regex.Matches(File.ReadAllText(f), + @"collector_name <> '\{CollectionCycleSkip\.CollectorName\}'").Count); + + Assert.Equal(9, count); + } + + [Fact] + public void TheDailySummaryAndCollectionHealthSql_CarryTheCollectorNamesValue_NotTheBracedTemplate() + { + /* #5644 merge guard: a literal that holds the {CollectionCycleSkip.CollectorName} hole but lost its $ prefix still + compiles, and the daily summary then compares collector_name against the template text and counts skipped + cycles as runs again. Read the built SQL, not the source. DailySummaryRangeSql is private, so it is read by + reflection; CollectionHealthSql is internal. */ + var daily = (string)typeof(LocalDataService) + .GetField("DailySummaryRangeSql", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! + .GetValue(null)!; + + foreach (var sql in new[] { daily, LocalDataService.CollectionHealthSql }) + { + Assert.Contains("collector_name <> '" + CollectionCycleSkip.CollectorName + "'", sql, StringComparison.Ordinal); + Assert.DoesNotContain("{CollectionCycleSkip", sql, StringComparison.Ordinal); + } + } + + [Fact] + public void EveryPredicateHole_SitsInAnInterpolatedLiteral() + { + /* The twin of the pin above for the readers whose SQL is composed inside a method (Overview, the collection-health + lookups, runs, config changes, the runtime precondition): walk each file's string literals and require that every + literal holding the hole starts with $. */ + var services = Path.Combine(RepoRootFromThisFile(), "Lite", "Services"); + const string hole = "{CollectionCycleSkip.CollectorName}"; + var holes = 0; + var bare = new List(); + foreach (var file in Directory.EnumerateFiles(services, "LocalDataService.*.cs")) + { + var s = File.ReadAllText(file); + var i = 0; + while (i < s.Length) + { + if (string.CompareOrdinal(s, i, "//", 0, 2) == 0) { var e = s.IndexOf('\n', i); i = e < 0 ? s.Length : e; } + else if (string.CompareOrdinal(s, i, "/*", 0, 2) == 0) { i = s.IndexOf("*/", i + 2, StringComparison.Ordinal) + 2; } + else if (s[i] == '\'') { i++; while (s[i] != '\'') i += s[i] == '\\' ? 2 : 1; i++; } + else + { + var start = i; + while (i < s.Length && (s[i] == '$' || s[i] == '@')) i++; + if (i < s.Length && s[i] == '"') + { + var prefix = s[start..i]; + var verbatim = prefix.Contains('@', StringComparison.Ordinal); + var j = i + 1; + while (true) + { + if (prefix.Contains('$', StringComparison.Ordinal) && s[j] == '{') + { + /* An interpolation hole may hold its own quotes (string.Join(", ", ids)); skip it whole. */ + if (s[j + 1] == '{') { j += 2; continue; } + var depth = 0; + do + { + if (s[j] == '{') depth++; + else if (s[j] == '}') depth--; + else if (s[j] == '"') { j++; while (s[j] != '"') j += s[j] == '\\' ? 2 : 1; } + j++; + } while (depth > 0); + } + else if (verbatim && string.CompareOrdinal(s, j, "\"\"", 0, 2) == 0) j += 2; + else if (!verbatim && s[j] == '\\') j += 2; + else if (s[j] == '"') break; + else j++; + } + var body = s[i..j]; + var at = body.IndexOf(hole, StringComparison.Ordinal); + if (at >= 0) + { + holes++; + if (!prefix.Contains('$', StringComparison.Ordinal)) + bare.Add(Path.GetFileName(file) + ":" + (s.Take(i + at).Count(c => c == '\n') + 1)); + } + i = j + 1; + } + else if (i == start) i++; + } + } + } + + Assert.Empty(bare); + Assert.True(holes >= 8, "expected the nine predicates to sit in at least eight literals, saw " + holes); + } + + private static string RepoRootFromThisFile([CallerFilePath] string thisFile = "") + => Path.GetFullPath(Path.Combine(Path.GetDirectoryName(thisFile)!, "..")); + + // ── the loop is wired to it ────────────────────────────────────────────────────────────────────────── + + private static string LoopSource([CallerFilePath] string thisFile = "") + => File.ReadAllText(Path.GetFullPath(Path.Combine(Path.GetDirectoryName(thisFile)!, "..", "Lite", "Services", "CollectionBackgroundService.cs"))); + + [Fact] + public void TheLoop_TimesEveryAwaitedStep_AndReportsAfterTheWait() + { + var source = LoopSource(); + var body = source[source.IndexOf("protected override async Task ExecuteAsync", StringComparison.Ordinal)..]; + body = body[..body.IndexOf("The pileup loop ends on the same token", StringComparison.Ordinal)]; + + var lastIndex = -1; + foreach (var step in new[] { "connection_check", "collectors", "archival", "query_store_backfill", "retention", + "findings_cleanup", "dismissed_alerts_cleanup", "analysis" }) + { + var index = body.IndexOf($"Lap(\"{step}\")", StringComparison.Ordinal); + Assert.True(index > lastIndex, $"step {step} is not timed in loop order"); + lastIndex = index; + } + + var wait = body.IndexOf("await WaitForNextCycleAsync(", StringComparison.Ordinal); + var report = body.IndexOf("await ReportSkippedSlotsAsync(previousCycleStart, cycleStart", StringComparison.Ordinal); + Assert.True(wait > 0 && report > wait, "the skipped-slot report must follow the wait that returns the new stamp"); + /* A step added to the loop without a Lap falls out of the report: every awaited housekeeping call is followed by one. */ + Assert.Equal(8, System.Text.RegularExpressions.Regex.Matches(body, @"Lap[(]""").Count); + Assert.Equal(6, System.Text.RegularExpressions.Regex.Matches(body, @"await Run\w+IfDueAsync\([^)]*\);\s*Lap\(").Count); + } +}