From a6f67ec341fa6979053b8d827c5108fca92e4327 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:26:33 -0400 Subject: [PATCH 1/7] The newest-snapshot block-count test anchors its window inside one chunk, so it no longer fails between 00:00 and 04:00 UTC (#5608) --- Darling/Darling.Tests/LiveClock.cs | 48 +++++++++++++++++++ .../NewestSnapshotReadsLiveTests.cs | 10 ++-- 2 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 Darling/Darling.Tests/LiveClock.cs diff --git a/Darling/Darling.Tests/LiveClock.cs b/Darling/Darling.Tests/LiveClock.cs new file mode 100644 index 0000000000..2cf1f5c659 --- /dev/null +++ b/Darling/Darling.Tests/LiveClock.cs @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; + +namespace Darling.Tests; + +/// +/// The one clock a live test seeds against (#5608). A live test that seeds rows "N hours before now" puts them in +/// different Timescale chunks depending on the hour the test runs: the store's raw chunks are one UTC day, so between +/// 00:00 and 04:00 UTC a 4-hour window straddles a chunk boundary, and between 00:00 and 24:00 a window of a day or +/// more always does. A test that counts blocks, reads a plan or looks at which chunks hold rows then answers +/// differently around midnight, and fails only on a pull request that happens to run then. +/// +/// is a fixed point inside one chunk (22:00 UTC on the previous UTC day), so a window +/// of up to four hours ending there never leaves its chunk and the answer is the same at any hour. A test that +/// wants a window of a day or more still straddles a boundary, but at the same place every run. +/// +/// The environment variable DARLING_TEST_CLOCK_UTC (an ISO-8601 UTC time, read only here) replaces the +/// real clock, so a run at 12:00 can be proved as if it were 00:30 without touching the machine's clock. +/// +internal static class LiveClock +{ + /// Name of the test-only override; it is read nowhere in product code. + public const string OverrideVariable = "DARLING_TEST_CLOCK_UTC"; + + /// "Now" for a live test: the override when set, else the UTC clock, cut to the second, unspecified kind (a store timestamp). + public static DateTime Now() + { + var text = Environment.GetEnvironmentVariable(OverrideVariable); + var now = string.IsNullOrWhiteSpace(text) + ? DateTime.UtcNow + : DateTime.Parse(text, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal); + return DateTime.SpecifyKind(new DateTime(now.Ticks - (now.Ticks % TimeSpan.TicksPerSecond)), DateTimeKind.Unspecified); + } + + /// + /// A fixed point inside one raw chunk: 22:00 UTC on the previous UTC day. Four hours before it is 18:00 the same day, + /// so a window of up to four hours never crosses a 24, 12 or 6 hour chunk boundary, whatever hour the test runs. + /// + public static DateTime AnchoredEnd() => Now().Date.AddDays(-1).AddHours(22); +} diff --git a/Darling/Darling.Tests/NewestSnapshotReadsLiveTests.cs b/Darling/Darling.Tests/NewestSnapshotReadsLiveTests.cs index 07edbfd74f..25844d36b9 100644 --- a/Darling/Darling.Tests/NewestSnapshotReadsLiveTests.cs +++ b/Darling/Darling.Tests/NewestSnapshotReadsLiveTests.cs @@ -112,11 +112,11 @@ AND sample_interval_seconds > 0 private static string? BaseConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); - private static DateTime Now() - { - var now = DateTime.UtcNow; - return DateTime.SpecifyKind(new DateTime(now.Ticks - (now.Ticks % TimeSpan.TicksPerSecond)), DateTimeKind.Unspecified); - } + /* #5608: the end of every window is the fixed anchor, never the clock. The perfmon read's block count depends on where + the day-sized raw chunk boundary falls in the seeded window: the 4-hour window straddles it between 00:00 and 04:00 + UTC, and the newest read then walks a different number of rows inside the newest chunk, so a 3x margin that held at + midday failed at 00:50 (perfmon old=264 new=120). The anchor keeps the window inside one chunk at any hour. */ + private static DateTime Now() => LiveClock.AnchoredEnd(); private static AnalysisContext Context(int serverId, DateTime end) => new() { From 7ce9bcf97b5d0e8deaa92a769958299843ffc914 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:36:54 -0400 Subject: [PATCH 2/7] Live tests that count plan blocks read the clock through LiveClock, and a census fails one that does not (#5608) --- Darling/Darling.Tests/LiveClockCensusTests.cs | 121 ++++++++++++++++++ .../PlanRegressionDailyReadShapeLiveTests.cs | 4 +- .../QueryStoreIntervalFloorWarmUpTests.cs | 5 +- 3 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 Darling/Darling.Tests/LiveClockCensusTests.cs diff --git a/Darling/Darling.Tests/LiveClockCensusTests.cs b/Darling/Darling.Tests/LiveClockCensusTests.cs new file mode 100644 index 0000000000..0603f36888 --- /dev/null +++ b/Darling/Darling.Tests/LiveClockCensusTests.cs @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using Xunit; + +namespace Darling.Tests; + +/// +/// A live Postgres test that counts blocks from a plan (Shared Hit Blocks / Shared Read Blocks) seeds its +/// window through or is on the allow list with its reason (#5608). Block counts depend on where +/// a Timescale chunk boundary falls in the seeded rows, and rows seeded "N hours before the clock" move that boundary +/// with the hour the test runs: the newest-snapshot test measured perfmon old=264 new=120 at 00:50 UTC against +/// new=57 at 01:13, and failed the pull request that happened to run then. keeps +/// a window inside one chunk; carries the test-only clock override, so a run at midday can +/// be proved as if it were 00:30. +/// +/// The rule is per file, as does it. "Reads a wall clock" is +/// DateTime.UtcNow, DateTime.Now, DateTimeOffset.UtcNow or DateTimeOffset.Now in code +/// (comments and string literals do not count); "counts blocks" is one of the two plan property names above. A file +/// that does both and does not call LiveClock. fails unless it is named in the allow list with a reason, and an +/// allow entry that no longer matches fails, so the list only shrinks. +/// +/// Not covered: tests that read plan SHAPE or chunk catalogs (about 70 files use EXPLAIN or the chunk +/// catalog and a wall clock). Their assertions are not block counts, so a boundary does not change the number they +/// compare, but each is a candidate for the same anchor; the PR body lists them. +/// +[Trait("Stage", "Guard")] +public sealed class LiveClockCensusTests +{ + private static readonly Dictionary s_allowed = new(StringComparer.Ordinal) + { + // Empty on purpose: every file that counts blocks anchors through LiveClock. Add an entry only with a reason that + // says why the count cannot depend on where a chunk boundary falls. + }; + + private static readonly Regex s_clock = new(@"\bDateTime(?:Offset)?\s*\.\s*(?:UtcNow|Now)\b", RegexOptions.Compiled); + + private static readonly Regex s_blocks = new(@"Shared (?:Hit|Read) Blocks", RegexOptions.Compiled); + + private static readonly Regex s_live = new(@"DARLING_TEST_PG|ScratchPostgres", RegexOptions.Compiled); + + private static readonly Regex s_anchor = new(@"\bLiveClock\s*\.", RegexOptions.Compiled); + + /// Anti-vacuity: at least this many files count blocks, so a scan that stopped matching fails. + private const int BlockCountingFloor = 2; + + internal sealed record Row(string File, bool Live, bool ReadsClock, bool CountsBlocks, bool UsesAnchor); + + internal static Row ScanText(string file, string text) + { + var code = CSharpSourceWalker.StripCommentsAndStrings(text); + return new Row(file, s_live.IsMatch(text), s_clock.IsMatch(code), s_blocks.IsMatch(text), s_anchor.IsMatch(code)); + } + + internal static List Problems(IEnumerable rows, IReadOnlyDictionary allowed) => + rows.Where(r => r.Live && r.CountsBlocks && r.ReadsClock && !r.UsesAnchor && !allowed.ContainsKey(r.File)) + .Select(r => r.File).ToList(); + + internal static List Stale(IEnumerable rows, IReadOnlyDictionary allowed) + { + var byFile = rows.ToDictionary(r => r.File, StringComparer.Ordinal); + return allowed.Keys + .Where(f => !byFile.TryGetValue(f, out var r) || !(r.Live && r.CountsBlocks && r.ReadsClock && !r.UsesAnchor)) + .ToList(); + } + + private static List Scan() => + Directory.EnumerateFiles(TestsRoot(), "*.cs", SearchOption.TopDirectoryOnly) + .Select(p => ScanText(Path.GetFileName(p), File.ReadAllText(p))) + .ToList(); + + [Fact] + public void TheDetector_FlagsAClockedBlockCounter_AndIgnoresAnchoredProseAndStringFiles() + { + var bare = ScanText("Bare.cs", "// DARLING_TEST_PG\nclass A { void B() { var e = DateTime.UtcNow; var s = \"Shared Hit Blocks\"; } }"); + var anchored = ScanText("Anchored.cs", "// DARLING_TEST_PG\nclass A { void B() { var e = LiveClock.AnchoredEnd(); var s = \"Shared Hit Blocks\"; var n = DateTime.UtcNow; } }"); + var prose = ScanText("Prose.cs", "// DARLING_TEST_PG DateTime.UtcNow\nclass A { void B() { var s = \"DateTime.UtcNow Shared Hit Blocks\"; } }"); + var noBlocks = ScanText("NoBlocks.cs", "// DARLING_TEST_PG\nclass A { void B() { var e = DateTime.UtcNow; } }"); + + Assert.Equal(new[] { "Bare.cs" }, Problems(new[] { bare, anchored, prose, noBlocks }, new Dictionary())); + Assert.Empty(Problems(new[] { bare }, new Dictionary { ["Bare.cs"] = "reason" })); + Assert.Equal(new[] { "Gone.cs" }, Stale(new[] { bare }, new Dictionary { ["Bare.cs"] = "reason", ["Gone.cs"] = "reason" })); + } + + [Fact] + public void EveryLiveTestThatCountsBlocks_SeedsThroughTheLiveClock_OrIsAllowedWithAReason() + { + var rows = Scan(); + + var problems = Problems(rows, s_allowed); + Assert.True( + problems.Count == 0, + "A live Postgres test counts plan blocks and reads the wall clock without LiveClock (#5608). The count depends on where a " + + "chunk boundary falls in the seeded rows, so it changes with the hour the test runs. Seed the end of the window with " + + "LiveClock.AnchoredEnd() (or LiveClock.Now() for a window that may straddle a boundary the same way every run), or name the " + + "file in LiveClockCensusTests.s_allowed with the reason the count cannot depend on a boundary:\n " + string.Join("\n ", problems)); + + var stale = Stale(rows, s_allowed); + Assert.True( + stale.Count == 0, + "These allow-list entries no longer name a live block-counting file that reads the wall clock without LiveClock; delete them:\n " + + string.Join("\n ", stale)); + + Assert.True( + rows.Count(r => r.Live && r.CountsBlocks) >= BlockCountingFloor, + "the scan found fewer live block-counting files than expected, so it stopped matching and 'no problems' would be vacuous"); + } + + private static string TestsRoot([CallerFilePath] string thisFile = "") => Path.GetDirectoryName(thisFile)!; +} diff --git a/Darling/Darling.Tests/PlanRegressionDailyReadShapeLiveTests.cs b/Darling/Darling.Tests/PlanRegressionDailyReadShapeLiveTests.cs index 8c91123b5d..34473edaef 100644 --- a/Darling/Darling.Tests/PlanRegressionDailyReadShapeLiveTests.cs +++ b/Darling/Darling.Tests/PlanRegressionDailyReadShapeLiveTests.cs @@ -134,7 +134,7 @@ store is promoted as of 19 days ago (the legacy bound is T-17, the create-ahead await connection.OpenAsync(ct); await PgMigrations.MigrateAsync(connection, ct); - var utcNow = DateTime.UtcNow; + var utcNow = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); // #5608: the clock override reaches the seed var promoted = await QueryStoreIntervalPartitions.RunPromotionAsync( connection, QueryStoreIntervalPartitions.Latest, utcNow.AddDays(-19), Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(50), ct); @@ -184,7 +184,7 @@ store is promoted as of 19 days ago (the legacy bound is T-17, the create-ahead /// private static async Task<(DateTime Today, List Built)> SeedAndBuildAsync(NpgsqlConnection connection, CancellationToken ct) { - var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var now = LiveClock.Now(); // #5608: whole UTC days from here; the clock override reaches the seed var today = now.Date; /* The late-row trigger fires per row for anything in its 16-day reach; the seed is not a late arrival, so it is diff --git a/Darling/Darling.Tests/QueryStoreIntervalFloorWarmUpTests.cs b/Darling/Darling.Tests/QueryStoreIntervalFloorWarmUpTests.cs index 2b64b78db2..73d69e23f4 100644 --- a/Darling/Darling.Tests/QueryStoreIntervalFloorWarmUpTests.cs +++ b/Darling/Darling.Tests/QueryStoreIntervalFloorWarmUpTests.cs @@ -389,7 +389,7 @@ public async Task APassThatOnlyDropsWholePartitions_DoesNotWarm() await PgMigrations.MigrateAsync(connection, ct); await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString); var log = new CapturingTestLogger(); - var utcNow = DateTime.UtcNow; + var utcNow = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); // #5608: the test clock override reaches the seed var dropped = new List(); foreach (var table in QueryStoreIntervalPartitions.All) @@ -478,7 +478,8 @@ await ExecuteAsync( retention pass row-purges it by name: that is where the dead index entries are. */ await ExecuteAsync(connection, $"ALTER TABLE {table.Legacy} SET (autovacuum_enabled = false)", ct); - var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + /* #5608: the rows land in the one legacy table (MINVALUE..MAXVALUE), so no chunk or day boundary is in play; the clock goes through LiveClock so a run can be proved at any hour. */ + var utcNow = LiveClock.Now(); var expiredStart = utcNow.AddDays(-(table.HorizonDays + 5)); var keptStart = utcNow.AddDays(-1); foreach (var serverId in ServerIds) From 09282bf8af2cf304902103a89339d509ae144b77 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:09:35 -0400 Subject: [PATCH 3/7] Live tests that pass a window end to the product read it through LiveClock, so a run can be proved at 00:30 and 12:00 UTC (#5608) --- ...ParameterSensitiveDrillDownTextLiveTests.cs | 2 +- .../PgTargetConfigSnapshotBoundTests.cs | 4 ++-- .../PlanRegressionDrillDownReuseLiveTests.cs | 2 +- .../RawChunkIntervalReconcilerLiveTests.cs | 12 ++++++------ .../Darling.Tests/RollupBackfillLiveTests.cs | 18 +++++++++--------- .../Darling.Tests/RollupFloorCacheLiveTests.cs | 4 ++-- .../TopCpuQueriesTextLiveTests.cs | 2 +- .../TopTextLookupPlanLiveTests.cs | 2 +- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Darling/Darling.Tests/ParameterSensitiveDrillDownTextLiveTests.cs b/Darling/Darling.Tests/ParameterSensitiveDrillDownTextLiveTests.cs index 59d418eae4..440bb24c18 100644 --- a/Darling/Darling.Tests/ParameterSensitiveDrillDownTextLiveTests.cs +++ b/Darling/Darling.Tests/ParameterSensitiveDrillDownTextLiveTests.cs @@ -171,7 +171,7 @@ public async Task TheDrillDown_ReturnsTheOraclesRows_AndResolvesTextForThePrinte try { - var windowEnd = TruncateToSeconds(DateTime.UtcNow); + var windowEnd = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var windowStart = windowEnd.AddHours(-4); var context = new AnalysisContext { diff --git a/Darling/Darling.Tests/PgTargetConfigSnapshotBoundTests.cs b/Darling/Darling.Tests/PgTargetConfigSnapshotBoundTests.cs index bf3a4ff0d9..f2dbf39917 100644 --- a/Darling/Darling.Tests/PgTargetConfigSnapshotBoundTests.cs +++ b/Darling/Darling.Tests/PgTargetConfigSnapshotBoundTests.cs @@ -212,7 +212,7 @@ public async Task TheDayPlansOnlyItsOwnChunks_AndTheNewestSnapshotStillWins_Agai await PgTargetFactCollectorTests.RegisterServerAsync(connection, PlanShapeServerId, ServerName, "postgres", 18, ct); /* shared_buffers 128 MB (16384 x 8kB) on every snapshot but the newest, which says 256 MB. */ - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); using (var seed = new NpgsqlCommand(@" INSERT INTO pg_server_config (collection_id, collection_time, server_id, server_name, name, setting, unit, source) SELECT 9_392_801_000 + (EXTRACT(EPOCH FROM t)::bigint % 1_000_000) * 10 + s.k, t, $1, $2, s.name, @@ -283,7 +283,7 @@ public async Task ADarkTargetAndAHistoricalWindow_ReadTheSameSnapshotsAsTheUnbou { await DeleteSentinelsAsync(connection, ct); await using var postgres = NpgsqlDataSource.Create(connectionString!); - var now = TruncateToSeconds(DateTime.UtcNow); + var now = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); /* Dark: the newest snapshot is 72 hours before the window's end; an older one disagrees with it. */ await PgTargetFactCollectorTests.RegisterServerAsync(connection, DarkServerId, ServerName, "postgres", 18, ct); diff --git a/Darling/Darling.Tests/PlanRegressionDrillDownReuseLiveTests.cs b/Darling/Darling.Tests/PlanRegressionDrillDownReuseLiveTests.cs index 247d09f79a..9e9ee03cb6 100644 --- a/Darling/Darling.Tests/PlanRegressionDrillDownReuseLiveTests.cs +++ b/Darling/Darling.Tests/PlanRegressionDrillDownReuseLiveTests.cs @@ -85,7 +85,7 @@ public async Task TheDrillDown_ReadsTheFactsOffendersOnly_AndReturnsWhatTheWhole try { - var periodEnd = TruncateToSeconds(DateTime.UtcNow); + var periodEnd = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var periodStart = periodEnd.AddHours(-4); await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) diff --git a/Darling/Darling.Tests/RawChunkIntervalReconcilerLiveTests.cs b/Darling/Darling.Tests/RawChunkIntervalReconcilerLiveTests.cs index 062c4ea574..713ff9c5e7 100644 --- a/Darling/Darling.Tests/RawChunkIntervalReconcilerLiveTests.cs +++ b/Darling/Darling.Tests/RawChunkIntervalReconcilerLiveTests.cs @@ -70,7 +70,7 @@ await ExecAsync(connection, "SELECT create_hypertable('collect.rci_test', by_range('ts', INTERVAL '24 hours'), if_not_exists => true, migrate_data => true)", ct); await ExecAsync(connection, "ALTER TABLE collect.rci_test SET (timescaledb.compress, timescaledb.compress_orderby = 'ts')", ct); - var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var utcNow = LiveClock.Now(); await PlantChunkAsync(connection, utcNow.AddDays(-3), ct); await PlantChunkAsync(connection, utcNow.AddDays(-2), ct); await ExecAsync(connection, "SELECT compress_chunk(c) FROM show_chunks('collect.rci_test') c", ct); @@ -78,7 +78,7 @@ await ExecAsync(connection, Assert.Equal(24, await CurrentIntervalHoursAsync(connection, ct)); /* ---- phase 1: narrows one rung, applies, records ---- */ - var changed1 = await RawChunkIntervalReconciler.ReconcileAsync(connection, budgetBytes: 1.0, DateTime.UtcNow, logger: null, ct); + var changed1 = await RawChunkIntervalReconciler.ReconcileAsync(connection, budgetBytes: 1.0, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), logger: null, ct); Assert.Equal(1, changed1); Assert.Equal(12, await CurrentIntervalHoursAsync(connection, ct)); @@ -96,7 +96,7 @@ await ExecAsync(connection, Assert.Equal(1L, await ScalarLongAsync(connection, "SELECT COUNT(*) FROM collect.raw_chunk_interval_reconcile_runs", ct)); /* ---- phase 2: same day, held — proves IntervalLastChangedUtc came from the history table ---- */ - var changed2 = await RawChunkIntervalReconciler.ReconcileAsync(connection, budgetBytes: 1.0, DateTime.UtcNow, logger: null, ct); + var changed2 = await RawChunkIntervalReconciler.ReconcileAsync(connection, budgetBytes: 1.0, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), logger: null, ct); Assert.Equal(0, changed2); Assert.Equal(12, await CurrentIntervalHoursAsync(connection, ct)); Assert.Equal(1L, await ScalarLongAsync(connection, "SELECT COUNT(*) FROM collect.raw_chunk_interval_rung_history", ct)); @@ -233,7 +233,7 @@ await ExecAsync(connection, "SELECT create_hypertable('collect.rated_test', by_range('ts', INTERVAL '24 hours'), if_not_exists => true, migrate_data => true)", ct); await ExecAsync(connection, "ALTER TABLE collect.rated_test SET (timescaledb.compress, timescaledb.compress_orderby = 'ts')", ct); - var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var utcNow = LiveClock.Now(); for (var day = 1; day <= 5; day++) { await PlantRatedChunkAsync(connection, utcNow.AddDays(-day), ct); @@ -263,7 +263,7 @@ JOIN timescaledb_information.chunks c var budgetBytes = openBytesAt24Hours / 2.55; var logger = new CapturingTestLogger(); - var changed = await RawChunkIntervalReconciler.ReconcileAsync(connection, budgetBytes, DateTime.UtcNow, logger, ct); + var changed = await RawChunkIntervalReconciler.ReconcileAsync(connection, budgetBytes, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), logger, ct); Assert.Equal(1, changed); Assert.Equal(12, await CurrentIntervalHoursAsync(connection, "rated_test", ct)); @@ -282,7 +282,7 @@ summary line still logs at Information with the ratio under 1. The rated table i either pass runs — that held line is asserted at Information too, by exact prefix. ---- */ var loggerNoChange = new CapturingTestLogger(); var changedNoChange = await RawChunkIntervalReconciler.ReconcileAsync( - connection, budgetBytes: openBytesAt24Hours * 1_000, DateTime.UtcNow, loggerNoChange, ct); + connection, budgetBytes: openBytesAt24Hours * 1_000, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), loggerNoChange, ct); Assert.Equal(0, changedNoChange); var summaryNoChange = loggerNoChange.Lines.FirstOrDefault(l => diff --git a/Darling/Darling.Tests/RollupBackfillLiveTests.cs b/Darling/Darling.Tests/RollupBackfillLiveTests.cs index a369cf3e73..554b380934 100644 --- a/Darling/Darling.Tests/RollupBackfillLiveTests.cs +++ b/Darling/Darling.Tests/RollupBackfillLiveTests.cs @@ -66,7 +66,7 @@ public async Task RollupCreatedOverExistingHistory_ReadsEmptyBelowItsFloor_Until await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); /* All timestamps Kind-Unspecified — naive-UTC storage, see PgCollectorRowWriter. */ - var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var now = LiveClock.Now(); var rawOldest = now.AddDays(-HistoryDays); /* ── 1. PRE-EXISTING HISTORY: raw rows going back well before any rollup exists. ── */ @@ -234,7 +234,7 @@ public async Task HealthyStore_WhereRawIsShallowerThanTheRollup_DoesNotFallBackT Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct)); await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); - var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var now = LiveClock.Now(); /* Plant history, materialize ALL of it, then purge raw back to a short horizon — the steady state a healthy store reaches once its purges are armed. */ @@ -304,7 +304,7 @@ public async Task BackfillRollupsVerb_DryRunsThenBackfills_AndReportsCoverageItA Assert.True(await TimescaleSupport.TryEnableAsync(setup, null, ct)); await TimescaleSupport.ConvertToHypertablesAsync(setup, null, ct); - var seedNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var seedNow = LiveClock.Now(); await SeedHourlyQueryStatsAsync(setup, seedNow.AddDays(-HistoryDays), seedNow, ct); await TimescaleSupport.EnsureContinuousAggregatesAsync(setup, null, ct); } @@ -431,7 +431,7 @@ public async Task BackfillRollupsVerb_ConvergesTheThreeLevelQueryStoreChain_Incl Assert.True(await TimescaleSupport.TryEnableAsync(setup, null, ct)); await TimescaleSupport.ConvertToHypertablesAsync(setup, null, ct); - var seedNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var seedNow = LiveClock.Now(); await SeedHourlyQueryStoreStatsAsync(setup, seedNow.AddDays(-HistoryDays), seedNow, ct); await TimescaleSupport.EnsureContinuousAggregatesAsync(setup, null, ct); } @@ -519,7 +519,7 @@ public async Task Probe_CalibrationSampleCountsBuckets_NotRows() Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct)); await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); - var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var now = LiveClock.Now(); await SeedHourlyQueryStatsAsync(connection, now.AddDays(-HistoryDays), now, ct); await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); @@ -586,7 +586,7 @@ public async Task MidSliceCancellation_LeavesTheFloorInsideTheRange_WithNoGapsAb /* Wide enough that the raw hypertable holds several chunks, so that one in the MIDDLE of the range exists to hold the refresh at. */ - var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var now = LiveClock.Now(); var rangeFrom = now.Date.AddDays(-CancellationHistoryDays); var rangeTo = now.Date; @@ -863,7 +863,7 @@ public async Task InterruptedBackfill_DoesNotLookComplete_AndTheArmingGateRefuse Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct)); await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); - var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var now = LiveClock.Now(); await SeedHourlyQueryStatsAsync(connection, now.AddDays(-HistoryDays), now, ct); await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); @@ -980,7 +980,7 @@ public async Task BackfillRollupsVerb_RefusesAnAbsurdPlan_WithoutClaimingSuccess Assert.True(await TimescaleSupport.TryEnableAsync(setup, null, ct)); await TimescaleSupport.ConvertToHypertablesAsync(setup, null, ct); - var seedNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var seedNow = LiveClock.Now(); await SeedHourlyQueryStatsAsync(setup, seedNow.AddDays(-2), seedNow, ct); /* ONE corrupt row, which is all it takes. */ @@ -1106,7 +1106,7 @@ public async Task HierarchicalDaily_ConvergesToItsHourly_NotToRaw_SoTheHourlyTie Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct)); await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); - var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var now = LiveClock.Now(); var historyFrom = now.Date.AddDays(-HistoryDays); await SeedHourlyQueryStatsAsync(connection, historyFrom, now, ct); diff --git a/Darling/Darling.Tests/RollupFloorCacheLiveTests.cs b/Darling/Darling.Tests/RollupFloorCacheLiveTests.cs index 87bd6c546f..f0e9f5c1ac 100644 --- a/Darling/Darling.Tests/RollupFloorCacheLiveTests.cs +++ b/Darling/Darling.Tests/RollupFloorCacheLiveTests.cs @@ -58,7 +58,7 @@ public async Task DetectRollupCoverageAsync_ReusesACompressedChunksFloor_UntilTh await stop.ExecuteNonQueryAsync(ct); } - var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var now = LiveClock.Now(); var oldest = now.AddDays(-3); /* ── 1. Plant three days of collect.query_store_stats and materialize the interval-honest hourly @@ -182,7 +182,7 @@ drop_chunks is called against the AGGREGATE VIEW (the only relation name that st midnight UTC gave the two stores of one test different seed starts. */ private static DateTime SeedStart(DateTime now) => DateTime.SpecifyKind(now.Date.AddDays(-3).AddHours(2), DateTimeKind.Unspecified); - private static DateTime SeedEnd() => DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + private static DateTime SeedEnd() => LiveClock.Now(); /// #5496: the seed start is a pure function of the one instant the test read, so it can be proved on both sides of /// midnight UTC without waiting for one. A second clock read would have moved the start a whole day when it crossed. diff --git a/Darling/Darling.Tests/TopCpuQueriesTextLiveTests.cs b/Darling/Darling.Tests/TopCpuQueriesTextLiveTests.cs index 459efd524b..c143812aa4 100644 --- a/Darling/Darling.Tests/TopCpuQueriesTextLiveTests.cs +++ b/Darling/Darling.Tests/TopCpuQueriesTextLiveTests.cs @@ -140,7 +140,7 @@ public async Task TheDrillDown_ReturnsTheOraclesRows_AndResolvesTextForThePrinte try { - var windowEnd = TruncateToSeconds(DateTime.UtcNow); + var windowEnd = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var windowStart = windowEnd.AddHours(-4); var context = new AnalysisContext { diff --git a/Darling/Darling.Tests/TopTextLookupPlanLiveTests.cs b/Darling/Darling.Tests/TopTextLookupPlanLiveTests.cs index 4f29ae6e36..6c716c15fd 100644 --- a/Darling/Darling.Tests/TopTextLookupPlanLiveTests.cs +++ b/Darling/Darling.Tests/TopTextLookupPlanLiveTests.cs @@ -320,7 +320,7 @@ private static async Task WithSeedAsync(Func Date: Thu, 8 Oct 2026 22:29:56 -0400 Subject: [PATCH 4/7] Live tests that read a plan or the chunk catalog take their clock from LiveClock, and the census covers them (#5608) Audit e1: live test files seed and read through LiveClock (five anchored inside one chunk). Files whose product path reads the real clock stay on it with a reason, and LiveClockCensusTests gains a second rule for live files that read EXPLAIN or the chunk catalog and a wall clock directly. --- .../BlockingListPlanFlagsLivePostgresTests.cs | 4 +- .../CaptureDownChunkOrderTests.cs | 3 +- .../CollectionHealthKeyedLookupParityTests.cs | 2 +- .../DailySummaryReadShapeTests.cs | 6 +- .../DarlingAgStatesReaderLiveEqualityTests.cs | 6 +- .../Darling.Tests/DarlingFleetReaderTests.cs | 4 +- ...ngPgIndexBloatCoverageLivePostgresTests.cs | 2 +- .../Darling.Tests/DarlingRetentionTests.cs | 4 +- .../Darling.Tests/DarlingSelfAlertTests.cs | 14 +- ...DarlingWatermarkFloorPlanShapeLiveTests.cs | 3 +- ...DarlingWatermarkFloorScanBoundLiveTests.cs | 3 +- ...rlingWatermarkIndexEquivalenceLiveTests.cs | 3 +- ...indowedReadsAreBoundedLivePostgresTests.cs | 4 +- .../FleetReadsAreBoundedByTheFleetTests.cs | 10 +- .../ForcePlanFailuresAccessPathTests.cs | 3 +- .../Darling.Tests/FrozenRollupLiveTests.cs | 9 +- .../JobHistoryPerServerTopNLiveTests.cs | 8 +- .../JobHistoryWatermarkEpochLiveTests.cs | 24 ++-- .../LatestCollectorOutcomePlanTests.cs | 2 +- .../Darling.Tests/LatestCpuReadShapeTests.cs | 6 +- .../Darling.Tests/LatestValueLookbackTests.cs | 22 ++-- Darling/Darling.Tests/LiveClockCensusTests.cs | 122 ++++++++++++++++-- .../MaterializationHoleProbeLiveTests.cs | 2 +- .../MaterializationHoleRepairTests.cs | 20 +-- .../MaterializationHoleScanShapeTests.cs | 4 +- 25 files changed, 203 insertions(+), 87 deletions(-) diff --git a/Darling/Darling.Tests/BlockingListPlanFlagsLivePostgresTests.cs b/Darling/Darling.Tests/BlockingListPlanFlagsLivePostgresTests.cs index 6c3c7c01cc..af9373826c 100644 --- a/Darling/Darling.Tests/BlockingListPlanFlagsLivePostgresTests.cs +++ b/Darling/Darling.Tests/BlockingListPlanFlagsLivePostgresTests.cs @@ -60,6 +60,7 @@ public async Task GetBlocking_FlagsThePagesRows_ByTheirOwnStoredPlans() try { await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + /* The tool reads its "last N hours" window from the real clock, which the test clock override cannot move (#5608). */ var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); /* Newest first by event_time: spid 101 (both plans), 102 (blocked only), 103 (blocking only), 104 (empty strings), @@ -132,6 +133,7 @@ public async Task GetDeadlocks_FlagsThePagesRows_ByTheirOwnStoredPlans() try { await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + /* The tool reads its "last N hours" window from the real clock, which the test clock override cannot move (#5608). */ var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); /* Newest first by deadlock_time: a plan, an empty string, NULL, and an older plan the page of three leaves out. */ @@ -180,7 +182,7 @@ public async Task TheListStatements_PlannerOutput_NamesNoPlanColumn() await connection.OpenAsync(ct); await PrepareStoreAsync(cs, connection, ct); - var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var now = LiveClock.Now(); foreach (var sql in new[] { DarlingBlockingReader.BlockedProcessReportsSql, diff --git a/Darling/Darling.Tests/CaptureDownChunkOrderTests.cs b/Darling/Darling.Tests/CaptureDownChunkOrderTests.cs index e9ff606b5f..ce5e3e5088 100644 --- a/Darling/Darling.Tests/CaptureDownChunkOrderTests.cs +++ b/Darling/Darling.Tests/CaptureDownChunkOrderTests.cs @@ -126,7 +126,8 @@ outside the collector catalog and so outside ConvertToHypertablesAsync. */ /* Eight days of one-minute blocked_process_report and five-minute deadlocks rows, plus a filler collector so the newest chunk holds rows the arms must skip past. All Kind-Unspecified: naive-UTC storage, see DarlingObservability.LogCollectionAsync. */ - var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + /* #5608: the end of the window is a fixed point inside one chunk, so the chunk layout the plan is read against is the same at any hour. */ + var utcNow = LiveClock.AnchoredEnd(); await SeedHistoryAsync(connection, utcNow, ct); /* The two verdict rows. deadlocks: SESSION_MISSING three days ago, then a SUCCESS a minute ago — NOT diff --git a/Darling/Darling.Tests/CollectionHealthKeyedLookupParityTests.cs b/Darling/Darling.Tests/CollectionHealthKeyedLookupParityTests.cs index fccde72c63..547247a4ff 100644 --- a/Darling/Darling.Tests/CollectionHealthKeyedLookupParityTests.cs +++ b/Darling/Darling.Tests/CollectionHealthKeyedLookupParityTests.cs @@ -261,7 +261,7 @@ private static void AssertOracleSaw(Result mcp, DateTime now) private static DateTime Now() { - var now = DateTime.UtcNow; + var now = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); return new DateTime(now.Ticks - (now.Ticks % TimeSpan.TicksPerMillisecond), DateTimeKind.Utc); } diff --git a/Darling/Darling.Tests/DailySummaryReadShapeTests.cs b/Darling/Darling.Tests/DailySummaryReadShapeTests.cs index 680099b002..727a250a8a 100644 --- a/Darling/Darling.Tests/DailySummaryReadShapeTests.cs +++ b/Darling/Darling.Tests/DailySummaryReadShapeTests.cs @@ -556,9 +556,9 @@ private static async Task SeedAsync(string what) await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); await RegisterServerAsync(connection, ct); - var d0 = DateTime.SpecifyKind(DateTime.UtcNow.Date.AddDays(-14), DateTimeKind.Unspecified); + var d0 = LiveClock.Now().Date.AddDays(-14); var days = Enumerable.Range(0, 10).Select(n => d0.AddDays(n)).ToArray(); - Assert.True(DateTime.UtcNow - days[^1].AddDays(1) > TimescaleSupport.DailyRefreshStartSpan, "the seed must sit below every refresh policy's window"); + Assert.True(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc) - days[^1].AddDays(1) > TimescaleSupport.DailyRefreshStartSpan, "the seed must sit below every refresh policy's window"); foreach (var (day, hashes) in new[] { (0, 3), (2, 5), (3, 4), (5, 6), (6, 2), (7, 3), (8, 1) }) { @@ -634,7 +634,7 @@ INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_major command.Parameters.AddWithValue(ServerId); command.Parameters.AddWithValue(ServerName); command.Parameters.AddWithValue(ServerName); - command.Parameters.AddWithValue(DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(LiveClock.Now()); await command.ExecuteNonQueryAsync(ct); } diff --git a/Darling/Darling.Tests/DarlingAgStatesReaderLiveEqualityTests.cs b/Darling/Darling.Tests/DarlingAgStatesReaderLiveEqualityTests.cs index 364758911a..5a5b10fb8f 100644 --- a/Darling/Darling.Tests/DarlingAgStatesReaderLiveEqualityTests.cs +++ b/Darling/Darling.Tests/DarlingAgStatesReaderLiveEqualityTests.cs @@ -179,7 +179,7 @@ public async Task AgStatesReader_MatchesTheOldUnboundedStatements_AcrossEverySer var bodySucceeded = false; try { - var now = Micro(DateTime.UtcNow); + var now = Micro(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await SeedAsync(connection, now, ct); var sentinelIds = new HashSet(AllNames.Select(Id)); @@ -328,7 +328,7 @@ public async Task AgStatesReader_StepTwoStaysBounded_ForCurrentAndStaleServers_A var bodySucceeded = false; try { - var now = Micro(DateTime.UtcNow); + var now = Micro(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await SeedAsync(connection, now, ct); var sentinelIds = new HashSet(AllNames.Select(Id)); @@ -443,7 +443,7 @@ ON CONFLICT (server_id) DO UPDATE SET is_enabled = $4, sql_major_version = 15 command.Parameters.AddWithValue(serverName); command.Parameters.AddWithValue(serverName); command.Parameters.AddWithValue(enabled); - command.Parameters.AddWithValue(DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(LiveClock.Now()); await command.ExecuteNonQueryAsync(ct); } diff --git a/Darling/Darling.Tests/DarlingFleetReaderTests.cs b/Darling/Darling.Tests/DarlingFleetReaderTests.cs index 1a79a062a2..bd31181e1c 100644 --- a/Darling/Darling.Tests/DarlingFleetReaderTests.cs +++ b/Darling/Darling.Tests/DarlingFleetReaderTests.cs @@ -807,7 +807,7 @@ public async Task GetFleetOverview_ServerWithNoCollectionHistory_ReadsAwaitingFi var bodySucceeded = false; try { - var now = DateTime.UtcNow; + var now = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); /* Registered, but no collection_log row is ever inserted for this server_id. */ await InsertServerAsync(connection, NoHistoryServerId, NoHistoryName, 3, ct); @@ -848,7 +848,7 @@ public async Task GetFleetOverview_PerServerFallbackAndBanding_AgainstDevPostgre var bodySucceeded = false; try { - var now = DateTime.UtcNow; + var now = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); var at = now.AddMinutes(-5); // inside the [now-1h, now] card window /* Plant representative editions so the read-through card platform can be asserted end-to-end: diff --git a/Darling/Darling.Tests/DarlingPgIndexBloatCoverageLivePostgresTests.cs b/Darling/Darling.Tests/DarlingPgIndexBloatCoverageLivePostgresTests.cs index cff387b117..129e0c6b11 100644 --- a/Darling/Darling.Tests/DarlingPgIndexBloatCoverageLivePostgresTests.cs +++ b/Darling/Darling.Tests/DarlingPgIndexBloatCoverageLivePostgresTests.cs @@ -92,7 +92,7 @@ public async Task TheCensusExecutesAndAnswersOverThePopulation() { await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); - var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var newest = now.AddHours(-2); var older = now.AddHours(-26); diff --git a/Darling/Darling.Tests/DarlingRetentionTests.cs b/Darling/Darling.Tests/DarlingRetentionTests.cs index 855f07c677..64fc7c7dcb 100644 --- a/Darling/Darling.Tests/DarlingRetentionTests.cs +++ b/Darling/Darling.Tests/DarlingRetentionTests.cs @@ -601,7 +601,7 @@ public async Task EndToEnd_PurgeDeletesExpiredRowsAndKeepsFreshOnes_AgainstDevPo try { /* All timestamps Kind-Unspecified — naive-UTC storage, see PgCollectorRowWriter. */ - var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var utcNow = LiveClock.Now(); /* wait_stats retention is 30 days: one row well past it, one fresh. Payload columns are nullable, so the standard prefix is enough. */ @@ -905,7 +905,7 @@ exercised on every run. */ var bodySucceeded = false; try { - var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var utcNow = LiveClock.Now(); using (var insert = new NpgsqlCommand( "INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name) VALUES ($1, $2, $3, $4)", connection)) diff --git a/Darling/Darling.Tests/DarlingSelfAlertTests.cs b/Darling/Darling.Tests/DarlingSelfAlertTests.cs index 788360d9b0..bb980cfac0 100644 --- a/Darling/Darling.Tests/DarlingSelfAlertTests.cs +++ b/Darling/Darling.Tests/DarlingSelfAlertTests.cs @@ -5675,7 +5675,7 @@ public async Task LiveStoreReads_ComputeCollectionStoppedAndCaptureDown() var bodySucceeded = false; try { - var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var utcNow = LiveClock.Now(); /* One old SUCCESS (45 min ago), then 12 recent ERRORs — the last 10 runs are all failures and the last success is well past the staleness window. */ @@ -5693,14 +5693,14 @@ the last success is well past the staleness window. */ Assert.Equal(DarlingSelfAlertEvaluator.ConsecutiveFailureThreshold, recentRuns); Assert.Equal(0, recentSuccess); Assert.True(DarlingSelfAlertEvaluator.IsCollectionStopped( - lastSuccess, recentRuns, recentSuccess, DateTime.UtcNow, out _)); + lastSuccess, recentRuns, recentSuccess, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), out _)); /* Full path (#4757): the seeded 45-minute-old success and the stored failure streak are both pre-restart rows to an evaluator that has just started, so at startup it must stay silent: the staleness is judged from the service start (the later of the two), and the stored streak is not armed until the server has been seen online. Once it has, the streak fires. Real-time clock so the seeded rows read as stale in the store. */ - var h = new Harness { Now = DateTime.UtcNow }; + var h = new Harness { Now = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc) }; var evaluator = h.Build(); await evaluator.EvaluateStoreAlertsAsync(postgres, LiveServerId, Name, connected: true, ct); @@ -5713,7 +5713,7 @@ clock so the seeded rows read as stale in the store. */ /* A second evaluator that never sees the server online is the server that stays down across a restart: silent at the start, fired by the staleness arm once the window has passed since the start. Taken before the capture-down rows below, which add a fresh success. */ - var hDown = new Harness { Now = DateTime.UtcNow }; + var hDown = new Harness { Now = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc) }; var downEvaluator = hDown.Build(); await downEvaluator.EvaluateStoreAlertsAsync(postgres, LiveServerId, Name, connected: false, ct); @@ -5780,7 +5780,7 @@ public async Task LiveStoreReads_RecentWindow_IsNewestByTimeFirst_WithLogIdBreak var bodySucceeded = false; try { - var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var utcNow = LiveClock.Now(); var older = utcNow.AddMinutes(-10); var newest = utcNow.AddMinutes(-1); @@ -5863,7 +5863,7 @@ public async Task LiveStoreReads_ExecuteBothAgQueries_AndReturnTheNewestSnapshot var bodySucceeded = false; try { - var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var utcNow = LiveClock.Now(); var older = utcNow.AddMinutes(-5); /* An older snapshot that must be filtered out by the MAX(collection_time) predicate, a current one @@ -5901,7 +5901,7 @@ public async Task LiveStoreReads_ExecuteBothAgQueries_AndReturnTheNewestSnapshot /* End to end through the sweep entry point, against rows that are genuinely fresh: the disconnect and the suspend are first sightings (silent baselines), and the 900-second lag is past the 300-second default, so exactly one alert lands. */ - var h = new Harness { Now = DateTime.UtcNow }; + var h = new Harness { Now = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc) }; var evaluator = h.Build(); await evaluator.EvaluateStoreAlertsAsync(postgres, LiveServerId, Name, connected: true, ct); diff --git a/Darling/Darling.Tests/DarlingWatermarkFloorPlanShapeLiveTests.cs b/Darling/Darling.Tests/DarlingWatermarkFloorPlanShapeLiveTests.cs index c912332026..e9d26f9652 100644 --- a/Darling/Darling.Tests/DarlingWatermarkFloorPlanShapeLiveTests.cs +++ b/Darling/Darling.Tests/DarlingWatermarkFloorPlanShapeLiveTests.cs @@ -80,7 +80,8 @@ sits outside the collector catalog and so outside ConvertToHypertablesAsync. */ var bodySucceeded = false; try { - var nowUtc = DateTime.UtcNow; + /* #5608: the end of the window is a fixed point inside one chunk, so the chunk layout the plan is read against is the same at any hour. */ + var nowUtc = DateTime.SpecifyKind(LiveClock.AnchoredEnd(), DateTimeKind.Utc); long logId = 4_469_000_000; /* 30 days of history, one row per collector per day, most of it old enough to compress — diff --git a/Darling/Darling.Tests/DarlingWatermarkFloorScanBoundLiveTests.cs b/Darling/Darling.Tests/DarlingWatermarkFloorScanBoundLiveTests.cs index 48b56b111a..78b1c0053a 100644 --- a/Darling/Darling.Tests/DarlingWatermarkFloorScanBoundLiveTests.cs +++ b/Darling/Darling.Tests/DarlingWatermarkFloorScanBoundLiveTests.cs @@ -75,7 +75,8 @@ read under test — the same race the plan-shape file's own live seeds guard aga var bodySucceeded = false; try { - var nowUtc = DateTime.UtcNow; + /* #5608: the end of the window is a fixed point inside one chunk, so the chunk layout the plan is read against is the same at any hour. */ + var nowUtc = DateTime.SpecifyKind(LiveClock.AnchoredEnd(), DateTimeKind.Utc); long logId = 4_469_200_000; /* 30 days of history, one row per collector per day — most chunks close to a day wide and old diff --git a/Darling/Darling.Tests/DarlingWatermarkIndexEquivalenceLiveTests.cs b/Darling/Darling.Tests/DarlingWatermarkIndexEquivalenceLiveTests.cs index 11f3e0246c..1041308ad2 100644 --- a/Darling/Darling.Tests/DarlingWatermarkIndexEquivalenceLiveTests.cs +++ b/Darling/Darling.Tests/DarlingWatermarkIndexEquivalenceLiveTests.cs @@ -58,7 +58,8 @@ public async Task ReadCollectorWatermarksAsync_MatchesTheBoundedGroupByOracle() var bodySucceeded = false; try { - var nowUtc = DateTime.UtcNow; + /* #5608: the end of the window is a fixed point inside one chunk, so the chunk layout the plan is read against is the same at any hour. */ + var nowUtc = DateTime.SpecifyKind(LiveClock.AnchoredEnd(), DateTimeKind.Utc); long logId = 4_150_100_000; /* Several real collector names (a mix of cadences), 10 days of history so most chunks diff --git a/Darling/Darling.Tests/EventWindowedReadsAreBoundedLivePostgresTests.cs b/Darling/Darling.Tests/EventWindowedReadsAreBoundedLivePostgresTests.cs index 2b5310c3e6..02e0d49d8e 100644 --- a/Darling/Darling.Tests/EventWindowedReadsAreBoundedLivePostgresTests.cs +++ b/Darling/Darling.Tests/EventWindowedReadsAreBoundedLivePostgresTests.cs @@ -265,7 +265,7 @@ public async Task TheFlooredReads_ReturnExactlyWhatTheUnflooredReadsReturned_Aga try { await DeleteSentinelRowsAsync(connection, ct); - var now = Micro(DateTime.UtcNow); + var now = Micro(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var marks = Marks(now); await SeedAsync(connection, now, marks, ct); @@ -365,7 +365,7 @@ public async Task TheFlooredReads_PlanAtMostThreeChunksForA24HourWindow_AgainstD try { await DeleteSentinelRowsAsync(connection, ct); - var now = Micro(DateTime.UtcNow); + var now = Micro(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var marks = Marks(now); await SeedAsync(connection, now, marks, ct); var floor = EventWindowFloor.For(marks.WindowStart); diff --git a/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs b/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs index e25362540e..aa4893ddef 100644 --- a/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs +++ b/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs @@ -382,7 +382,7 @@ public async Task TheBoundedReads_ReturnTheUnboundedReadsRows_AgainstDevPostgres try { await DeleteSentinelRowsAsync(connection, ct); - var now = Micro(DateTime.UtcNow); + var now = Micro(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await SeedAsync(connection, now, ct); /* The newest-row reads: identical rows for every compared sentinel. */ @@ -513,7 +513,7 @@ public async Task TheOverview_BuildsEachCardFromItsNewestReading_AgainstDevPostg try { await DeleteSentinelRowsAsync(connection, ct); - var now = Micro(DateTime.UtcNow); + var now = Micro(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await SeedAsync(connection, now, ct); var result = await DarlingFleetReader.GetFleetOverviewAsync(postgres, now.AddHours(-1), now, now, cancellationToken: ct); @@ -551,7 +551,7 @@ before the window (an index descent, FleetOlderCollectionSql), so the page can s Assert.Equal(ServerCollectionStatus.Offline.Word(), dark.Status); Assert.False(dark.AwaitingFirstCollection); Assert.NotNull(dark.LastCollectionTime); - Assert.InRange((DateTime.UtcNow - dark.LastCollectionTime!.Value).TotalDays, 4.5, 5.5); + Assert.InRange((DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc) - dark.LastCollectionTime!.Value).TotalDays, 4.5, 5.5); var never = cards[NeverCollected]; Assert.Null(never.CpuPercent); @@ -620,7 +620,7 @@ public async Task TheOlderCollectionRead_NeedsOnlyOneConnectionAtATime_AndIsCach try { await DeleteSentinelRowsAsync(connection, ct); - var now = Micro(DateTime.UtcNow); + var now = Micro(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await SeedAsync(connection, now, ct); var first = await DarlingFleetReader.ReadLastCollectionAsync(postgres, now, null, ct); @@ -680,7 +680,7 @@ public async Task TheBoundedReads_StopAtTheNewestChunk_AndPlanOnlyTheWindowsChun try { await DeleteSentinelRowsAsync(connection, ct); - var now = Micro(DateTime.UtcNow); + var now = Micro(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await SeedAsync(connection, now, ct); using (var analyze = new NpgsqlCommand("ANALYZE collect.memory_stats; ANALYZE collect.deadlocks; ANALYZE collect.servers;", connection)) { diff --git a/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs b/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs index 84215d1a01..9cc9f5f22e 100644 --- a/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs +++ b/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs @@ -207,7 +207,8 @@ public async Task TheShippedRead_PlansAsAnIndexOnlyScanOnTheCoveringIndex_Agains 100 ns, so a raw UtcNow does not survive the round trip and the #3579 stamp assertion below (tick-equality against what was seeded) would fail on any clock that is not itself microsecond-aligned — Windows' is not; the first CI run proved it by three ticks. */ - var rawNow = DateTime.UtcNow; + /* #5608: the end of the window is a fixed point inside one chunk, so the chunk layout the plan is read against is the same at any hour. */ + var rawNow = DateTime.SpecifyKind(LiveClock.AnchoredEnd(), DateTimeKind.Utc); var utcNow = DateTime.SpecifyKind(new DateTime(rawNow.Ticks - (rawNow.Ticks % 10)), DateTimeKind.Unspecified); for (var pass = 7; pass >= 0; pass--) { diff --git a/Darling/Darling.Tests/FrozenRollupLiveTests.cs b/Darling/Darling.Tests/FrozenRollupLiveTests.cs index 10103da5de..8c4e395bdb 100644 --- a/Darling/Darling.Tests/FrozenRollupLiveTests.cs +++ b/Darling/Darling.Tests/FrozenRollupLiveTests.cs @@ -133,7 +133,7 @@ is not vacuous. */ (DarlingCliCommands' rollup-backfill verb) — the service start path never runs a backfill slice on its own, so there is no third step to run here. */ await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); - await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, DateTime.UtcNow, ct); + await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), ct); var after = await ReadDailyTotalsAsync(connection, TimescaleSupport.QueryStatsDailyView, ServerId, D0, cutoff, ct); Assert.Equal(before, after); @@ -1869,7 +1869,7 @@ public async Task Upgrade_NoOutage_EmptySuccessor_RawPurgeStaysCovered() /* now() anchor, truncated to the hour so the seeded hourly rows line up with the buckets the probe walks. The legacy froze a few hours back (H-6..H-4); raw kept collecting every hour since, up to and including the current hour, but the successor never ran its first refresh. */ - var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var h = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0, DateTimeKind.Unspecified); for (var hoursAgo = 0; hoursAgo <= 6; hoursAgo++) @@ -1949,7 +1949,7 @@ public async Task Upgrade_NonEmptySuccessor_HoleInItsOwnSpan_SeamProbeStopsAtIts itself (H-72, H-71) holds no hole. The successor also materializes a much later bucket, H-2, but skips H-30 (T2) in between — a real hole, entirely inside the successor's own span above T1, which the seam probe must stop short of and never reach. */ - var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var h = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0, DateTimeKind.Unspecified); await InsertProcedureStatsAsync(connection, h.AddHours(-72), "seam_proc_legacy_floor", 900, 9, 3600, ct); @@ -2032,7 +2032,7 @@ public async Task Upgrade_RealOutageInsideTheFallbackWindow_RawPurgeStaysNotSafe var bodySucceeded = false; try { - var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var h = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0, DateTimeKind.Unspecified); var legacyLast = h.AddDays(-3); @@ -2111,6 +2111,7 @@ public async Task Upgrade_FallbackHorizon_GivesTheSameVerdictInAnySessionTimeZon var bodySucceeded = false; try { + /* The freeze verdict reads the store session's own clock, which the test clock override cannot move (#5608), so this seed stays on the real clock. */ var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); var h = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0, DateTimeKind.Unspecified); diff --git a/Darling/Darling.Tests/JobHistoryPerServerTopNLiveTests.cs b/Darling/Darling.Tests/JobHistoryPerServerTopNLiveTests.cs index 8879e84334..af39ef6c6d 100644 --- a/Darling/Darling.Tests/JobHistoryPerServerTopNLiveTests.cs +++ b/Darling/Darling.Tests/JobHistoryPerServerTopNLiveTests.cs @@ -199,7 +199,7 @@ public async Task DeregisteredServersRow_StillSurfaces_LikeTheOldLeftJoin() var bodySucceeded = false; try { - var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var sinceUtc = now.AddHours(-1); var at = now.AddMinutes(-5); @@ -255,7 +255,7 @@ public async Task NewSql_MatchesOldSql_ExactlyAcrossOffsetsTiesAndTheCapBoundary var bodySucceeded = false; try { - var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var sinceUtc = now.AddDays(-3); await DarlingMcpTestData.RegisterServerAsync(connection, ServerPlus60, "darling-4477-plus60", ct); @@ -413,7 +413,7 @@ public async Task ScopedRead_UsesTheIndex_WithNoExtraSort() { await DarlingMcpTestData.RegisterServerAsync(connection, ServerZeroOffset, "darling-4477-plan", ct); - var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var sinceUtc = now.AddDays(-3); for (var i = 0; i < 500; i++) @@ -667,7 +667,7 @@ private static async Task InsertServerPropertiesAsync( @"INSERT INTO server_properties (collection_id, collection_time, server_id, server_name, utc_offset_minutes) VALUES ($1, $2, $3, $4, $5)", connection); command.Parameters.AddWithValue(4_477_000_000L + serverId); - command.Parameters.AddWithValue(DarlingMcpTestData.Naive(DateTime.UtcNow)); + command.Parameters.AddWithValue(DarlingMcpTestData.Naive(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc))); command.Parameters.AddWithValue(serverId); command.Parameters.AddWithValue("srv" + serverId.ToString(CultureInfo.InvariantCulture)); command.Parameters.AddWithValue(offsetMinutes); diff --git a/Darling/Darling.Tests/JobHistoryWatermarkEpochLiveTests.cs b/Darling/Darling.Tests/JobHistoryWatermarkEpochLiveTests.cs index 4e42f9a7f5..52ba044030 100644 --- a/Darling/Darling.Tests/JobHistoryWatermarkEpochLiveTests.cs +++ b/Darling/Darling.Tests/JobHistoryWatermarkEpochLiveTests.cs @@ -70,7 +70,7 @@ public sealed class JobHistoryWatermarkEpochLiveTests private static async Task<(DateTime? Watermark, bool WatermarkFromUtcColumn, long? NumericWatermark, long ServerWatermarkMs, bool WatermarkCacheEligible, bool ServerWatermarkDiscarded)> ResolveAsync( DarlingCollectorRunner runner, ServerRuntime server, ICollectorDefinition definition, CancellationToken ct) { - var probe = MakeContext(server, DateTime.UtcNow); + var probe = MakeContext(server, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var task = (Task)ResolveMethod.MakeGenericMethod(typeof(TRow)).Invoke( runner, new object?[] { server, definition, probe, null, ct })!; await task; @@ -157,8 +157,8 @@ public async Task GetLastCollectedInstanceIdAsync_ReturnsTheNewestBatchsMax_NotT var bodySucceeded = false; try { - var oldEpochTime = DateTime.UtcNow.AddDays(-3); - var newEpochTime = DateTime.UtcNow.AddDays(-1); // outside the 6h bounded probe: unbounded fallback text + var oldEpochTime = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc).AddDays(-3); + var newEpochTime = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc).AddDays(-1); // outside the 6h bounded probe: unbounded fallback text await PlantBatchAsync( connection, server.ServerId, server.StorageName, oldEpochTime, oldEpochTime, Range(10_200_001, 10_200_040), ct); @@ -209,8 +209,8 @@ public async Task GetLastCollectedInstanceIdAsync_ReturnsTheNewestBatchsMax_OnTh var bodySucceeded = false; try { - var oldEpochTime = DateTime.UtcNow.AddHours(-4); // inside the 6h bounded window too - var newEpochTime = DateTime.UtcNow.AddHours(-1); // inside the 6h bounded probe window + var oldEpochTime = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc).AddHours(-4); // inside the 6h bounded window too + var newEpochTime = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc).AddHours(-1); // inside the 6h bounded probe window await PlantBatchAsync( connection, server.ServerId, server.StorageName, oldEpochTime, oldEpochTime, Range(10_200_001, 10_200_040), ct); @@ -271,7 +271,7 @@ public async Task ResolvedWatermarks_FeedTheFailbackQueryBound_FromTheStoredValu { // Batch A: the older epoch, planted first. var batchARunDateTime = new DateTime(2026, 9, 10, 0, 0, 0, DateTimeKind.Unspecified); - var batchACollectionTime = DateTime.UtcNow.AddDays(-2); + var batchACollectionTime = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc).AddDays(-2); await PlantBatchAsync( connection, server.ServerId, server.StorageName, batchACollectionTime, batchARunDateTime, Range(500, 520), ct); @@ -279,7 +279,7 @@ await PlantBatchAsync( // Batch B: the newer epoch, planted second — its run_datetime spans 09-20 through 09-21, its // newest row (12) landing last so the newest instance_id and the newest run_datetime agree. var batchBNewestRunDateTime = new DateTime(2026, 9, 21, 0, 0, 0, DateTimeKind.Unspecified); - var batchBCollectionTime = DateTime.UtcNow.AddHours(-2); + var batchBCollectionTime = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc).AddHours(-2); await using (var earlier = new NpgsqlCommand( "INSERT INTO collect.job_history " + "(job_history_id, collection_time, server_id, server_name, instance_id, job_id, job_name, run_status, run_datetime) " + @@ -327,7 +327,7 @@ await PlantBatchAsync( { ServerId = server.ServerId, ServerName = server.StorageName, - CollectionTime = DateTime.UtcNow, + CollectionTime = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), Deltas = new CollectorDeltaCalculator(), Target = server.Target, NumericWatermark = resolvedNumeric, @@ -430,7 +430,7 @@ public async Task WriteBatchAsync_FailbackToAPriorEpoch_StoresZeroDuplicateNatur var bodySucceeded = false; try { - var now = DateTime.UtcNow; + var now = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); /* A1: epoch A, instance_id 500..520 (21 rows), run_datetime spread D-3d .. D-1d, collected 3 days ago. AddTicks(7) plants sub-microsecond ticks on purpose (#4496): Windows' @@ -528,7 +528,7 @@ public async Task WriteBatchAsync_ALongRunningStep_IsStoredOnce_AndDroppedOnRepl var bodySucceeded = false; try { - var now = DateTime.UtcNow; + var now = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); /* AddTicks(7) plants sub-microsecond ticks on purpose (#4496), same reason as the A→B→A pin above: the replay batch's re-delivered RunDateTime must be truncated to whole microseconds before it matches the stored, already-truncated key. */ @@ -593,7 +593,7 @@ public async Task WriteBatchAsync_ASameKeyRowWithADifferentJobIdOrStepId_IsStore var bodySucceeded = false; try { - var now = DateTime.UtcNow; + var now = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); var runDateTime = now.AddHours(-1); var storedRow = MakeRow(700, "job-x", 0, runDateTime); var context1 = MakeContext(server, now.AddHours(-1)); @@ -657,7 +657,7 @@ public async Task DropAlreadyStoredJobHistoryRowsAsync_Cost_OverThirtyDailyChunk var bodySucceeded = false; try { - var now = DateTime.UtcNow; + var now = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); /* 30 daily chunks, ~2,000 rows/day, one server. */ for (var daysAgo = 29; daysAgo >= 0; daysAgo--) diff --git a/Darling/Darling.Tests/LatestCollectorOutcomePlanTests.cs b/Darling/Darling.Tests/LatestCollectorOutcomePlanTests.cs index c470165843..ca1f0d33c9 100644 --- a/Darling/Darling.Tests/LatestCollectorOutcomePlanTests.cs +++ b/Darling/Darling.Tests/LatestCollectorOutcomePlanTests.cs @@ -57,7 +57,7 @@ public async Task TheShippedRead_ExecutesOnlyTheNewestChunk_AgainstDevPostgres() await stop.ExecuteNonQueryAsync(ct); } - var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var utcNow = LiveClock.Now(); await SeedHistoryAsync(connection, utcNow, ct); using (var analyze = new NpgsqlCommand("ANALYZE collect.collection_log", connection)) diff --git a/Darling/Darling.Tests/LatestCpuReadShapeTests.cs b/Darling/Darling.Tests/LatestCpuReadShapeTests.cs index ca58d651ac..bb27a5ef3f 100644 --- a/Darling/Darling.Tests/LatestCpuReadShapeTests.cs +++ b/Darling/Darling.Tests/LatestCpuReadShapeTests.cs @@ -339,7 +339,7 @@ public async Task LatestCpuRead_ReturnsTheOracleRow_AtEveryUtcOffset_AgainstDevP /* Three polls per server one minute apart, each carrying one new ring-buffer sample: the steady state at the collector's one-minute frequency. The CPU values carry the poll ordinal so the row that comes back is identifiable rather than merely equal to the oracle's. */ - var newestUtc = TruncateToSeconds(DateTime.UtcNow); + var newestUtc = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); for (var i = 0; i < s_offsetMinutes.Length; i++) { for (var poll = 0; poll < 3; poll++) @@ -398,6 +398,8 @@ public async Task TheSampleTimePredicate_LosesEveryServerBehindTheStoresClock_Ag { await DeleteSentinelsAsync(connection, cancellationToken); + /* The trap predicate compares against the store's own now(), which the test clock override cannot move (#5608), + so this seed stays on the real clock. */ var newestUtc = TruncateToSeconds(DateTime.UtcNow); /* One server behind the store by more than the window, one in the store's own frame. */ @@ -445,7 +447,7 @@ public async Task LatestCpuRead_PicksTheNewestSampleInsideOneBatch_AgainstDevPos /* One poll catching up on a ring-buffer backlog: thirty samples under ONE collection_time. They go in NEWEST-sample-first, so heap order runs opposite to sample order and a scan of the tied group hands back the OLDEST sample unless the tiebreak sorts it. */ - var collectionTime = TruncateToSeconds(DateTime.UtcNow); + var collectionTime = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); for (var minutesAgo = 0; minutesAgo < 30; minutesAgo++) { await InsertSampleAsync( diff --git a/Darling/Darling.Tests/LatestValueLookbackTests.cs b/Darling/Darling.Tests/LatestValueLookbackTests.cs index 5e9b88c8d9..e4bf05c065 100644 --- a/Darling/Darling.Tests/LatestValueLookbackTests.cs +++ b/Darling/Darling.Tests/LatestValueLookbackTests.cs @@ -420,7 +420,7 @@ public async Task ALiveServer_ReportsOnlyWhatItStillHas_AndTheDrillDownListsTheS { await DeleteSentinelsAsync(connection, ct); - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await SeedLiveAndGhostAsync(connection, LiveServerId, end, ct); await using var postgres = NpgsqlDataSource.Create(connectionString!); @@ -487,7 +487,7 @@ public async Task AServerWhoseNewestSampleIsOlderThanTheLookback_EmitsNoLatestVa await DeleteSentinelsAsync(connection, ct); /* Everything a day and an hour old: the collectors stopped, or the server went away. */ - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); var stale = end.AddHours(-25); await InsertFileIoAsync(connection, StaleServerId, stale, "StaleDb", "StaleDb_data", 4096, ct); await InsertSizeAsync(connection, StaleServerId, stale, "StaleDb", 1, "ROWS", "StaleDb_data", 20_480, percentGrowth: true, "D:\\", 1_000_000, 10_000, ct); @@ -545,7 +545,7 @@ public async Task TraceFlagTurnedOffAfterLastConnect_DropsOutOfTheOnLoadWindow_W { await DeleteSentinelsAsync(connection, ct); - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await InsertTraceFlagAsync(connection, TraceFlagClearServerId, end.AddDays(-3), 3604, status: true, ct); await InsertTraceFlagAsync(connection, TraceFlagClearServerId, end.AddDays(-1), 2371, status: true, ct); await InsertTraceFlagAsync(connection, TraceFlagClearServerId, end.AddHours(-6), 2371, status: true, ct); @@ -589,7 +589,7 @@ public async Task AFlagTurnedOffWhileAnotherStaysOn_DropsOutAtTheNextCapture_Not { await DeleteSentinelsAsync(connection, ct); - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await InsertTraceFlagAsync(connection, TraceFlagClearServerId, end.AddDays(-1), 3604, status: true, ct); await InsertTraceFlagAsync(connection, TraceFlagClearServerId, end.AddDays(-1), 2371, status: true, ct); await InsertTraceFlagAsync(connection, TraceFlagClearServerId, end.AddHours(-6), 2371, status: true, ct); @@ -635,7 +635,7 @@ public async Task ServerWhoseOnlyTraceFlagRowIsOutsideTheWindow_EmitsNoTraceFlag { await DeleteSentinelsAsync(connection, ct); - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await InsertTraceFlagAsync(connection, TraceFlagAllOffServerId, end.AddDays(-3), 1222, status: true, ct); await using var postgres = NpgsqlDataSource.Create(connectionString!); @@ -668,7 +668,7 @@ public async Task AHistoricalWindow_ReadsTheStateAsItStoodAtItsOwnEnd_AgainstDev { await DeleteSentinelsAsync(connection, ct); - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await SeedLiveAndGhostAsync(connection, LiveServerId, end, ct); /* Twenty hours ago the dropped database was still there (last seen five hours before that), and none @@ -733,7 +733,7 @@ public async Task TheShippedDatabaseSizeRead_PlansOnlyTheLookbacksChunks_Against { await DeleteSentinelsAsync(connection, ct); - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); using (var seed = new NpgsqlCommand(@" INSERT INTO file_io_stats (collection_id, collection_time, server_id, server_name, database_name, file_name, file_type, size_mb) @@ -805,7 +805,7 @@ public async Task ACollectorSlowedToDaily_KeepsItsFactForTwoIntervals_ThenLosesI { await DeleteSentinelsAsync(connection, ct); - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await InsertOverrideAsync(connection, DailyServerId, "database_size_stats", 1440, ct); await InsertOverrideAsync(connection, DailyStaleServerId, "database_size_stats", 1440, ct); @@ -871,7 +871,7 @@ public async Task AnOnLoadOnlyCollector_AnchorsOnItsNewestCapture_AgainstDevPost { await DeleteSentinelsAsync(connection, ct); - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await InsertOverrideAsync(connection, OnLoadServerId, "file_io_stats", 0, ct); await InsertOverrideAsync(connection, OnLoadServerId, "memory_stats", 0, ct); @@ -921,7 +921,7 @@ public async Task AFleetWideOverride_WidensEveryServer_AndAServersOwnRowWins_Aga { await DeleteSentinelsAsync(connection, ct); - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await InsertOverrideAsync(connection, null, FleetOverrideCollector, 1440, ct); await InsertOverrideAsync(connection, FleetOverriddenServerId, FleetOverrideCollector, 60, ct); await InsertClerkAsync(connection, FleetServerId, end.AddHours(-30), "MEMORYCLERK_SQLBUFFERPOOL", 800, ct); @@ -973,7 +973,7 @@ public async Task ADailyCollectorsRead_StillPlansOnlyTheChunksItsLookbackSpans_A { await DeleteSentinelsAsync(connection, ct); - var end = TruncateToSeconds(DateTime.UtcNow); + var end = TruncateToSeconds(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc)); await InsertOverrideAsync(connection, PlanShapeDailyServerId, "database_size_stats", 1440, ct); using (var seed = new NpgsqlCommand(@" INSERT INTO database_size_stats diff --git a/Darling/Darling.Tests/LiveClockCensusTests.cs b/Darling/Darling.Tests/LiveClockCensusTests.cs index 0603f36888..4afdd39d9a 100644 --- a/Darling/Darling.Tests/LiveClockCensusTests.cs +++ b/Darling/Darling.Tests/LiveClockCensusTests.cs @@ -31,9 +31,11 @@ namespace Darling.Tests; /// that does both and does not call LiveClock. fails unless it is named in the allow list with a reason, and an /// allow entry that no longer matches fails, so the list only shrinks. /// -/// Not covered: tests that read plan SHAPE or chunk catalogs (about 70 files use EXPLAIN or the chunk -/// catalog and a wall clock). Their assertions are not block counts, so a boundary does not change the number they -/// compare, but each is a candidate for the same anchor; the PR body lists them. +/// The second rule covers plan shape and the chunk catalog. A live file that reads EXPLAIN, show_chunks, +/// timescaledb_information.chunks, compress_chunk or ChunkAppend and reads the wall clock directly asserts +/// on chunk layout, or may, and the layout moves with the hour. Such a file seeds and reads through , or +/// is named in s_allowedPlanShape with the reason its assertions cannot depend on where a boundary falls (it reads no +/// time, or the product reads the real clock itself and the assertion holds for any layout). A stale entry fails here too. /// [Trait("Stage", "Guard")] public sealed class LiveClockCensusTests @@ -44,10 +46,62 @@ public sealed class LiveClockCensusTests // says why the count cannot depend on where a chunk boundary falls. }; + private const string E2Audits = "e2 audits this file (#5608): lane e2 replaces this entry with its verdict or removes it when it converts the file."; + + /// + /// Live files that touch a plan or the chunk catalog and read the clock directly, with the reason each is safe as it is. + /// Every other such file goes through . + /// + private static readonly Dictionary s_allowedPlanShape = new(StringComparer.Ordinal) + { + ["AnomalyObjectStatsLatestSnapshotsLiveTests.cs"] = "The read has no time predicate and the plan assertion names an index, not a chunk. The two snapshots are two days apart, so they sit in different chunks at any hour, and nothing passes the seeded time to the product.", + ["CaggGroupIndexDropUpgradeLiveTests.cs"] = "The only clock reads time a stopwatch around an upgrade. No row is seeded from the wall clock.", + ["CollectionGapAtStartTests.cs"] = "Seeds 35 days of hourly rows ending now, so 'at least four chunks' holds at any hour. The read takes no time, and the plan assertion is the access path (an index scan, no sequential scan), which a boundary does not change.", + ["CollectionHealthAggregateTests.cs"] = "Seeds with SQL now() and the policy refresh runs on the server's own clock, so the window cannot be anchored from outside. Its assertions (at least N daily chunks over eight days, at least one covered chunk below the watermark and outside the hole's chunk) hold for any layout.", + ["CollectionLogSegmentByLiveTests.cs"] = "Seeds from midnight six days back to now minus 15 minutes and compresses chunks older than three days with SQL now(), so the product reads the real clock. The assertions are the segment-by setting of every chunk, the same for each chunk, so no layout changes the answer.", + ["DarlingDeltaSeederTests.cs"] = "SeedFromStoreAsync reads a window back from the product's own real clock, so a seed placed by an override falls outside it (the two live tests failed under 00:30Z). The rows sit minutes before now, and the assertions are delta values, not plan shape or chunk membership.", + ["DarlingModuleMapGroupedSourceLiveTests.cs"] = "The daily refresh statement reads now() - 2 days with no bound parameter, so the product reads the real clock and a seed placed by an override falls outside its window (the refresh test failed under 12:00Z). The rows sit three hours before now, far inside the window, and the assertions are the newest name per handle, not chunk layout.", + ["DarlingStoreUpgradeTests.cs"] = "The only clock read is a 30 second polling deadline for a marker file. No row is seeded from it.", + ["PerfmonRegroupLiveTests.cs"] = "The only direct clock reads are a poll deadline. The chunk-window question the class documents (the day-3 chunk dropping out of the count at midnight UTC) is audited by lane e2 together with PerfmonRegroupLiveSupport.cs.", + ["ParameterSensitiveDrillDownTextLiveTests.cs"] = E2Audits, + ["PayloadDimensionLiveTests.cs"] = E2Audits, + ["PerfmonRegroupLiveSupport.cs"] = E2Audits, + ["PgDeadlockRemaskTests.cs"] = E2Audits, + ["PgPlanCaptureCsvLiveTests.cs"] = E2Audits, + ["PgPlanCaptureLiveTests.cs"] = E2Audits, + ["PgServerConfigToolBoundTests.cs"] = E2Audits, + ["PgTargetConfigSnapshotBoundTests.cs"] = E2Audits, + ["PgTargetSeqScanTests.cs"] = E2Audits, + ["PlanRegressionDailyBuilderLiveTests.cs"] = E2Audits, + ["PlanRegressionDrillDownReuseLiveTests.cs"] = E2Audits, + ["QueryStoreBackfillCutChunkLiveTests.cs"] = E2Audits, + ["QueryStoreBackgroundIndexesLiveTests.cs"] = E2Audits, + ["QueryStoreIntervalPurgeRowCappedTests.cs"] = E2Audits, + ["QueryStoreIntervalWideBelowFloorLiveTests.cs"] = E2Audits, + ["QueryStoreIntervalWideGridLiveTests.cs"] = E2Audits, + ["QueryStoreTopLiteralEndStraddleLiveTests.cs"] = E2Audits, + ["QueryStoreTopMcpLiveTests.cs"] = E2Audits, + ["RawChunkIntervalReconcilerLiveTests.cs"] = E2Audits, + ["RawPurgeTriggerGateErrorTests.cs"] = E2Audits, + ["RawPurgeTriggerLiveTests.cs"] = E2Audits, + ["RawTablesLeaveCatalogSweepLiveTests.cs"] = E2Audits, + ["RollupBackfillLiveTests.cs"] = E2Audits, + ["RollupFloorCacheLiveTests.cs"] = E2Audits, + ["ServerListAndSummaryPlanShapeTests.cs"] = E2Audits, + ["StoreMetricsLatestSkipScanLiveTests.cs"] = E2Audits, + ["StoreSelfMetricsTests.cs"] = E2Audits, + ["TimescaleSupportTests.cs"] = E2Audits, + ["TopCpuQueriesTextLiveTests.cs"] = E2Audits, + ["TopTextLookupPlanLiveTests.cs"] = E2Audits, + }; + private static readonly Regex s_clock = new(@"\bDateTime(?:Offset)?\s*\.\s*(?:UtcNow|Now)\b", RegexOptions.Compiled); private static readonly Regex s_blocks = new(@"Shared (?:Hit|Read) Blocks", RegexOptions.Compiled); + /// A plan or the chunk catalog: the second family of assertions that depend on where a chunk boundary falls. + private static readonly Regex s_plan = new(@"EXPLAIN|show_chunks|timescaledb_information\.chunks|compress_chunk|ChunkAppend", RegexOptions.Compiled); + private static readonly Regex s_live = new(@"DARLING_TEST_PG|ScratchPostgres", RegexOptions.Compiled); private static readonly Regex s_anchor = new(@"\bLiveClock\s*\.", RegexOptions.Compiled); @@ -55,23 +109,33 @@ public sealed class LiveClockCensusTests /// Anti-vacuity: at least this many files count blocks, so a scan that stopped matching fails. private const int BlockCountingFloor = 2; - internal sealed record Row(string File, bool Live, bool ReadsClock, bool CountsBlocks, bool UsesAnchor); + /// Anti-vacuity for the plan and chunk-catalog rule: dozens of live files read a plan today. + private const int PlanReadingFloor = 20; + + internal sealed record Row(string File, bool Live, bool ReadsClock, bool CountsBlocks, bool UsesAnchor, bool ReadsPlanOrChunks = false); internal static Row ScanText(string file, string text) { var code = CSharpSourceWalker.StripCommentsAndStrings(text); - return new Row(file, s_live.IsMatch(text), s_clock.IsMatch(code), s_blocks.IsMatch(text), s_anchor.IsMatch(code)); + return new Row(file, s_live.IsMatch(text), s_clock.IsMatch(code), s_blocks.IsMatch(text), s_anchor.IsMatch(code), s_plan.IsMatch(text)); } internal static List Problems(IEnumerable rows, IReadOnlyDictionary allowed) => - rows.Where(r => r.Live && r.CountsBlocks && r.ReadsClock && !r.UsesAnchor && !allowed.ContainsKey(r.File)) + Problems(rows, allowed, r => r.CountsBlocks); + + internal static List Stale(IEnumerable rows, IReadOnlyDictionary allowed) => + Stale(rows, allowed, r => r.CountsBlocks); + + /// A live file of the given kind that reads the wall clock directly, not through , and is not allowed. + internal static List Problems(IEnumerable rows, IReadOnlyDictionary allowed, Func kind) => + rows.Where(r => r.Live && kind(r) && r.ReadsClock && !r.UsesAnchor && !allowed.ContainsKey(r.File)) .Select(r => r.File).ToList(); - internal static List Stale(IEnumerable rows, IReadOnlyDictionary allowed) + internal static List Stale(IEnumerable rows, IReadOnlyDictionary allowed, Func kind) { var byFile = rows.ToDictionary(r => r.File, StringComparer.Ordinal); return allowed.Keys - .Where(f => !byFile.TryGetValue(f, out var r) || !(r.Live && r.CountsBlocks && r.ReadsClock && !r.UsesAnchor)) + .Where(f => !byFile.TryGetValue(f, out var r) || !(r.Live && kind(r) && r.ReadsClock && !r.UsesAnchor)) .ToList(); } @@ -117,5 +181,47 @@ public void EveryLiveTestThatCountsBlocks_SeedsThroughTheLiveClock_OrIsAllowedWi "the scan found fewer live block-counting files than expected, so it stopped matching and 'no problems' would be vacuous"); } + [Fact] + public void TheDetector_FlagsAClockedPlanReader_AndIgnoresAnchoredAndUnclockedFiles() + { + var bare = ScanText("Bare.cs", "// DARLING_TEST_PG\nclass A { void B() { var e = DateTime.UtcNow; var s = \"EXPLAIN (COSTS OFF) SELECT 1\"; } }"); + var catalog = ScanText("Catalog.cs", "// ScratchPostgres\nclass A { void B() { var e = DateTimeOffset.UtcNow; var s = \"SELECT * FROM timescaledb_information.chunks\"; } }"); + var anchored = ScanText("Anchored.cs", "// DARLING_TEST_PG\nclass A { void B() { var e = LiveClock.AnchoredEnd(); var s = \"EXPLAIN SELECT 1\"; var n = DateTime.UtcNow; } }"); + var noClock = ScanText("NoClock.cs", "// DARLING_TEST_PG\nclass A { void B() { var s = \"EXPLAIN SELECT 1\"; } }"); + var notLive = ScanText("NotLive.cs", "class A { void B() { var e = DateTime.UtcNow; var s = \"EXPLAIN SELECT 1\"; } }"); + var noPlan = ScanText("NoPlan.cs", "// DARLING_TEST_PG\nclass A { void B() { var e = DateTime.UtcNow; } }"); + var rows = new[] { bare, catalog, anchored, noClock, notLive, noPlan }; + + Assert.Equal(new[] { "Bare.cs", "Catalog.cs" }, Problems(rows, new Dictionary(), r => r.ReadsPlanOrChunks)); + Assert.Equal(new[] { "Catalog.cs" }, Problems(rows, new Dictionary { ["Bare.cs"] = "reason" }, r => r.ReadsPlanOrChunks)); + Assert.Equal(new[] { "Gone.cs" }, Stale(rows, new Dictionary { ["Bare.cs"] = "reason", ["Gone.cs"] = "reason" }, r => r.ReadsPlanOrChunks)); + Assert.Equal(new[] { "Anchored.cs" }, Stale(rows, new Dictionary { ["Anchored.cs"] = "reason" }, r => r.ReadsPlanOrChunks)); + } + + [Fact] + public void EveryLiveTestThatReadsAPlanOrTheChunkCatalog_SeedsThroughTheLiveClock_OrIsAllowedWithAReason() + { + var rows = Scan(); + + var problems = Problems(rows, s_allowedPlanShape, r => r.ReadsPlanOrChunks); + Assert.True( + problems.Count == 0, + "A live Postgres test reads a plan or the chunk catalog and reads the wall clock without LiveClock (#5608). Its seeded window " + + "can straddle a chunk boundary, and the plan, the chunks scanned or the chunk membership then change with the hour the test " + + "runs. Seed and read through LiveClock.AnchoredEnd() (or LiveClock.Now() where the answer holds in any layout), or name the file " + + "in LiveClockCensusTests.s_allowedPlanShape with the reason no assertion in it depends on where a boundary falls:\n " + string.Join("\n ", problems)); + + var stale = Stale(rows, s_allowedPlanShape, r => r.ReadsPlanOrChunks); + Assert.True( + stale.Count == 0, + "These s_allowedPlanShape entries no longer name a live plan or chunk-catalog file that reads the wall clock without LiveClock; delete them:\n " + + string.Join("\n ", stale)); + + Assert.All(s_allowedPlanShape, entry => Assert.False(string.IsNullOrWhiteSpace(entry.Value), entry.Key + " needs a reason")); + Assert.True( + rows.Count(r => r.Live && r.ReadsPlanOrChunks) >= PlanReadingFloor, + "the scan found fewer live plan-reading files than expected, so it stopped matching and 'no problems' would be vacuous"); + } + private static string TestsRoot([CallerFilePath] string thisFile = "") => Path.GetDirectoryName(thisFile)!; } diff --git a/Darling/Darling.Tests/MaterializationHoleProbeLiveTests.cs b/Darling/Darling.Tests/MaterializationHoleProbeLiveTests.cs index 967ccdde38..7482330e90 100644 --- a/Darling/Darling.Tests/MaterializationHoleProbeLiveTests.cs +++ b/Darling/Darling.Tests/MaterializationHoleProbeLiveTests.cs @@ -213,7 +213,7 @@ private static async Task OpenAsync(string what) Assert.NotNull(materialization); /* Midnight UTC twenty days back: below every refresh policy's window. */ - var h0 = DateTime.SpecifyKind(DateTime.UtcNow.Date.AddDays(-20), DateTimeKind.Unspecified); + var h0 = LiveClock.Now().Date.AddDays(-20); return new Store(scratch, connection, h0, target, materialization.Value); } catch diff --git a/Darling/Darling.Tests/MaterializationHoleRepairTests.cs b/Darling/Darling.Tests/MaterializationHoleRepairTests.cs index 8c030d9563..f30cc6ab29 100644 --- a/Darling/Darling.Tests/MaterializationHoleRepairTests.cs +++ b/Darling/Darling.Tests/MaterializationHoleRepairTests.cs @@ -703,7 +703,7 @@ admitted the same way the legacy used to admit it. */ /* Twelve hours, all inside the raw horizon: H0-H2 collected and refreshed before the outage, H3-H4 the pre-outage TAIL (collected, never refreshed), H5-H8 the outage (nothing collected), H9-H10 collected after resume and refreshed by the first post-resume window. H11 is the still-filling current hour. */ - var h0 = DateTime.SpecifyKind(DateTime.UtcNow.Date.AddDays(-1), DateTimeKind.Unspecified); + var h0 = LiveClock.Now().Date.AddDays(-1); DateTime H(int n) => h0.AddHours(n); await InsertHoursAsync(connection, new[] { 0, 1, 2 }, H, ct); @@ -740,7 +740,7 @@ own trailing window. */ /* THE REPAIR: one forced refresh over [H3, H5) and nothing else. */ var log = new CapturingTestLogger(); - var summary = await TimescaleSupport.RepairMaterializationHolesAsync(connection, log, DateTime.UtcNow, ct); + var summary = await TimescaleSupport.RepairMaterializationHolesAsync(connection, log, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), ct); Assert.Equal(1, summary.HolesRepaired); Assert.Equal(2, summary.BucketsRepaired); @@ -782,7 +782,7 @@ assertion is what says the premise moved. */ carries the cost of proving it. A pass that fell silent here would read the same as one that never ran; a pass that returns this cannot. */ var quietLog = new CapturingTestLogger(); - var again = await TimescaleSupport.RepairMaterializationHolesAsync(connection, quietLog, DateTime.UtcNow, ct); + var again = await TimescaleSupport.RepairMaterializationHolesAsync(connection, quietLog, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), ct); Assert.Equal(0, again.HolesRepaired); Assert.Equal(0, again.Failures); Assert.Equal(0, again.HolesFound); @@ -814,7 +814,7 @@ carries the cost of proving it. A pass that fell silent here would read the same Assert.Equal(new[] { H(1) }, await ScanAsync(connection, target, materialization.Value, H(0), H(10), ct)); var forcedLog = new CapturingTestLogger(); - var forcedSummary = await TimescaleSupport.RepairMaterializationHolesAsync(connection, forcedLog, DateTime.UtcNow, ct); + var forcedSummary = await TimescaleSupport.RepairMaterializationHolesAsync(connection, forcedLog, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), ct); Assert.Equal(1, forcedSummary.HolesRepaired); Assert.Equal(1, forcedSummary.HolesForced); Assert.Equal(0, forcedSummary.HolesRemaining); @@ -892,10 +892,10 @@ public async Task OneHoleRepairedAndOneDeferred_TheTallyCarriesFoundRepairedAndD /* H0 materialized (the floor), H1..H26 collected and never refreshed (26 buckets of hole, cap + 2), H27 collected and refreshed (the ceiling). H28 = now - 32 h sits below the policy window; H0 = now - 60 h sits well inside the 4-day raw horizon. */ - var h0 = TimescaleSupport.AlignDown(DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified), TimescaleSupport.HourlyBucket).AddHours(-60); + var h0 = TimescaleSupport.AlignDown(LiveClock.Now(), TimescaleSupport.HourlyBucket).AddHours(-60); DateTime H(int n) => h0.AddHours(n); - Assert.True(H(28) < DateTime.UtcNow - TimescaleSupport.HourlyRefreshStartSpan, "the whole plant must sit below the hourly policy window"); - Assert.True(H(0) > DateTime.UtcNow - TimescaleSupport.RawRetentionSpan, "the whole plant must sit inside the raw horizon"); + Assert.True(H(28) < DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc) - TimescaleSupport.HourlyRefreshStartSpan, "the whole plant must sit below the hourly policy window"); + Assert.True(H(0) > DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc) - TimescaleSupport.RawRetentionSpan, "the whole plant must sit inside the raw horizon"); await InsertHoursAsync(connection, new[] { 0 }, H, ct); await RefreshAsync(connection, view, H(0), H(1), ct); @@ -911,7 +911,7 @@ sits well inside the 4-day raw horizon. */ /* PASS ONE: one hole found (26 buckets), 24 repaired, 2 deferred past the cap — on the same pass. */ var log = new CapturingTestLogger(); - var first = await TimescaleSupport.RepairMaterializationHolesAsync(connection, log, DateTime.UtcNow, ct); + var first = await TimescaleSupport.RepairMaterializationHolesAsync(connection, log, DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), ct); Assert.Equal(0, first.Failures); Assert.Equal(1, first.HolesFound); @@ -937,7 +937,7 @@ sits well inside the 4-day raw horizon. */ Assert.Equal(new[] { H(25), H(26) }, await ScanAsync(connection, target, materialization.Value, H(0), H(27), ct)); /* PASS TWO: the deferred remainder is the one hole now — found, repaired whole, nothing deferred. */ - var second = await TimescaleSupport.RepairMaterializationHolesAsync(connection, new CapturingTestLogger(), DateTime.UtcNow, ct); + var second = await TimescaleSupport.RepairMaterializationHolesAsync(connection, new CapturingTestLogger(), DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), ct); Assert.Equal(0, second.Failures); Assert.Equal(1, second.HolesFound); Assert.Equal(2, second.BucketsFound); @@ -949,7 +949,7 @@ sits well inside the 4-day raw horizon. */ Assert.Equal(Enumerable.Range(0, 28).Select(H).ToArray(), await MaterializedBucketsAsync(connection, view, ct)); /* PASS THREE: the zero-hole walk, with the walk still in the tally. */ - var third = await TimescaleSupport.RepairMaterializationHolesAsync(connection, new CapturingTestLogger(), DateTime.UtcNow, ct); + var third = await TimescaleSupport.RepairMaterializationHolesAsync(connection, new CapturingTestLogger(), DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc), ct); Assert.Equal(0, third.HolesFound); Assert.Equal(0, third.BucketsFound); Assert.Equal(0, third.HolesRepaired); diff --git a/Darling/Darling.Tests/MaterializationHoleScanShapeTests.cs b/Darling/Darling.Tests/MaterializationHoleScanShapeTests.cs index b3d6befd10..1a7cec2532 100644 --- a/Darling/Darling.Tests/MaterializationHoleScanShapeTests.cs +++ b/Darling/Darling.Tests/MaterializationHoleScanShapeTests.cs @@ -216,8 +216,8 @@ private static async Task SeedAsync(string what) await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); - var h0 = DateTime.SpecifyKind(DateTime.UtcNow.Date.AddDays(-6), DateTimeKind.Unspecified); - Assert.True(DateTime.UtcNow - h0.AddHours(48) > TimescaleSupport.DailyRefreshStartSpan, "the seed must sit below every refresh policy's window"); + var h0 = LiveClock.Now().Date.AddDays(-6); + Assert.True(DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc) - h0.AddHours(48) > TimescaleSupport.DailyRefreshStartSpan, "the seed must sit below every refresh policy's window"); var collected = Enumerable.Range(0, 48).Where(h => h is not (>= 5 and <= 8) and not 20).ToArray(); await PlantHoursAsync(connection, h0, collected, ct); From f2089630e6f4b1beb0df8b7ebb997fa6c8163a56 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:42:07 -0400 Subject: [PATCH 5/7] ForcePlanFailuresAccessPathTests reads the clock through LiveClock.Now (#5608) --- Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs b/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs index 9cc9f5f22e..03c8c394ba 100644 --- a/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs +++ b/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs @@ -207,8 +207,7 @@ public async Task TheShippedRead_PlansAsAnIndexOnlyScanOnTheCoveringIndex_Agains 100 ns, so a raw UtcNow does not survive the round trip and the #3579 stamp assertion below (tick-equality against what was seeded) would fail on any clock that is not itself microsecond-aligned — Windows' is not; the first CI run proved it by three ticks. */ - /* #5608: the end of the window is a fixed point inside one chunk, so the chunk layout the plan is read against is the same at any hour. */ - var rawNow = DateTime.SpecifyKind(LiveClock.AnchoredEnd(), DateTimeKind.Utc); + var rawNow = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); var utcNow = DateTime.SpecifyKind(new DateTime(rawNow.Ticks - (rawNow.Ticks % 10)), DateTimeKind.Unspecified); for (var pass = 7; pass >= 0; pass--) { From 2848555af677544616415c15957c9c37a85fe3ee Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:42:57 -0400 Subject: [PATCH 6/7] Drop the census allow entries for the files lane e2 converted (#5608) --- Darling/Darling.Tests/LiveClockCensusTests.cs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Darling/Darling.Tests/LiveClockCensusTests.cs b/Darling/Darling.Tests/LiveClockCensusTests.cs index 4afdd39d9a..0708f78ef4 100644 --- a/Darling/Darling.Tests/LiveClockCensusTests.cs +++ b/Darling/Darling.Tests/LiveClockCensusTests.cs @@ -63,17 +63,14 @@ public sealed class LiveClockCensusTests ["DarlingModuleMapGroupedSourceLiveTests.cs"] = "The daily refresh statement reads now() - 2 days with no bound parameter, so the product reads the real clock and a seed placed by an override falls outside its window (the refresh test failed under 12:00Z). The rows sit three hours before now, far inside the window, and the assertions are the newest name per handle, not chunk layout.", ["DarlingStoreUpgradeTests.cs"] = "The only clock read is a 30 second polling deadline for a marker file. No row is seeded from it.", ["PerfmonRegroupLiveTests.cs"] = "The only direct clock reads are a poll deadline. The chunk-window question the class documents (the day-3 chunk dropping out of the count at midnight UTC) is audited by lane e2 together with PerfmonRegroupLiveSupport.cs.", - ["ParameterSensitiveDrillDownTextLiveTests.cs"] = E2Audits, ["PayloadDimensionLiveTests.cs"] = E2Audits, ["PerfmonRegroupLiveSupport.cs"] = E2Audits, ["PgDeadlockRemaskTests.cs"] = E2Audits, ["PgPlanCaptureCsvLiveTests.cs"] = E2Audits, ["PgPlanCaptureLiveTests.cs"] = E2Audits, ["PgServerConfigToolBoundTests.cs"] = E2Audits, - ["PgTargetConfigSnapshotBoundTests.cs"] = E2Audits, ["PgTargetSeqScanTests.cs"] = E2Audits, ["PlanRegressionDailyBuilderLiveTests.cs"] = E2Audits, - ["PlanRegressionDrillDownReuseLiveTests.cs"] = E2Audits, ["QueryStoreBackfillCutChunkLiveTests.cs"] = E2Audits, ["QueryStoreBackgroundIndexesLiveTests.cs"] = E2Audits, ["QueryStoreIntervalPurgeRowCappedTests.cs"] = E2Audits, @@ -81,18 +78,13 @@ public sealed class LiveClockCensusTests ["QueryStoreIntervalWideGridLiveTests.cs"] = E2Audits, ["QueryStoreTopLiteralEndStraddleLiveTests.cs"] = E2Audits, ["QueryStoreTopMcpLiveTests.cs"] = E2Audits, - ["RawChunkIntervalReconcilerLiveTests.cs"] = E2Audits, ["RawPurgeTriggerGateErrorTests.cs"] = E2Audits, ["RawPurgeTriggerLiveTests.cs"] = E2Audits, ["RawTablesLeaveCatalogSweepLiveTests.cs"] = E2Audits, - ["RollupBackfillLiveTests.cs"] = E2Audits, - ["RollupFloorCacheLiveTests.cs"] = E2Audits, ["ServerListAndSummaryPlanShapeTests.cs"] = E2Audits, ["StoreMetricsLatestSkipScanLiveTests.cs"] = E2Audits, ["StoreSelfMetricsTests.cs"] = E2Audits, ["TimescaleSupportTests.cs"] = E2Audits, - ["TopCpuQueriesTextLiveTests.cs"] = E2Audits, - ["TopTextLookupPlanLiveTests.cs"] = E2Audits, }; private static readonly Regex s_clock = new(@"\bDateTime(?:Offset)?\s*\.\s*(?:UtcNow|Now)\b", RegexOptions.Compiled); From 2641503a44e56d5266f3962fe2a37a2933cacc70 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:04:28 -0400 Subject: [PATCH 7/7] Census allow entries carry their own reasons, and the force-plan access-path test seeds from the clock the product reads (#5608) The 22 entries that said another lane would audit the file now state why the file is safe as it is. The force-plan access-path test went back to the real clock for its seed: the product read takes its two-hour window from its own clock, so a seed placed by the test clock override fell outside it. --- .../ForcePlanFailuresAccessPathTests.cs | 6 ++- Darling/Darling.Tests/LiveClockCensusTests.cs | 49 +++++++++---------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs b/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs index 03c8c394ba..6569896611 100644 --- a/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs +++ b/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs @@ -207,7 +207,11 @@ public async Task TheShippedRead_PlansAsAnIndexOnlyScanOnTheCoveringIndex_Agains 100 ns, so a raw UtcNow does not survive the round trip and the #3579 stamp assertion below (tick-equality against what was seeded) would fail on any clock that is not itself microsecond-aligned — Windows' is not; the first CI run proved it by three ticks. */ - var rawNow = DateTime.SpecifyKind(LiveClock.Now(), DateTimeKind.Utc); + /* #5608: the real clock on purpose, not LiveClock. GetForcePlanFailuresAsync below takes its two-hour + window from the product's own clock, so a seed placed by DARLING_TEST_CLOCK_UTC falls outside it and + the read comes back empty (five of five runs at a 00:30 override, with every plan assertion above + already passed). The plan assertions hold across a chunk boundary, so the seed needs no anchor. */ + var rawNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); var utcNow = DateTime.SpecifyKind(new DateTime(rawNow.Ticks - (rawNow.Ticks % 10)), DateTimeKind.Unspecified); for (var pass = 7; pass >= 0; pass--) { diff --git a/Darling/Darling.Tests/LiveClockCensusTests.cs b/Darling/Darling.Tests/LiveClockCensusTests.cs index 0708f78ef4..0073fd7de8 100644 --- a/Darling/Darling.Tests/LiveClockCensusTests.cs +++ b/Darling/Darling.Tests/LiveClockCensusTests.cs @@ -46,8 +46,6 @@ public sealed class LiveClockCensusTests // says why the count cannot depend on where a chunk boundary falls. }; - private const string E2Audits = "e2 audits this file (#5608): lane e2 replaces this entry with its verdict or removes it when it converts the file."; - /// /// Live files that touch a plan or the chunk catalog and read the clock directly, with the reason each is safe as it is. /// Every other such file goes through . @@ -62,29 +60,30 @@ public sealed class LiveClockCensusTests ["DarlingDeltaSeederTests.cs"] = "SeedFromStoreAsync reads a window back from the product's own real clock, so a seed placed by an override falls outside it (the two live tests failed under 00:30Z). The rows sit minutes before now, and the assertions are delta values, not plan shape or chunk membership.", ["DarlingModuleMapGroupedSourceLiveTests.cs"] = "The daily refresh statement reads now() - 2 days with no bound parameter, so the product reads the real clock and a seed placed by an override falls outside its window (the refresh test failed under 12:00Z). The rows sit three hours before now, far inside the window, and the assertions are the newest name per handle, not chunk layout.", ["DarlingStoreUpgradeTests.cs"] = "The only clock read is a 30 second polling deadline for a marker file. No row is seeded from it.", - ["PerfmonRegroupLiveTests.cs"] = "The only direct clock reads are a poll deadline. The chunk-window question the class documents (the day-3 chunk dropping out of the count at midnight UTC) is audited by lane e2 together with PerfmonRegroupLiveSupport.cs.", - ["PayloadDimensionLiveTests.cs"] = E2Audits, - ["PerfmonRegroupLiveSupport.cs"] = E2Audits, - ["PgDeadlockRemaskTests.cs"] = E2Audits, - ["PgPlanCaptureCsvLiveTests.cs"] = E2Audits, - ["PgPlanCaptureLiveTests.cs"] = E2Audits, - ["PgServerConfigToolBoundTests.cs"] = E2Audits, - ["PgTargetSeqScanTests.cs"] = E2Audits, - ["PlanRegressionDailyBuilderLiveTests.cs"] = E2Audits, - ["QueryStoreBackfillCutChunkLiveTests.cs"] = E2Audits, - ["QueryStoreBackgroundIndexesLiveTests.cs"] = E2Audits, - ["QueryStoreIntervalPurgeRowCappedTests.cs"] = E2Audits, - ["QueryStoreIntervalWideBelowFloorLiveTests.cs"] = E2Audits, - ["QueryStoreIntervalWideGridLiveTests.cs"] = E2Audits, - ["QueryStoreTopLiteralEndStraddleLiveTests.cs"] = E2Audits, - ["QueryStoreTopMcpLiveTests.cs"] = E2Audits, - ["RawPurgeTriggerGateErrorTests.cs"] = E2Audits, - ["RawPurgeTriggerLiveTests.cs"] = E2Audits, - ["RawTablesLeaveCatalogSweepLiveTests.cs"] = E2Audits, - ["ServerListAndSummaryPlanShapeTests.cs"] = E2Audits, - ["StoreMetricsLatestSkipScanLiveTests.cs"] = E2Audits, - ["StoreSelfMetricsTests.cs"] = E2Audits, - ["TimescaleSupportTests.cs"] = E2Audits, + ["PerfmonRegroupLiveTests.cs"] = "The only direct clock reads are a poll deadline. Seeds are whole UTC days from the run start date, the policy test seeds three to five days back (the two-days-back version was #5579's 00:00 to 03:00 failure), and the reach test counts chunks from the database clock, so the day-3 chunk cannot drop out of a count at midnight UTC.", + ["ForcePlanFailuresAccessPathTests.cs"] = "The shipped read takes its two-hour window from the product's own real clock, so a seed placed by the test clock override falls outside it and the read comes back empty (it failed five of five runs at a 00:30 override). The plan assertions hold across a chunk boundary: at that override they all passed before the read, so the real-clock seed needs no anchor.", + ["PayloadDimensionLiveTests.cs"] = "Seeds dates 45 and 400 days back, mid-chunk, and asserts on dimension rows, not on chunk or block counts. The clock is only a stamp.", + ["PerfmonRegroupLiveSupport.cs"] = "Every seed is a whole UTC day taken from the run start date, and the compress reach is read from the database clock, which a test override cannot move. The window does not move with the hour (#5579's two-days-back seed was the 00:00 to 03:00 defect; the policy test now seeds three to five days back).", + ["PgDeadlockRemaskTests.cs"] = "Seeds midday of a date or fixed dates, so no seeded window straddles a day boundary.", + ["PgPlanCaptureCsvLiveTests.cs"] = "The clock is only the collection stamp. The class needs an external target and skips on a rig without one.", + ["PgPlanCaptureLiveTests.cs"] = "The clock is only the collection stamp. The class needs an external target and skips on a rig without one.", + ["PgServerConfigToolBoundTests.cs"] = "The chunk-scan asserts are written as ranges, so they hold for any layout, and the reader's own read takes the real clock, so an override cannot reach it.", + ["PgTargetSeqScanTests.cs"] = "No EXPLAIN or chunks (the 'Seq Scan' is planted plan JSON), and the analyze tool rejects an end ahead of the real clock, so the test must stay on the real clock (an override at 12:00Z made it fail).", + ["PlanRegressionDailyBuilderLiveTests.cs"] = "Fixed March dates on plain tables, including the block count. The one real-clock fact seeds five whole days back and its trigger reads the database clock.", + ["QueryStoreBackfillCutChunkLiveTests.cs"] = "Fixed June dates. The one test that follows the real clock waits until the floor is 3 minutes clear of a boundary.", + ["QueryStoreBackgroundIndexesLiveTests.cs"] = "Seeds from SQL now() and compares the same store before and after the indexes, so the chunk layout is the same on both sides.", + ["QueryStoreIntervalPurgeRowCappedTests.cs"] = "Plain tables, fixed dates for the plan asserts, and whole-day margins around the retention horizon.", + ["QueryStoreIntervalWideBelowFloorLiveTests.cs"] = "Anchored to the date (6 days back); the clock is only a stamp.", + ["QueryStoreIntervalWideGridLiveTests.cs"] = "Fixed 15 September window; the clock is only a write stamp.", + ["QueryStoreTopLiteralEndStraddleLiveTests.cs"] = "Fixed 15 September window; the clock is only a write stamp.", + ["QueryStoreTopMcpLiveTests.cs"] = "Fixed 15 September window; the clock is only a write stamp.", + ["RawPurgeTriggerGateErrorTests.cs"] = "Seeds from the database clock one to ten days back, and its asserts are relative (chunks before greater than 0, after less than before, equal before and after).", + ["RawPurgeTriggerLiveTests.cs"] = "Seeds from the database clock one to ten days back, and its asserts are relative (chunks before greater than 0, after less than before, equal before and after).", + ["RawTablesLeaveCatalogSweepLiveTests.cs"] = "Seeds from the database clock one to ten days back, and its asserts are relative (chunks before greater than 0, after less than before, equal before and after).", + ["ServerListAndSummaryPlanShapeTests.cs"] = "The newest row is one minute old and the assert is at most two chunks visited, true on either side of midnight. No time goes to the product.", + ["StoreMetricsLatestSkipScanLiveTests.cs"] = "A plain table; it asserts index names, not chunks.", + ["StoreSelfMetricsTests.cs"] = "Seeds are placed at midday of a date (#1972); the clock is only a stamp.", + ["TimescaleSupportTests.cs"] = "Seeds are placed at midday of a date (#1972); the clock is only a stamp.", }; private static readonly Regex s_clock = new(@"\bDateTime(?:Offset)?\s*\.\s*(?:UtcNow|Now)\b", RegexOptions.Compiled);