From bd7dff99f9eb4a22613183e1516d6751bf75047a Mon Sep 17 00:00:00 2001 From: reference-week Date: Sat, 5 Sep 2026 01:46:02 +0200 Subject: [PATCH] test(cli): every row kind arrives in any order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A re-read appends, so one session's lines sit in different orders on two machines. The report is meant to be blind to that, and the existing check is four records and a single reversal. Four records cannot cover what has been added since. `by_flow` now keys an interval-derived row on a `FlowInterval` object, a tool-stated one on the skill's name, and the remainder on a symbol; `by_agent` keys two of its three rows on symbols. Mixed key kinds in one Map are exactly where insertion order leaks into output, since rows are ranked by size and ties broken on the row's own key — and nothing failed when either tie-break was broken. Verified against the real thing before writing it: 30,222 records of a live sink, shuffled within every day file, produced a byte-identical report to the unshuffled run. Three consecutive runs were byte-identical, and all eleven `--axis` artefacts stable. fast-check over 300 permutations of a fixture carrying every row kind once, plus a second test asserting the fixture exercises all of them — a permutation of records that produce one kind of row proves nothing about mixed keys. The `pickDeterministically` mutation survived the first fixture, which had no two records sharing a `billed_request_id`, so the function was never reached. Adding the pair is what made that guard real. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VWNxk63AGKkqE8HRqHLjGp --- .../plan.md | 55 ++++++ .../cost-report-order.property.unit.test.ts | 178 ++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 aidd_docs/tasks/2026_09/2026_09_05_every-row-kind-arrives-in-any-order/plan.md create mode 100644 cli/tests/domain/models/cost-report-order.property.unit.test.ts diff --git a/aidd_docs/tasks/2026_09/2026_09_05_every-row-kind-arrives-in-any-order/plan.md b/aidd_docs/tasks/2026_09/2026_09_05_every-row-kind-arrives-in-any-order/plan.md new file mode 100644 index 00000000..5e30c312 --- /dev/null +++ b/aidd_docs/tasks/2026_09/2026_09_05_every-row-kind-arrives-in-any-order/plan.md @@ -0,0 +1,55 @@ +--- +status: done +--- + +# Every row kind arrives in any order + +## Why + +A re-read appends, so one session's lines sit in different orders on two machines and nothing +a consumer does controls it. The report is supposed to be blind to that, and +`cost-report.unit.test.ts` already checks it — with four records and a single reversal. + +Four records cannot cover what has been added since. Two axes now mix key kinds inside one +`Map`, which is exactly where insertion order leaks into output: + +| Axis | Row | Key | +| --- | --- | --- | +| `by_flow` | the journal witnessed it | the `FlowInterval` object | +| `by_flow` | only the tool named it | the skill's name, a string | +| `by_flow` | joined neither | a symbol | +| `by_agent` | the tool named the agent | the agent's name | +| `by_agent` | the main thread | a symbol | +| `by_agent` | the tool never names one | a second symbol | + +Rows are ranked by size and ties are broken on the row's own key. A tie-break that forgets +part of the key, or a survivor picked as `group[0]`, answers differently for the same records +in a different order — and nothing failed when either was broken. + +## Verified before it was written + +30,222 records of a live sink, shuffled within every day file with a seeded shuffle, produced +a **byte-identical** report to the unshuffled run (`md5` equal). Three consecutive unshuffled +runs were byte-identical too, and all eleven `--axis` artefacts were stable across runs. The +property holds at scale; this is the guard that keeps it. + +## The test + +`cost-report-order.property.unit.test.ts`, fast-check over 300 permutations of a fixture +carrying every row kind once: both flow kinds and the remainder, all three agent +attributions, a named prompt and one that named none, two tools, a pair with identical +figures that only a tie-break can order, and two records sharing one `billed_request_id` so +`pickDeterministically` is actually reached. + +A second test asserts the fixture exercises what the property is about — a permutation of +records that produce one kind of row proves nothing about mixed keys. + +| Guard | Mutation that killed it | +| --- | --- | +| the agent tie-break keeps the row's key | return `""` — 1 | +| the flow tie-break keeps the row's key | return `""` — 1 | +| a superseded group's survivor does not depend on arrival | `pickDeterministically` returns `candidates[0]` — 1 | + +The third survived the first version of this fixture, which had no two records sharing a +`billed_request_id` — the function was never reached. Adding the pair is what made the guard +real, and the coverage test is what stops that regressing quietly. diff --git a/cli/tests/domain/models/cost-report-order.property.unit.test.ts b/cli/tests/domain/models/cost-report-order.property.unit.test.ts new file mode 100644 index 00000000..50d65b0a --- /dev/null +++ b/cli/tests/domain/models/cost-report-order.property.unit.test.ts @@ -0,0 +1,178 @@ +import "../../../src/domain/tools/ai/claude.js"; +import "../../../src/domain/tools/ai/codex.js"; +import * as fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { + buildCostReport, + type CostReportInput, + type CostReportSessionJournal, +} from "../../../src/domain/models/cost-report.js"; +import type { TelemetrySinkRecord } from "../../../src/domain/models/telemetry-sink-record.js"; + +/** + * A re-read appends, so one session's lines sit in different orders on two machines, and + * nothing a consumer does controls it. `cost-report.unit.test.ts` already reverses four + * records; this covers what that fixture cannot. + * + * Every row kind added since is keyed differently from the others, and mixed key kinds in + * one `Map` are exactly where insertion order leaks into output: `by_flow` keys an + * interval-derived row on the `FlowInterval` object and a tool-stated one on the skill's + * name, and `by_agent` keys two of its three rows on symbols. A report that ranks by size + * then breaks ties on a row key cannot be allowed to answer differently because the records + * arrived shuffled. + * + * Verified against the real thing before it was written: 30,222 records of a live sink, + * shuffled within every day file, produced a byte-identical report. This is the guard for + * it, over permutations rather than one reversal. + */ +const AT = "2026-08-18T10:00:00Z"; +const LATER = "2026-08-18T11:30:00Z"; + +const NAMES_AGENTS = { + localRead: { tokenCounters: true, amount: false, toolStatedStep: true, agentName: true }, + export: null, + journalAttributable: true, + taskAttributable: true, +} as const; + +const NAMES_NO_AGENT = { + localRead: { tokenCounters: true, amount: false, toolStatedStep: false, agentName: false }, + export: null, + journalAttributable: false, + taskAttributable: false, +} as const; + +const DECLARED = [ + { tool: "claude", coverage: "covered", capability: NAMES_AGENTS }, + { tool: "codex", coverage: "covered", capability: NAMES_NO_AGENT }, +] as const; + +/** One session the journal witnessed, so an interval-derived flow row exists beside a + * tool-stated one — the two key kinds this property is about. */ +const JOURNALS: readonly CostReportSessionJournal[] = [ + { + vendorId: "s-witnessed", + tool: "claude-code", + writtenPaths: [], + taskIntervals: [], + flowIntervals: [ + { + skill: "aidd-orchestrator:01-sdlc", + startMs: Date.parse("2026-08-18T09:00:00Z"), + endMs: Date.parse("2026-08-18T10:30:00Z"), + }, + ], + }, +]; + +function record(overrides: Partial): TelemetrySinkRecord { + return { + sink_schema_version: 2, + kind: "request", + provenance: "local-read", + tool: "claude", + vendor_id: "s-witnessed", + vendor_field: "sessionId", + step_attribution: "unattributed", + event_timestamp: AT, + cost_usd: 1, + ...overrides, + }; +} + +/** Every row kind the report can produce, once each: an interval-derived flow and a + * tool-stated one, all three agent attributions, a named prompt and one that named none, + * two models, two tools, and a pair with identical figures whose order only a tie-break can + * decide. */ +const RECORDS: readonly TelemetrySinkRecord[] = [ + // Inside the witnessed flow, agent named by the tool. + record({ turn_id: "a", agent_name: "aidd-dev:executor", model: "opus", prompt_id: "p-1" }), + // Inside the witnessed flow, no agent — the main thread, since claude names agents. + record({ turn_id: "b", model: "haiku", prompt_id: "p-1" }), + // Outside every interval, but the tool named an orchestrating skill: a tool-stated flow. + record({ + turn_id: "c", + vendor_id: "s-unwitnessed", + event_timestamp: LATER, + step_attribution: "tool-stated", + step: "aidd-orchestrator:01-sdlc", + model: "opus", + }), + record({ + turn_id: "d", + vendor_id: "s-unwitnessed", + event_timestamp: LATER, + step_attribution: "tool-stated", + step: "aidd-orchestrator:02-backlog", + model: "haiku", + }), + // A tool that never names an agent: the third agent row, and it must not read as a main + // thread however the records arrive. + record({ turn_id: "e", tool: "codex", vendor_id: "s-codex", event_timestamp: LATER }), + // Two rows with identical figures, so only the tie-break on the row's own key can order + // them — the case repetition alone never catches. + record({ turn_id: "f", model: "zulu", cost_usd: 3, prompt_id: "p-2" }), + record({ turn_id: "g", model: "alpha", cost_usd: 3, prompt_id: "p-3" }), + // One billed call two routes saw, with equal counters and different content: only + // `pickDeterministically` can choose a survivor, and picking `group[0]` would make the + // choice depend on which line the day file happened to list first. + record({ + turn_id: "h", + billed_request_id: "req-1", + model: "opus", + cost_usd: 4, + input_tokens: 10, + agent_name: "Explore", + }), + record({ + turn_id: "i", + billed_request_id: "req-1", + model: "opus", + cost_usd: 4, + input_tokens: 10, + prompt_id: "p-4", + }), +]; + +function reportOf(records: readonly TelemetrySinkRecord[]): string { + const input: CostReportInput = { + fromDay: "2026-08-17", + toDay: "2026-08-21", + records, + journals: JOURNALS, + declaredTools: DECLARED, + undatedRecords: 0, + unreadableLines: 0, + measurementEnabled: true, + }; + return JSON.stringify(buildCostReport(input)); +} + +describe("buildCostReport — every row kind, arriving in any order", () => { + it("answers the same report for every permutation of the same records", () => { + const expected = reportOf(RECORDS); + + fc.assert( + fc.property(fc.shuffledSubarray([...RECORDS], { minLength: RECORDS.length }), (shuffled) => { + expect(reportOf(shuffled)).toBe(expected); + }), + { numRuns: 300 } + ); + }); + + // The fixture has to actually exercise what the property is about: a permutation of records + // that produce only one kind of row proves nothing about mixed keys. + it("covers both flow row kinds and all three agent attributions", () => { + const report = JSON.parse(reportOf(RECORDS)) as { + byFlows: { attribution: string }[]; + byAgents: { attribution: string }[]; + }; + + expect(new Set(report.byFlows.map((row) => row.attribution))).toEqual( + new Set(["journal-interval", "tool-stated", "unattributed"]) + ); + expect(new Set(report.byAgents.map((row) => row.attribution))).toEqual( + new Set(["tool-stated", "main-thread", "not-stated"]) + ); + }); +});