test(cli): every row kind arrives in any order

aidd_docs/tasks/2026_09/2026_09_05_every-row-kind-arrives-in-any-order/plan.md

@@ -0,0 +1,55 @@
+---
+status: done
+---
+
+# Every row kind arrives in any order
+
+## Why
+
+A re-read appends, so one session's lines sit in different orders on two machines and nothing
+a consumer does controls it. The report is supposed to be blind to that, and
+`cost-report.unit.test.ts` already checks it — with four records and a single reversal.
+
+Four records cannot cover what has been added since. Two axes now mix key kinds inside one
+`Map`, which is exactly where insertion order leaks into output:
+
+| Axis | Row | Key |
+| --- | --- | --- |
+| `by_flow` | the journal witnessed it | the `FlowInterval` object |
+| `by_flow` | only the tool named it | the skill's name, a string |
+| `by_flow` | joined neither | a symbol |
+| `by_agent` | the tool named the agent | the agent's name |
+| `by_agent` | the main thread | a symbol |
+| `by_agent` | the tool never names one | a second symbol |
+
+Rows are ranked by size and ties are broken on the row's own key. A tie-break that forgets
+part of the key, or a survivor picked as `group[0]`, answers differently for the same records
+in a different order — and nothing failed when either was broken.
+
+## Verified before it was written
+
+30,222 records of a live sink, shuffled within every day file with a seeded shuffle, produced
+a **byte-identical** report to the unshuffled run (`md5` equal). Three consecutive unshuffled
+runs were byte-identical too, and all eleven `--axis` artefacts were stable across runs. The
+property holds at scale; this is the guard that keeps it.
+
+## The test
+
+`cost-report-order.property.unit.test.ts`, fast-check over 300 permutations of a fixture
+carrying every row kind once: both flow kinds and the remainder, all three agent
+attributions, a named prompt and one that named none, two tools, a pair with identical
+figures that only a tie-break can order, and two records sharing one `billed_request_id` so
+`pickDeterministically` is actually reached.
+
+A second test asserts the fixture exercises what the property is about — a permutation of
+records that produce one kind of row proves nothing about mixed keys.
+
+| Guard | Mutation that killed it |
+| --- | --- |
+| the agent tie-break keeps the row's key | return `""` — 1 |
+| the flow tie-break keeps the row's key | return `""` — 1 |
+| a superseded group's survivor does not depend on arrival | `pickDeterministically` returns `candidates[0]` — 1 |
+
+The third survived the first version of this fixture, which had no two records sharing a
+`billed_request_id` — the function was never reached. Adding the pair is what made the guard
+real, and the coverage test is what stops that regressing quietly.

cli/tests/domain/models/cost-report-order.property.unit.test.ts

@@ -0,0 +1,178 @@
+import "../../../src/domain/tools/ai/claude.js";
+import "../../../src/domain/tools/ai/codex.js";
+import * as fc from "fast-check";
+import { describe, expect, it } from "vitest";
+import {
+  buildCostReport,
+  type CostReportInput,
+  type CostReportSessionJournal,
+} from "../../../src/domain/models/cost-report.js";
+import type { TelemetrySinkRecord } from "../../../src/domain/models/telemetry-sink-record.js";
+
+/**
+ * A re-read appends, so one session's lines sit in different orders on two machines, and
+ * nothing a consumer does controls it. `cost-report.unit.test.ts` already reverses four
+ * records; this covers what that fixture cannot.
+ *
+ * Every row kind added since is keyed differently from the others, and mixed key kinds in
+ * one `Map` are exactly where insertion order leaks into output: `by_flow` keys an
+ * interval-derived row on the `FlowInterval` object and a tool-stated one on the skill's
+ * name, and `by_agent` keys two of its three rows on symbols. A report that ranks by size
+ * then breaks ties on a row key cannot be allowed to answer differently because the records
+ * arrived shuffled.
+ *
+ * Verified against the real thing before it was written: 30,222 records of a live sink,
+ * shuffled within every day file, produced a byte-identical report. This is the guard for
+ * it, over permutations rather than one reversal.
+ */
+const AT = "2026-08-18T10:00:00Z";
+const LATER = "2026-08-18T11:30:00Z";
+
+const NAMES_AGENTS = {
+  localRead: { tokenCounters: true, amount: false, toolStatedStep: true, agentName: true },
+  export: null,
+  journalAttributable: true,
+  taskAttributable: true,
+} as const;
+
+const NAMES_NO_AGENT = {
+  localRead: { tokenCounters: true, amount: false, toolStatedStep: false, agentName: false },
+  export: null,
+  journalAttributable: false,
+  taskAttributable: false,
+} as const;
+
+const DECLARED = [
+  { tool: "claude", coverage: "covered", capability: NAMES_AGENTS },
+  { tool: "codex", coverage: "covered", capability: NAMES_NO_AGENT },
+] as const;
+
+/** One session the journal witnessed, so an interval-derived flow row exists beside a
+ * tool-stated one — the two key kinds this property is about. */
+const JOURNALS: readonly CostReportSessionJournal[] = [
+  {
+    vendorId: "s-witnessed",
+    tool: "claude-code",
+    writtenPaths: [],
+    taskIntervals: [],
+    flowIntervals: [
+      {
+        skill: "aidd-orchestrator:01-sdlc",
+        startMs: Date.parse("2026-08-18T09:00:00Z"),
+        endMs: Date.parse("2026-08-18T10:30:00Z"),
+      },
+    ],
+  },
+];
+
+function record(overrides: Partial<TelemetrySinkRecord>): TelemetrySinkRecord {
+  return {
+    sink_schema_version: 2,
+    kind: "request",
+    provenance: "local-read",
+    tool: "claude",
+    vendor_id: "s-witnessed",
+    vendor_field: "sessionId",
+    step_attribution: "unattributed",
+    event_timestamp: AT,
+    cost_usd: 1,
+    ...overrides,
+  };
+}
+
+/** Every row kind the report can produce, once each: an interval-derived flow and a
+ * tool-stated one, all three agent attributions, a named prompt and one that named none,
+ * two models, two tools, and a pair with identical figures whose order only a tie-break can
+ * decide. */
+const RECORDS: readonly TelemetrySinkRecord[] = [
+  // Inside the witnessed flow, agent named by the tool.
+  record({ turn_id: "a", agent_name: "aidd-dev:executor", model: "opus", prompt_id: "p-1" }),
+  // Inside the witnessed flow, no agent — the main thread, since claude names agents.
+  record({ turn_id: "b", model: "haiku", prompt_id: "p-1" }),
+  // Outside every interval, but the tool named an orchestrating skill: a tool-stated flow.
+  record({
+    turn_id: "c",
+    vendor_id: "s-unwitnessed",
+    event_timestamp: LATER,
+    step_attribution: "tool-stated",
+    step: "aidd-orchestrator:01-sdlc",
+    model: "opus",
+  }),
+  record({
+    turn_id: "d",
+    vendor_id: "s-unwitnessed",
+    event_timestamp: LATER,
+    step_attribution: "tool-stated",
+    step: "aidd-orchestrator:02-backlog",
+    model: "haiku",
+  }),
+  // A tool that never names an agent: the third agent row, and it must not read as a main
+  // thread however the records arrive.
+  record({ turn_id: "e", tool: "codex", vendor_id: "s-codex", event_timestamp: LATER }),
+  // Two rows with identical figures, so only the tie-break on the row's own key can order
+  // them — the case repetition alone never catches.
+  record({ turn_id: "f", model: "zulu", cost_usd: 3, prompt_id: "p-2" }),
+  record({ turn_id: "g", model: "alpha", cost_usd: 3, prompt_id: "p-3" }),
+  // One billed call two routes saw, with equal counters and different content: only
+  // `pickDeterministically` can choose a survivor, and picking `group[0]` would make the
+  // choice depend on which line the day file happened to list first.
+  record({
+    turn_id: "h",
+    billed_request_id: "req-1",
+    model: "opus",
+    cost_usd: 4,
+    input_tokens: 10,
+    agent_name: "Explore",
+  }),
+  record({
+    turn_id: "i",
+    billed_request_id: "req-1",
+    model: "opus",
+    cost_usd: 4,
+    input_tokens: 10,
+    prompt_id: "p-4",
+  }),
+];
+
+function reportOf(records: readonly TelemetrySinkRecord[]): string {
+  const input: CostReportInput = {
+    fromDay: "2026-08-17",
+    toDay: "2026-08-21",
+    records,
+    journals: JOURNALS,
+    declaredTools: DECLARED,
+    undatedRecords: 0,
+    unreadableLines: 0,
+    measurementEnabled: true,
+  };
+  return JSON.stringify(buildCostReport(input));
+}
+
+describe("buildCostReport — every row kind, arriving in any order", () => {
+  it("answers the same report for every permutation of the same records", () => {
+    const expected = reportOf(RECORDS);
+
+    fc.assert(
+      fc.property(fc.shuffledSubarray([...RECORDS], { minLength: RECORDS.length }), (shuffled) => {
+        expect(reportOf(shuffled)).toBe(expected);
+      }),
+      { numRuns: 300 }
+    );
+  });
+
+  // The fixture has to actually exercise what the property is about: a permutation of records
+  // that produce only one kind of row proves nothing about mixed keys.
+  it("covers both flow row kinds and all three agent attributions", () => {
+    const report = JSON.parse(reportOf(RECORDS)) as {
+      byFlows: { attribution: string }[];
+      byAgents: { attribution: string }[];
+    };
+
+    expect(new Set(report.byFlows.map((row) => row.attribution))).toEqual(
+      new Set(["journal-interval", "tool-stated", "unattributed"])
+    );
+    expect(new Set(report.byAgents.map((row) => row.attribution))).toEqual(
+      new Set(["tool-stated", "main-thread", "not-stated"])
+    );
+  });
+});