diff --git a/aidd_docs/product/metrics-contract.md b/aidd_docs/product/metrics-contract.md index e851e1f4e..fd16627c8 100644 --- a/aidd_docs/product/metrics-contract.md +++ b/aidd_docs/product/metrics-contract.md @@ -734,6 +734,39 @@ ignores it exactly as it would any other field it does not recognize. truncated mid-write, or a host whose files carry no such identifier, which is every tool but Claude Code today. Never read as "no prompt ran". +#### `prompt_skill` +- **Type**: string. +- **Present**: conditional — Claude Code only, and only where a `Skill` call was + made inside the record's own prompt. +- **Meaning**: the skill that call invoked. The same fact the run journal writes + as `step_start`'s `turn_id`, read from the transcript instead of from a hook. + The first call wins where a prompt made several: a prompt that invokes two + skills invoked the second from inside the first, and it is named for the work + it began. +- **Why it exists**: the report never re-reads a transcript — it reads this sink + and the journals beside it — so an observation only a transcript holds has to + be written down when it is read or it is gone. It names a step for a session + the journal never saw, which is every session that ran before the hook was + installed. Measured on one machine: 28 such prompts across 22 days, 318 records + named by that route and by nothing else. +- **Scoped to one transcript**: Claude Code writes a session's subagents to their + own files, and a prompt is often spread across several — measured on one + machine, 1,038 of 5,564 prompts appear in more than one file. A record names the + first skill invoked inside its prompt *in the file it sits in*. A subagent that + invoked its own skill did that work under that skill; merging the files first + would have to pick one answer for both, and neither is true of both. +- **Not a duplicate of `step`**: that one reads `attributionSkill`, which Claude + Code writes per message — exact where it appears and sparse where it does not. + Measured inside the window one skill demonstrably ran: 142 lines carry counters + and 20 carry that field. Its absence is therefore not the tool saying no skill + ran, and naming the skill a prompt invoked contradicts nothing it states. +- **Never a judgement**: which step a record belongs to is derived fresh on every + report, from this and from the journal together. The journal wins where both + name a skill for the same prompt — it was written by a hook the host fired, + where this is read back afterwards. +- **If absent**: the record's prompt invoked no skill, the chain reached no + prompt at all, or the tool is not Claude Code. Never read as "no skill ran". + #### `duration_ms` - **Type**: number. - **Present**: conditional — measured so far only on Claude Code's export diff --git a/cli/src/application/use-cases/telemetry/report-cost-use-case.ts b/cli/src/application/use-cases/telemetry/report-cost-use-case.ts index 330152e66..630d29cf4 100644 --- a/cli/src/application/use-cases/telemetry/report-cost-use-case.ts +++ b/cli/src/application/use-cases/telemetry/report-cost-use-case.ts @@ -276,11 +276,27 @@ function matchOnPrompt( record: TelemetrySinkRecord, byPrompt: ReadonlyMap | undefined ): { readonly source: "prompt-matched"; readonly step: string } | null { - if (record.prompt_id === undefined || byPrompt === undefined) return null; - const step = byPrompt.get(record.prompt_id); + const step = journalNamedStep(record, byPrompt) ?? record.prompt_skill; return step === undefined ? null : { source: "prompt-matched", step }; } +/** What the run journal says the record's own prompt opened, asked first. + * + * Both sides name the same fact from the same identifier, so they can only disagree if one + * of them is wrong — and the journal was written by a hook the host itself fired, while + * `prompt_skill` is read back off a transcript afterwards. The reading with a witness wins. + * + * A session the journal never saw at all has no answer here and falls through to the + * record's own. Measured on the real sink: 28 prompts across 22 days ran before the hook + * was installed, and 318 records are named by that route and by nothing else. */ +function journalNamedStep( + record: TelemetrySinkRecord, + byPrompt: ReadonlyMap | undefined +): string | undefined { + if (record.prompt_id === undefined || byPrompt === undefined) return undefined; + return byPrompt.get(record.prompt_id); +} + /** Every record's step, taken from the journal rather than from the record. * * **A judgement is derived; only an observation is trusted from disk.** `step_attribution` diff --git a/cli/src/domain/formats/claude-code-transcript.ts b/cli/src/domain/formats/claude-code-transcript.ts index 8c870c4c8..b762bccbd 100644 --- a/cli/src/domain/formats/claude-code-transcript.ts +++ b/cli/src/domain/formats/claude-code-transcript.ts @@ -53,6 +53,7 @@ interface ClaudeTranscriptLine { readonly model?: unknown; readonly id?: unknown; readonly usage?: ClaudeUsage; + readonly content?: unknown; }; } @@ -188,6 +189,26 @@ function uuidOf(line: string): string | undefined { * arrived, or a cycle a damaged file leaves behind, must end the walk rather than search * forever. A hop cap would also terminate, but it would silently stop answering for a * legitimately deep chain, which is the kind of number nobody could ever justify. */ +/** The skill a `Skill` tool call on this line invokes, or `undefined` for every other line. + * + * Only a `Skill` call names a step. Every other tool call is work done inside whatever step + * was already running, and reading one as a start would name a skill for a prompt that + * invoked none. `input.skill` is the field Claude Code puts the name in - the same one + * `skill-detection.cjs` reads out of the hook payload, so the transcript and the run + * journal name a step identically. */ +function skillInvokedOn(line: ClaudeTranscriptLine): string | undefined { + const content = line.message?.content; + if (!Array.isArray(content)) return undefined; + for (const part of content) { + if (typeof part !== "object" || part === null) continue; + const call = part as { type?: unknown; name?: unknown; input?: { skill?: unknown } }; + if (call.type !== "tool_use" || call.name !== "Skill") continue; + const skill = asString(call.input?.skill); + if (skill !== undefined) return skill; + } + return undefined; +} + function resolvePromptId( startUuid: string | undefined, parents: ReadonlyMap, @@ -258,6 +279,10 @@ class ClaudeCodeTranscriptAccumulator implements TranscriptLineAccumulator { // chain from a call to its prompt runs through lines that carry no counters at all. private readonly parents = new Map(); private readonly prompts = new Map(); + /** Every `Skill` call the transcript holds, in the order it holds them, paired with the + * line that made it. Resolved to prompts in `build()` and not here, for the reason the + * class already resolves prompts there: a walk run mid-stream reads a half-built chain. */ + private readonly skillCalls: { readonly uuid: string; readonly skill: string }[] = []; push(line: string): void { this.rememberLinks(line); @@ -279,12 +304,36 @@ class ClaudeCodeTranscriptAccumulator implements TranscriptLineAccumulator { if (parent !== undefined) this.parents.set(uuid, parent); const prompt = asString(parsed.promptId); if (prompt !== undefined) this.prompts.set(uuid, prompt); + const skill = skillInvokedOn(parsed); + if (skill !== undefined) this.skillCalls.push({ uuid, skill }); + } + + /** The skill each prompt invoked, first call wins. + * + * The first and not the last: a prompt that invokes two skills invoked the second from + * inside the first, and the prompt is named for the work it began - the same rule + * `promptToSkill` follows over the run journal's own `step_start` lines, so the two + * sources cannot disagree about a prompt they both saw. */ + private skillByPrompt(): ReadonlyMap { + const byPrompt = new Map(); + for (const { uuid, skill } of this.skillCalls) { + const prompt = resolvePromptId(uuid, this.parents, this.prompts); + if (prompt !== undefined && !byPrompt.has(prompt)) byPrompt.set(prompt, skill); + } + return byPrompt; } build(): readonly LocalCostCandidateRecord[] { + const skillByPrompt = this.skillByPrompt(); return [...this.byKey.entries()].map(([key, record]) => { const promptId = resolvePromptId(this.uuidByKey.get(key), this.parents, this.prompts); - return promptId === undefined ? record : { ...record, prompt_id: promptId }; + if (promptId === undefined) return record; + const promptSkill = skillByPrompt.get(promptId); + return { + ...record, + prompt_id: promptId, + ...(promptSkill === undefined ? {} : { prompt_skill: promptSkill }), + }; }); } } diff --git a/cli/src/domain/models/telemetry-sink-record.ts b/cli/src/domain/models/telemetry-sink-record.ts index cc2a55e45..304800620 100644 --- a/cli/src/domain/models/telemetry-sink-record.ts +++ b/cli/src/domain/models/telemetry-sink-record.ts @@ -87,6 +87,30 @@ export interface TelemetrySinkRecord { * * Absent wherever a tool's files cannot say, which is every host but Claude Code today. */ readonly prompt_id?: string; + /** The skill a `Skill` call invoked inside this record's own prompt — the same fact the + * run journal writes as `step_start`'s `turn_id`, seen from the transcript instead. + * + * Stored because the report never re-reads a transcript: it reads this sink and the + * journals beside it, so an observation only a transcript holds has to be written down + * when it is read or it is gone. An observation, and never a judgement — which step a + * record belongs to is `report-cost-use-case.ts`'s question, derived fresh every run + * from this and from the journal together. + * + * Scoped to the transcript the record itself sits in, which is what the reader accumulates: + * Claude Code writes a session's subagents to their own files under + * `/subagents/`, and a prompt is often spread across several — measured on one + * machine, 1,038 of 5,564 prompts appear in more than one file. A subagent that invoked its + * own skill did that work under that skill, so its records name it, while the main + * transcript's records name whatever the main flow invoked. Merging the files first would + * have to pick one of the two for both, and neither choice is true of both. + * + * It does not duplicate `step`. That one reads `attributionSkill`, which Claude Code + * writes per message: exact where it appears and sparse where it does not. Measured on + * the one orchestrated session captured, 2026-09-04, inside the window + * `aidd-dev:01-plan` demonstrably ran, 142 lines carry counters and 20 carry that field. + * So its absence is not the tool saying no skill ran, and naming the skill a prompt + * invoked contradicts nothing the tool states. */ + readonly prompt_skill?: string; /** How `step` came to be known. Never optional, for the same reason `provenance` is not: * an absent field would be read as "no step ran", which is exactly the assertion nothing * on a transcript or a journal can support. See `domain/models/step-attribution.ts`. */ diff --git a/cli/tests/application/use-cases/telemetry/report-cost-use-case.unit.test.ts b/cli/tests/application/use-cases/telemetry/report-cost-use-case.unit.test.ts index e93c211e5..e648c18a8 100644 --- a/cli/tests/application/use-cases/telemetry/report-cost-use-case.unit.test.ts +++ b/cli/tests/application/use-cases/telemetry/report-cost-use-case.unit.test.ts @@ -646,6 +646,66 @@ describe("a report that catches the sink up first", () => { ); }); + /** + * A session whose journal never opened the step, because the hook was not installed when + * it ran. The record still carries what its own transcript said: the skill a `Skill` call + * invoked inside that prompt. Same fact, same identifier, read from the other side. + * + * Measured on the real sink: 28 such prompts across 22 days, 318 records named this way + * and by nothing else. + */ + it("attributes on the skill the record's own prompt invoked, where no journal saw it", async () => { + journals.set(SESSION, journalAt("2026-08-18T09:00:00Z")); + await sink.appendRecord( + record({ + vendor_id: SESSION, + event_timestamp: "2026-08-18T10:00:00.000Z", + prompt_id: "p-abc", + prompt_skill: "aidd-dev:01-plan", + }), + STORED_ON + ); + + const built = await reportWith().execute({ ...BASE_OPTIONS, period: PERIOD }); + + expect(built.bySteps).toContainEqual( + expect.objectContaining({ attribution: "prompt-matched", step: "aidd-dev:01-plan" }) + ); + }); + + // The journal is the stronger of the two: it was written by a hook the host itself fired, + // while the transcript is read back afterwards. They can only disagree if one of them is + // wrong, and the reading with a witness wins. + it("keeps the journal's own answer when both sides name a skill for the same prompt", async () => { + const journal = journalAt("2026-08-18T09:00:00Z"); + journals.set(SESSION, { + ...journal, + boundaries: [ + { + type: "step_start", + at: "2026-08-18T11:00:00Z", + skill: "aidd-pm:04-spec", + turn_id: "p-abc", + }, + ], + }); + await sink.appendRecord( + record({ + vendor_id: SESSION, + event_timestamp: "2026-08-18T10:00:00.000Z", + prompt_id: "p-abc", + prompt_skill: "aidd-dev:01-plan", + }), + STORED_ON + ); + + const built = await reportWith().execute({ ...BASE_OPTIONS, period: PERIOD }); + + expect(built.bySteps).toContainEqual( + expect.objectContaining({ attribution: "prompt-matched", step: "aidd-pm:04-spec" }) + ); + }); + it("derives a stored record's step from the journal rather than trusting the stored one", async () => { const at = "2026-08-18T10:00:00.000Z"; const journal = journalAt("2026-08-18T09:00:00Z"); diff --git a/cli/tests/domain/formats/claude-code-transcript.unit.test.ts b/cli/tests/domain/formats/claude-code-transcript.unit.test.ts index 6839d34b7..940079cec 100644 --- a/cli/tests/domain/formats/claude-code-transcript.unit.test.ts +++ b/cli/tests/domain/formats/claude-code-transcript.unit.test.ts @@ -74,6 +74,119 @@ describe("mapClaudeCodeTranscriptToSinkRecords — the prompt a billed call belo expect(record?.prompt_id).toBeUndefined(); }); + /** + * The skill a `Skill` call started inside this record's own prompt. + * + * `attributionSkill`, which the record's `step` already reads, is exact where it appears + * and sparse where it does not: measured on the one orchestrated session captured, + * 2026-09-04, inside the window `aidd-dev:01-plan` demonstrably ran, 142 billed lines + * carry counters and 20 carry that field. Its absence is not the tool saying no skill + * ran, so naming the skill a prompt invoked contradicts nothing it states. + * + * Stored rather than judged: which step a record belongs to is the reader's question, + * and this is the observation it answers from — the same fact the run journal writes as + * `step_start`'s `turn_id`, seen from the transcript instead. + */ + it("names the skill a Skill call invoked inside the record's own prompt", () => { + const content = chain([ + { type: "user", uuid: "u1", promptId: "p-abc" }, + { + type: "assistant", + uuid: "a1", + parentUuid: "u1", + sessionId: SID, + message: { + content: [{ type: "tool_use", name: "Skill", input: { skill: "aidd-dev:01-plan" } }], + }, + }, + assistantLine({ uuid: "a2", parentUuid: "a1" }), + ]); + + const [record] = mapClaudeCodeTranscriptToSinkRecords(content); + + expect(record?.prompt_skill).toBe("aidd-dev:01-plan"); + }); + + // A record whose prompt started no skill states none, rather than borrowing the last one + // seen: two prompts are two prompts however their moments overlap, which is the whole + // reason this reads a prompt and not a moment. + it("names no skill for a prompt that invoked none", () => { + const content = chain([ + { type: "user", uuid: "u1", promptId: "p-one" }, + { + type: "assistant", + uuid: "a1", + parentUuid: "u1", + sessionId: SID, + message: { + content: [{ type: "tool_use", name: "Skill", input: { skill: "aidd-dev:01-plan" } }], + }, + }, + { type: "user", uuid: "u2", parentUuid: "a1", promptId: "p-two" }, + assistantLine({ uuid: "a2", parentUuid: "u2" }), + ]); + + const records = mapClaudeCodeTranscriptToSinkRecords(content); + + expect(records.at(-1)?.prompt_skill).toBeUndefined(); + }); + + // The first, never the last: a prompt that invokes two skills invoked the second from + // inside the first, and the prompt is named for the work it began. + it("keeps the first skill a prompt invoked when it invoked more than one", () => { + const content = chain([ + { type: "user", uuid: "u1", promptId: "p-abc" }, + { + type: "assistant", + uuid: "a1", + parentUuid: "u1", + sessionId: SID, + message: { + content: [ + { type: "tool_use", name: "Skill", input: { skill: "aidd-orchestrator:01-sdlc" } }, + ], + }, + }, + { + type: "assistant", + uuid: "a2", + parentUuid: "a1", + sessionId: SID, + message: { + content: [{ type: "tool_use", name: "Skill", input: { skill: "aidd-pm:04-spec" } }], + }, + }, + assistantLine({ uuid: "a3", parentUuid: "a2" }), + ]); + + const [record] = mapClaudeCodeTranscriptToSinkRecords(content); + + expect(record?.prompt_skill).toBe("aidd-orchestrator:01-sdlc"); + }); + + // Only a `Skill` call names a step. Every other tool call is work done inside whatever + // step was already running, and reading one as a step start would name a skill for a + // prompt that never invoked any. + it("ignores a tool call that is not a Skill call", () => { + const content = chain([ + { type: "user", uuid: "u1", promptId: "p-abc" }, + { + type: "assistant", + uuid: "a1", + parentUuid: "u1", + sessionId: SID, + message: { + content: [{ type: "tool_use", name: "Bash", input: { skill: "aidd-dev:01-plan" } }], + }, + }, + assistantLine({ uuid: "a2", parentUuid: "a1" }), + ]); + + const [record] = mapClaudeCodeTranscriptToSinkRecords(content); + + expect(record?.prompt_skill).toBeUndefined(); + }); + // A transcript is appended to by a live process and can be truncated mid-write; a parent // pointing at a line that never arrived must end the walk, not search forever. it("stops at a parent the transcript does not hold, rather than looping", () => {