From 2d7e42b1015fe58ba5043d9c33ac77a820419797 Mon Sep 17 00:00:00 2001 From: Baptiste LAFOURCADE Date: Thu, 10 Sep 2026 05:25:18 +0200 Subject: [PATCH] fix(telemetry): opencode reads journal.cjs where the build puts it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `opencode-plugin.js` spawned `./journal.cjs` as its own sibling. Since the build stopped materialising every hook script flat into `.opencode/plugin/` and started namespacing them under `.opencode/hooks//`, that path names nothing: every spawn exited MODULE_NOT_FOUND, the plugin's own try/catch swallowed it, and OpenCode recorded no session at all while still reading as covered. Measured, not inferred: a real `opencode run` in an installed project wrote zero journal lines; the same run after repointing the path wrote session_start, task_declared and turn_end, and `telemetry read` returned 2 requests, 16,303 input and 39,104 cache-read tokens. The literal now matches what the generated bridge two files away already resolves, `../hooks//`. Two guards, because neither alone would have caught it. build.unit.test.ts reads the JOURNAL_SCRIPT literal out of the shipped module and resolves it against the path the build contract declares — red on flipping the literal back, red on moving OPENCODE_HOOKS_DIR. opencode-plugin.test.js staged its fixture in the layout the build no longer writes, so it agreed with the bug; it now mirrors the split the loader actually sees, and goes red 2 of 3 on the old path. The plugin README drops to what a reader needs to understand the thing: what it is, why it exists, how it works, how to start, what each tool can answer. Every partly-measurable tool keeps its reason, since a limit a reader has to look up gets read as a zero. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VWNxk63AGKkqE8HRqHLjGp AIDD-Session-Id: 2c21d903-3a7e-47ac-83f8-d8b7ae3aa579 --- .../profiles/opencode/build.unit.test.ts | 22 ++ plugins/aidd-telemetry/README.md | 190 +++++------------- .../aidd-telemetry/hooks/opencode-plugin.js | 8 +- scripts/__tests__/opencode-plugin.test.js | 21 +- 4 files changed, 92 insertions(+), 149 deletions(-) diff --git a/cli/tests/contexts/tools/domain/profiles/opencode/build.unit.test.ts b/cli/tests/contexts/tools/domain/profiles/opencode/build.unit.test.ts index 730ee5250..f51658fb3 100644 --- a/cli/tests/contexts/tools/domain/profiles/opencode/build.unit.test.ts +++ b/cli/tests/contexts/tools/domain/profiles/opencode/build.unit.test.ts @@ -1,3 +1,5 @@ +import { readFileSync } from "node:fs"; +import { join, posix } from "node:path"; import { describe, expect, it } from "vitest"; import type { ArtifactContract, @@ -8,6 +10,12 @@ import { transformMcpToOpencode, } from "../../../../../../src/contexts/tools/domain/profiles/opencode/build.js"; import { InMemoryFileAdapter } from "../../../../../helpers/ports/in-memory-file-adapter.js"; +import { REPOSITORY_ROOT } from "../../../../../helpers/repository-root.js"; + +const OPENCODE_PLUGIN_MODULE = readFileSync( + join(REPOSITORY_ROOT, "plugins/aidd-telemetry/hooks/opencode-plugin.js"), + "utf8" +); // Built, not written literally: biome reads a string holding "${...}" as a lost template. const ROOT = "$" + "{CLAUDE_PLUGIN_ROOT}"; @@ -169,6 +177,20 @@ describe("buildOpencodeFlatContract()", () => { }); }); + it("delivers journal.cjs where the shipped plugin module resolves it, not beside the loader", () => { + const hooks = supported(buildOpencodeFlatContract().artifacts.hooks); + const named = + /JOURNAL_SCRIPT = fileURLToPath\(\s*new URL\("([^"]+)", import\.meta\.url\)/u.exec( + OPENCODE_PLUGIN_MODULE + )?.[1]; + const loaderEntry = hooks.path("aidd-telemetry", "hooks/opencode-plugin.js"); + + expect(named).toBeDefined(); + expect(posix.normalize(posix.join(posix.dirname(loaderEntry), named ?? ""))).toBe( + hooks.path("aidd-telemetry", "hooks/journal.cjs") + ); + }); + it("generates no bridge for a plugin whose hooks name no event OpenCode delivers", () => { const hooks = supported(buildOpencodeFlatContract().artifacts.hooks); diff --git a/plugins/aidd-telemetry/README.md b/plugins/aidd-telemetry/README.md index d6bb6f0c5..6c8fd11d9 100644 --- a/plugins/aidd-telemetry/README.md +++ b/plugins/aidd-telemetry/README.md @@ -5,14 +5,13 @@ Know what a piece of work cost: which skill, which step and which task spent the tokens. > Status: beta. Proven end to end on Claude Code; the other four tools are covered to the -> extent their own files allow, see [Coverage](#coverage). Off the curated install path -> until it has run on other people's machines. +> extent their own files allow. Off the curated install path until it has run on other +> people's machines. -## What it does +## What it is Your provider can tell you a developer burned four million tokens on Tuesday. This plugin -tells you that `aidd-dev:02-implement` spent 78,188 of them, on task `2026_09_01_the-upward-link`, -inside one orchestrated flow. It attributes consumption to the framework's own units of work. +tells you which skill spent them, on which task. ```text period 2026-08-21 to 2026-08-21 @@ -27,6 +26,16 @@ period 2026-08-21 to 2026-08-21 aidd-ui:01-hello 33% 38,490 tokens from a journal interval ``` +Every figure says how it was attributed. `stated by the tool` is exact, `from a journal +interval` is an inference, `unattributed` means neither source could say — never "no step +ran". An unknown is named, never shown as a zero, and no figure is in currency: pricing +tokens is a separate service's job. + +## Why it exists + +A provider meters an account. Only the framework knows its own units of work — the skill, +the step, the task, the flow — so only it can say what one piece of work cost. + Three things it never does: it never sends anything anywhere, it never stores a prompt, a diff or a line of code, and it never records until you turn it on. @@ -51,19 +60,17 @@ flowchart LR CLI -->|joins by session, keeps a record| Store --> Answer ``` -- **The hooks journal.** While measurement is on, every session appends one line per - observation to `aidd_docs/runs/__.jsonl`, git-ignored, never rewritten. - No token, no cost, no model lands there. +- **The hooks journal.** Every session appends one line per observation to + `aidd_docs/runs/__.jsonl`, git-ignored, never rewritten. No token, no + cost, no model lands there. - **Your tool writes its own transcript**, in its own place and format. It holds the tokens and knows nothing about skills. -- **`aidd telemetry report` joins the two.** It reads the transcript, normalises it into one - shape whatever tool produced it, matches each record against the journal and keeps the - result under `~/.config/aidd/telemetry/`. The join cannot happen live: when a hook fires, - the tokens for that turn are not written yet. +- **`aidd telemetry report` joins the two**, by session, and keeps the result under + `~/.config/aidd/telemetry/`. The join cannot happen live: when a hook fires, the tokens + for that turn are not written yet. -Recording is the one act that depends on nothing: the hooks run under plain `node`, so a -session is measured whether or not `aidd` is installed. Allowing and answering go through -the CLI, so the figure is computed once, in one place, whatever asked for it. +Recording depends on nothing but `node`, so a session is measured whether or not `aidd` is +installed. Allowing and answering go through the CLI, so the figure is computed once. ## Getting started @@ -72,8 +79,8 @@ npm install -g @ai-driven-dev/cli aidd plugin install aidd-telemetry ``` -Then talk to your AI tool. Each skill reaches the CLI and stops with the reason if `aidd` -does not answer, rather than reporting an empty figure. +Then ask your AI tool for a skill. Each one stops with the reason if `aidd` does not answer, +rather than reporting an empty figure. | Ask your tool for | It runs | You get | | --- | --- | --- | @@ -81,48 +88,6 @@ does not answer, rather than reporting an empty figure. | `01-cost` | `aidd telemetry report` | what a period or one task consumed, by step, model, task, flow, tool or person | | `02-check` | `aidd telemetry check` | whether the chain is actually recording, and what to fix if not | -Your tool's own plugin mechanism installs the plugin just as well. The CLI stays required -for one reason: nothing recorded can be read without it. - -```mermaid -sequenceDiagram - participant You - participant Tool as AI tool - participant Hook as journal.cjs - participant CLI as aidd telemetry - You->>Tool: ask for 00-init - Tool->>CLI: telemetry on - Note over CLI: .aidd/config.json telemetry.enabled = true
aidd_docs/runs/ git-ignored - Tool->>Hook: SessionStart - Hook->>Hook: session_start line - loop every turn - Tool->>Hook: PostToolUse - Hook->>Hook: step_start, task_declared, file_written - Tool->>Hook: Stop - Hook->>Hook: turn_end line - end - You->>Tool: ask for 01-cost - Tool->>CLI: telemetry report - CLI->>CLI: read transcript + journal, join by session - CLI-->>Tool: figures per step and task -``` - -## What a figure tells you about itself - -Every attributed figure says how it was attributed, because the two ways are not the same -claim. - -| Reads | Means | -| --- | --- | -| stated by the tool | the tool named the running skill itself, on the line with the counters: exact | -| from a journal interval | derived from the interval between two boundaries the journal recorded: an inference | -| unattributed | neither source could say. It never means "no step ran" | - -An absent figure is named, never shown as a zero. A tool that cannot be read, one that -carries no amount, one that measured nothing and one whose reader failed are four -different answers. No tool read locally writes a figure in currency: reports give tokens, -and turning tokens into money is a separate service's job. - ## Coverage | Tool | Tokens | Step | Task | @@ -130,102 +95,47 @@ and turning tokens into money is a separate service's job. | **Claude Code** | ✅ proven on live sessions | ✅ stated by the tool, and by interval | ✅ | | **Codex** | ✅ on captured rollouts | ✅ by interval | ✅ | | **OpenCode** | ✅ | ✅ through its own plugin API | ✅ | -| **Copilot** | ⚠️ session total only, no per-request figure (one cumulative total at shutdown) | ✅ by interval ([#663](https://github.com/ai-driven-dev/framework/issues/663)) | ✅ | +| **Copilot** | ⚠️ session total only, no per-request figure (one cumulative total at shutdown) | ✅ by interval | ✅ | | **Cursor** | ❌ no token count in any file it writes | ✅ | ✅ | -
-Measured limits, per tool +A limit a reader has to look up gets read as a zero, so each one is named here: - **Codex needs one interactive approval.** Its hook trust is per entry and a headless run - never sees the prompt, so a Codex session journals nothing until someone approves once, - in an interactive session. -- **OpenCode never announces a session, so the plugin opens it.** Measured on one live - `opencode 1.14.20` run (2026-08-31, `--print-logs`): `session.created` is published on - the bus but never reaches the hook, while `session.idle` reaches every session. The first - call a session produces therefore opens it, under the directory that call was going to - use. What is lost: on a server serving several directories, a session it never announced - is journalled under the plugin's own init-time directory. Details in - `scripts/__tests__/fixtures/README.md`, "OpenCode's plugin events". -- **A task is declared from a tool call's own arguments, on every host.** A `Read`, a `Bash` - command line or an object keyed `path` naming a file under a task folder is enough; the - journal reads that text rather than asking the host to cooperate. OpenCode joined on - 2026-08-31 once a completed tool part's arguments were measured to reach its `event` - hook. One real capture per host is kept in `scripts/__tests__/fixtures/README.md`. -- **These are raw counters, not your tool's usage screen.** A vendor's page weights a - cached token by what it charges for it; these figures are the counts the tool wrote down. - The two disagree on cache lines by construction, and neither is wrong. -- **A period means when the work ran**, not when it was billed. -- **A sweep reaches a session only where the journal was installed.** Work done before you - turned measurement on is not there, and nothing reconstructs it. - -
+ never sees the prompt, so a Codex session journals nothing until someone approves once. +- **OpenCode never announces a session, so the plugin opens it.** `session.created` is + published on its bus but never reaches the hook, so the first call a session produces + opens it, under the directory that call was going to use. What is lost: on a server + serving several directories, a session it never announced is + journalled under the plugin's own init-time directory. +- **These are raw counters, not your tool's usage screen.** A vendor's page weights a cached + token by what it charges for it; these are the counts the tool wrote down. The two + disagree on cache lines by construction, and neither is wrong. +- **A period means when the work ran**, not when it was billed, and nothing reconstructs + work done before you turned measurement on. ## Privacy -- **Nothing leaves the machine.** Every code path that once could, the export writer, its - endpoint, the server it talked to, is deleted. Everything measured is read back from - where it was written. On a machine where an older version once configured an export - endpoint, `aidd telemetry check` and `aidd telemetry off` both detect the settings file - it left and name what to remove by hand. +- **Nothing leaves the machine.** Every code path that once could is deleted. On a machine + where an older version configured an export endpoint, `aidd telemetry check` and + `aidd telemetry off` both detect it and name what to remove by hand. - **No prompt, no code, no diff.** The stored shape is an allowlist, field by field, in [the record contract](../../aidd_docs/product/metrics-contract.md). - **The switch is a file you commit or do not**, per project (`.aidd/config.json`). Once committed on, it applies to everyone who clones. Refuse it for yourself alone with `AIDD_TELEMETRY=0`, which overrides the file unconditionally. -- **`off` keeps what you measured.** It stops the recording, not the record. - **`aidd telemetry forget` removes it**: this project's journal, this machine's stored - records (every project ever measured on this machine) and this machine's identity file. - It shows what would go before anything happens and removes nothing without `--yes`. -- **Your identity is yours to attach.** `aidd telemetry identity` opts a person in or out - of naming themselves on their own records; nothing about a colleague's identity arrives on - its own. - -## Where things live - -| What | Where | Why there | -| --- | --- | --- | -| The switch | `.aidd/config.json` in the project | a property of the repository, shared by committing it | -| The journal | `aidd_docs/runs/` in the project, git-ignored | every line names a repository-relative path or a task folder | -| The figures | `~/.config/aidd/telemetry/` (`%APPDATA%\aidd\telemetry\` on Windows), or `AIDD_TELEMETRY_DIR` | a session's consumption belongs to the person who ran it | -| The identity | the OS profile, on every platform | it never follows a shared directory | - -**Share `AIDD_TELEMETRY_DIR`, never `AIDD_USER_CONFIG_DIR`.** Point the first at a directory -a team shares and every figure follows it. The second relocates a machine's whole AIDD -configuration, `auth.json` and its GitHub token included, and two people pointing at one -directory would overwrite each other's token file. A setup made before this split still -works through `AIDD_USER_CONFIG_DIR`, and should be changed. - -On a shared sink, a colleague's records stay `unresolved` in your report until you run -`aidd telemetry identity link` on their identifier yourself. Linking declares "this -identifier is me", and the CLI cannot check that claim against anything the colleague wrote. - -## The backlog link - -A task folder can say which backlog item it delivers. `aidd-pm:04-spec` and -`aidd-dev:01-plan` write `backlog-link.json` beside `spec.md` and `plan.md` when the request -they build from names one. No file is the normal state, never an error. - -```json -{ - "backlog": "owner/repo#123", - "written_at": "2026-08-21T09:00:00Z", - "written_by": "aidd-pm:04-spec" -} -``` - -`backlog` names the item wherever it lives, a forge reference or a project-relative -Markdown path, one field for both per -[`persistence.md`](../aidd-pm/skills/10-task/references/persistence.md). `written_at` and -`written_by` are provenance, not status. It is a plain file: edit `backlog` and the next -report reads it as it stands. Nothing here re-derives or overwrites a declaration that -exists. It carries no steps, no file list, no branch and no status: the journal, git and -the artefacts' own frontmatter already own those. +- **`off` keeps what you measured**; `aidd telemetry forget` removes it — this project's + journal, this machine's records and its identity file — and removes nothing without + `--yes`. +- **Your identity is yours to attach**, through `aidd telemetry identity`. To share figures + across a team, point `AIDD_TELEMETRY_DIR` at a shared directory — never + `AIDD_USER_CONFIG_DIR`, which also relocates `auth.json` and its GitHub token. ## Where things are written down - [`aidd_docs/runs/README.md`](../../aidd_docs/runs/README.md): what the journal records, and what it deliberately does not. - [`cost-report-contract.md`](../../aidd_docs/product/cost-report-contract.md): the object - `report --json` prints, for a skill or a program to consume. -- [`metrics-contract.md`](../../aidd_docs/product/metrics-contract.md): one stored line, - for a service that prices them. + `report --json` prints, and the `backlog-link.json` a task folder may carry to say which + backlog item it delivers. +- [`metrics-contract.md`](../../aidd_docs/product/metrics-contract.md): one stored line, for + a service that prices them. diff --git a/plugins/aidd-telemetry/hooks/opencode-plugin.js b/plugins/aidd-telemetry/hooks/opencode-plugin.js index 2b9c51679..26d30144b 100644 --- a/plugins/aidd-telemetry/hooks/opencode-plugin.js +++ b/plugins/aidd-telemetry/hooks/opencode-plugin.js @@ -16,7 +16,13 @@ import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; -const JOURNAL_SCRIPT = fileURLToPath(new URL("./journal.cjs", import.meta.url)); +// Not a sibling: OpenCode's loader scans `plugin/` one level deep, so the build delivers this +// module there alone and every other hook script under `hooks//`. That is the same +// `../hooks//` the generated bridge resolves (opencode-hooks-bridge.ts's HOOKS_DIR), +// and build.unit.test.ts checks this literal against where the build actually writes it. +const JOURNAL_SCRIPT = fileURLToPath( + new URL("../hooks/aidd-telemetry/journal.cjs", import.meta.url) +); // Never `process.execPath`: OpenCode ships as its own standalone binary, so that path names // `opencode` itself, not a Node runtime that can run journal.cjs. diff --git a/scripts/__tests__/opencode-plugin.test.js b/scripts/__tests__/opencode-plugin.test.js index 321d3a18b..384f07c23 100644 --- a/scripts/__tests__/opencode-plugin.test.js +++ b/scripts/__tests__/opencode-plugin.test.js @@ -28,9 +28,10 @@ function makeTempDir(prefix) { return dir; } -// Mirrors what a real install delivers: opencode-plugin.js copied verbatim beside -// journal.cjs and lib/ (see plugin-content-translator.ts, flatHooksFiles) - not the -// source tree, so this exercises the exact sibling-file layout OpenCode's loader sees. +// Mirrors what a real install delivers: OpenCode's loader scans `plugin/` one level deep, so +// the build puts this plugin's own module there alone, renamed after the plugin, and every +// other hook script under `hooks//` (opencode-paths.ts) - not the source tree, where +// they are siblings, so this exercises the split layout the loader actually sees. function makeInstalledRepo() { const repo = makeTempDir("aidd-opencode-plugin-repo-"); execFileSync("git", ["init", "-q"], { cwd: repo, env: CLEAN_ENV }); @@ -43,13 +44,17 @@ function makeInstalledRepo() { JSON.stringify({ telemetry: { enabled: true, endpoint: "http://127.0.0.1:4318" } }) ); const pluginDir = path.join(repo, ".opencode", "plugin"); + const scriptsDir = path.join(repo, ".opencode", "hooks", "aidd-telemetry"); fs.mkdirSync(pluginDir, { recursive: true }); + fs.mkdirSync(scriptsDir, { recursive: true }); const hooksSrc = path.dirname(PLUGIN_SOURCE); for (const entry of fs.readdirSync(hooksSrc, { withFileTypes: true })) { if (entry.name === "hooks.json") continue; - fs.cpSync(path.join(hooksSrc, entry.name), path.join(pluginDir, entry.name), { - recursive: true, - }); + const loaderEntry = entry.name === path.basename(PLUGIN_SOURCE); + const target = loaderEntry + ? path.join(pluginDir, "aidd-telemetry.js") + : path.join(scriptsDir, entry.name); + fs.cpSync(path.join(hooksSrc, entry.name), target, { recursive: true }); } // A byte-identical `.mjs` twin, for these tests alone. // @@ -60,8 +65,8 @@ function makeInstalledRepo() { // Plain Node does consult it, and there is none to consult: nothing up this tree declares // one, so Node reaches the file as typeless, finds ESM syntax, and reparses. Naming the // extension explicitly is what these tests do instead, and it is the only difference. - const esmTwin = path.join(pluginDir, "opencode-plugin.mjs"); - fs.copyFileSync(path.join(pluginDir, "opencode-plugin.js"), esmTwin); + const esmTwin = path.join(pluginDir, "aidd-telemetry.mjs"); + fs.copyFileSync(path.join(pluginDir, "aidd-telemetry.js"), esmTwin); return { repo, pluginDir, esmTwin }; }