From 931b32ffc3fa34cffd4fcef915bfeb787a0561eb Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Sat, 13 Jun 2026 08:58:00 -0700 Subject: [PATCH 1/6] feat(cli): Collapse help_* into cli_help { topic }; drop cli_info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 4 of the telemetry rework. Replace help_index / help_ / help_unknown with a single cli_help carrying a topic property (the served topic, "(index)" for a no-arg invocation, or the attempted topic when unknown) — help intent is now one event filtered by topic. Remove info's bespoke cli_info / cli_info_completed events (covered by cli_run) and its now-unused getTelemetry import. detect.ts (cli_detect) is not on this branch's lineage — it lives in the unmerged local-rule-routing stack and is reconciled when both land. Adds test/help-telemetry.test.ts asserting cli_help { topic } across the served / index / unknown cases and that no legacy help_* event fires. Full suite green (259). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../restructure-cli-telemetry/tasks.md | 8 +- packages/cli/src/commands/help.ts | 17 +-- packages/cli/src/commands/info.ts | 144 ++++++++---------- packages/cli/test/help-telemetry.test.ts | 64 ++++++++ 4 files changed, 140 insertions(+), 93 deletions(-) create mode 100644 packages/cli/test/help-telemetry.test.ts diff --git a/openspec/changes/restructure-cli-telemetry/tasks.md b/openspec/changes/restructure-cli-telemetry/tasks.md index 48cd2edb..e53116de 100644 --- a/openspec/changes/restructure-cli-telemetry/tasks.md +++ b/openspec/changes/restructure-cli-telemetry/tasks.md @@ -49,10 +49,10 @@ keeps the suite green on its own. ## 4. Phase 4 — cli_help { topic } + drop bespoke info/detect events (PR 4, on PR 3) -- [ ] 4.1 `commands/help.ts`: replace `help_index`, `help_`, `help_unknown` with one `cli_help { topic }` (served topic, an index marker for no-arg, the attempted topic for unknown) -- [ ] 4.2 `commands/info.ts`, `commands/detect.ts`: remove their bespoke `cli_info(_completed)` / `cli_detect` events — covered by `cli_run` -- [ ] 4.3 Update `test/help-extensions.test.ts` / `test/help-routing-telemetry.test.ts` and info/detect tests; assert `cli_help` carries `topic` and no `help_*` event is emitted -- [ ] 4.4 typecheck + lint + suite green; commit; open PR 4 +- [x] 4.1 `commands/help.ts`: replace `help_index`, `help_`, `help_unknown` with one `cli_help { topic }` (served topic, `"(index)"` marker for no-arg, the attempted topic for unknown) +- [x] 4.2 `commands/info.ts`: remove bespoke `cli_info(_completed)` (covered by `cli_run`); also drop its now-unused `getTelemetry` import. NOTE: `detect.ts`/`cli_detect` is NOT on this branch's lineage (it lives in the unmerged local-rule-routing stack) — no change needed here; it will be reconciled when that stack and this one both land +- [x] 4.3 Assert `cli_help` carries `topic` and no `help_*` event — added `test/help-telemetry.test.ts` (served topic, index marker, unknown topic, and no legacy `help_*`) +- [x] 4.4 typecheck + lint + suite green; commit; open PR 4 ## 5. Phase 5 — finalize (PR 5, tip) diff --git a/packages/cli/src/commands/help.ts b/packages/cli/src/commands/help.ts index 71793854..f6e12938 100644 --- a/packages/cli/src/commands/help.ts +++ b/packages/cli/src/commands/help.ts @@ -153,8 +153,8 @@ export function createHelpCommand(subCommands: SubCommandsDef) { const telemetry = await getTelemetry(cwd); if (positionals.length === 0) { - // help_index: agent fetched the topic list - telemetry.capture("help_index"); + // cli_help with the index marker: agent fetched the topic list + telemetry.capture("cli_help", { topic: "(index)" }); console.log("Taskless CLI\n"); console.log( @@ -198,16 +198,13 @@ export function createHelpCommand(subCommands: SubCommandsDef) { : helpMap.get(key); if (content) { - // help_: agent fetched a specific recipe (intent signal) - const topicEvent = `help_${key.replaceAll("-", "_")}`; - telemetry.capture(topicEvent, { - topic: positionals.join(" "), - anonymous: args.anonymous, - }); + // cli_help: agent fetched a specific recipe (intent signal). The topic + // is the served topic; filtering on it replaces the old per-topic events. + telemetry.capture("cli_help", { topic: positionals.join(" ") }); console.log(renderRecipe(content, key).trimEnd()); } else { - // help_unknown: agent asked for a topic that does not exist - telemetry.capture("help_unknown", { topic: positionals.join(" ") }); + // cli_help for an unknown topic — still the attempted topic string. + telemetry.capture("cli_help", { topic: positionals.join(" ") }); console.error(`Unknown command: ${positionals.join(" ")}`); console.error("Run `taskless help` for available commands."); process.exitCode = 1; diff --git a/packages/cli/src/commands/info.ts b/packages/cli/src/commands/info.ts index 5ebfafb7..1da8a835 100644 --- a/packages/cli/src/commands/info.ts +++ b/packages/cli/src/commands/info.ts @@ -5,7 +5,6 @@ import { checkStaleness } from "../install/install"; import { getToken } from "../auth/token"; import { fetchWhoami } from "../auth/whoami"; import { outputSchema as infoOutputSchema } from "../schemas/info"; -import { getTelemetry } from "../telemetry"; import { makeErrorEnvelope } from "../types/errors"; export const infoCommand = defineCommand({ @@ -32,100 +31,87 @@ export const infoCommand = defineCommand({ }, async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); - const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_info"); - let success = false; - try { - const [tools, token] = await Promise.all([ - checkStaleness(cwd), - args.anonymous ? Promise.resolve() : getToken(cwd), - ]); + const [tools, token] = await Promise.all([ + checkStaleness(cwd), + args.anonymous ? Promise.resolve() : getToken(cwd), + ]); - let auth: { user: string; email: string; orgs: string[] } | undefined; - if (!args.anonymous && token) { - const whoami = await fetchWhoami(token); - if (whoami) { - auth = { - user: whoami.user, - email: whoami.email, - orgs: whoami.orgs.map((o) => o.name), - }; - } + let auth: { user: string; email: string; orgs: string[] } | undefined; + if (!args.anonymous && token) { + const whoami = await fetchWhoami(token); + if (whoami) { + auth = { + user: whoami.user, + email: whoami.email, + orgs: whoami.orgs.map((o) => o.name), + }; } + } - const result = { - success: true as const, - version: __VERSION__, - tools, - loggedIn: token !== undefined, - auth, - }; + const result = { + success: true as const, + version: __VERSION__, + tools, + loggedIn: token !== undefined, + auth, + }; - if (args.json) { - const parsed = infoOutputSchema.safeParse(result); - if (!parsed.success) { - console.log( - JSON.stringify( - makeErrorEnvelope( - "INTERNAL_ERROR", - "Internal schema validation failed" - ) + if (args.json) { + const parsed = infoOutputSchema.safeParse(result); + if (!parsed.success) { + console.log( + JSON.stringify( + makeErrorEnvelope( + "INTERNAL_ERROR", + "Internal schema validation failed" ) - ); - process.exitCode = 1; - return; - } - console.log(JSON.stringify(parsed.data)); - success = true; + ) + ); + process.exitCode = 1; return; } + console.log(JSON.stringify(parsed.data)); + return; + } - // Human-readable output - console.log(`Taskless CLI v${__VERSION__}\n`); + // Human-readable output + console.log(`Taskless CLI v${__VERSION__}\n`); - if (tools.length === 0) { - console.log("Tools: none detected"); - } else { - console.log("Tools:"); - for (const tool of tools) { - const total = tool.skills.length; - const upToDate = tool.skills.filter((s) => s.current).length; - const stale = total - upToDate; + if (tools.length === 0) { + console.log("Tools: none detected"); + } else { + console.log("Tools:"); + for (const tool of tools) { + const total = tool.skills.length; + const upToDate = tool.skills.filter((s) => s.current).length; + const stale = total - upToDate; - if (stale === 0) { - console.log( - ` ${tool.name}: ${String(total)} skills (all up to date)` - ); - } else { - console.log( - ` ${tool.name}: ${String(total)} skills (${String(stale)} outdated)` - ); - for (const skill of tool.skills) { - if (!skill.current) { - console.log( - ` - ${skill.name}: ${skill.installedVersion ?? "missing"} → ${skill.currentVersion}` - ); - } + if (stale === 0) { + console.log( + ` ${tool.name}: ${String(total)} skills (all up to date)` + ); + } else { + console.log( + ` ${tool.name}: ${String(total)} skills (${String(stale)} outdated)` + ); + for (const skill of tool.skills) { + if (!skill.current) { + console.log( + ` - ${skill.name}: ${skill.installedVersion ?? "missing"} → ${skill.currentVersion}` + ); } } } } + } - console.log(""); - if (auth) { - const orgs = auth.orgs.length > 0 ? ` (${auth.orgs.join(", ")})` : ""; - console.log(`Auth: logged in as ${auth.user}${orgs}`); - } else { - console.log("Auth: not logged in"); - } - success = true; - } finally { - telemetry.capture("cli_info_completed", { - success, - durationMs: Date.now() - startedAt, - }); + console.log(""); + if (auth) { + const orgs = auth.orgs.length > 0 ? ` (${auth.orgs.join(", ")})` : ""; + console.log(`Auth: logged in as ${auth.user}${orgs}`); + } else { + console.log("Auth: not logged in"); } }, }); diff --git a/packages/cli/test/help-telemetry.test.ts b/packages/cli/test/help-telemetry.test.ts new file mode 100644 index 00000000..1bc7f14a --- /dev/null +++ b/packages/cli/test/help-telemetry.test.ts @@ -0,0 +1,64 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +// Spy on telemetry by mocking the module the help command imports. The factory +// is invoked lazily at import time (same pattern as telemetry.test.ts). +const capture = vi.fn(); +vi.mock("../src/telemetry", () => ({ + getTelemetry: vi.fn(() => + Promise.resolve({ capture, shutdown: () => Promise.resolve() }) + ), + shutdownTelemetry: () => Promise.resolve(), +})); + +const { createHelpCommand } = await import("../src/commands/help"); + +interface RunnableCommand { + run: (context: { + args: { dir: string; anonymous: boolean }; + rawArgs: string[]; + }) => Promise; +} + +async function runHelp(rawArguments: string[]): Promise { + const command = createHelpCommand({}) as unknown as RunnableCommand; + await command.run({ + args: { dir: process.cwd(), anonymous: false }, + rawArgs: rawArguments, + }); +} + +describe("help emits cli_help { topic }", () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + capture.mockClear(); + logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it("captures the served topic", async () => { + await runHelp(["help", "rule", "create"]); + expect(capture).toHaveBeenCalledWith("cli_help", { topic: "rule create" }); + }); + + it("captures an index marker when invoked with no topic", async () => { + await runHelp(["help"]); + expect(capture).toHaveBeenCalledWith("cli_help", { topic: "(index)" }); + }); + + it("captures the attempted topic for an unknown topic, and no legacy help_* event", async () => { + await runHelp(["help", "nope"]); + expect(capture).toHaveBeenCalledWith("cli_help", { topic: "nope" }); + + const events = capture.mock.calls.map((call) => call[0] as string); + expect(events).not.toContain("help_index"); + expect(events).not.toContain("help_unknown"); + expect(events.every((event) => !event.startsWith("help_"))).toBe(true); + }); +}); From 7b8603a9da32d0b1d0e2a098dbc7251d801ced85 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Sat, 13 Jun 2026 09:08:19 -0700 Subject: [PATCH 2/6] chore(openspec): Archive restructure-cli-telemetry and sync specs All five phases of the telemetry rework are complete, so finalize on the tip of the stack: apply the analytics delta into the main spec (add the cli_run denominator requirement; rewrite the cli_ taxonomy, the wrong-topic funnel, and the standard-properties scenarios) and move the change to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/. Legacy event sweep is clean (the only _completed is the intentional cli_check_completed concrete event); validate/typecheck/lint/suite green; commands smoke-tested end-to-end. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/analytics/spec.md | 0 .../tasks.md | 8 +- openspec/specs/analytics/spec.md | 121 +++++++++++++----- 6 files changed, 92 insertions(+), 37 deletions(-) rename openspec/changes/{restructure-cli-telemetry => archive/2026-06-13-restructure-cli-telemetry}/.openspec.yaml (100%) rename openspec/changes/{restructure-cli-telemetry => archive/2026-06-13-restructure-cli-telemetry}/design.md (100%) rename openspec/changes/{restructure-cli-telemetry => archive/2026-06-13-restructure-cli-telemetry}/proposal.md (100%) rename openspec/changes/{restructure-cli-telemetry => archive/2026-06-13-restructure-cli-telemetry}/specs/analytics/spec.md (100%) rename openspec/changes/{restructure-cli-telemetry => archive/2026-06-13-restructure-cli-telemetry}/tasks.md (90%) diff --git a/openspec/changes/restructure-cli-telemetry/.openspec.yaml b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/.openspec.yaml similarity index 100% rename from openspec/changes/restructure-cli-telemetry/.openspec.yaml rename to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/.openspec.yaml diff --git a/openspec/changes/restructure-cli-telemetry/design.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md similarity index 100% rename from openspec/changes/restructure-cli-telemetry/design.md rename to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md diff --git a/openspec/changes/restructure-cli-telemetry/proposal.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/proposal.md similarity index 100% rename from openspec/changes/restructure-cli-telemetry/proposal.md rename to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/proposal.md diff --git a/openspec/changes/restructure-cli-telemetry/specs/analytics/spec.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/specs/analytics/spec.md similarity index 100% rename from openspec/changes/restructure-cli-telemetry/specs/analytics/spec.md rename to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/specs/analytics/spec.md diff --git a/openspec/changes/restructure-cli-telemetry/tasks.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/tasks.md similarity index 90% rename from openspec/changes/restructure-cli-telemetry/tasks.md rename to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/tasks.md index e53116de..a6e0541a 100644 --- a/openspec/changes/restructure-cli-telemetry/tasks.md +++ b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/tasks.md @@ -56,7 +56,7 @@ keeps the suite green on its own. ## 5. Phase 5 — finalize (PR 5, tip) -- [ ] 5.1 Grep the CLI for any remaining old event names (`_completed`, `help_index`, `help_`, `help_unknown`, legacy `cli_` starts); remove any stragglers -- [ ] 5.2 Run `pnpm openspec validate restructure-cli-telemetry`; `pnpm typecheck`; `pnpm lint`; full suite green -- [ ] 5.3 Manual smoke: run a couple of commands with telemetry mocked/inspected — confirm one `cli_run` per invocation plus the expected concrete event, and no legacy names -- [ ] 5.4 Archive the change (`openspec archive restructure-cli-telemetry`) so the tip carries the spec sync + dated archive; commit; open PR 5 +- [x] 5.1 Grep the CLI for any remaining old event names (`_completed`, `help_index`, `help_`, `help_unknown`, legacy `cli_` starts); remove any stragglers — clean; the only `_completed` is the intentional concrete event `cli_check_completed` +- [x] 5.2 Run `pnpm openspec validate restructure-cli-telemetry`; `pnpm typecheck`; `pnpm lint`; full suite green (259) +- [x] 5.3 Manual smoke: `info`, `help check`, `help` (index) run end-to-end after the refactor; concrete events + cli_run/cli_help/cli_error verified by the in-process tests +- [x] 5.4 Archive the change (`openspec archive restructure-cli-telemetry`) so the tip carries the spec sync + dated archive; commit; open PR 5 diff --git a/openspec/specs/analytics/spec.md b/openspec/specs/analytics/spec.md index 6f1173c1..dab92816 100644 --- a/openspec/specs/analytics/spec.md +++ b/openspec/specs/analytics/spec.md @@ -111,13 +111,13 @@ Every `capture()` call SHALL include the `cli` property (anonymous UUID), the `c #### Scenario: Anonymous capture includes standard properties -- **WHEN** `capture("cli_check")` is called without authentication +- **WHEN** `capture("cli_run")` is called without authentication - **THEN** the event SHALL include `{ cli: anonymousUuid, cliVersion: , scaffoldVersion: }` - **AND** the event SHALL NOT include a `groups` parameter #### Scenario: Authenticated capture includes standard properties and group -- **WHEN** `capture("cli_rule_create")` is called with authentication +- **WHEN** `capture("cli_rule_created")` is called with authentication - **THEN** the event SHALL include `{ cli: anonymousUuid, cliVersion: , scaffoldVersion: }` - **AND** the `groups` parameter SHALL include `{ organization: String(orgId) }` @@ -134,56 +134,88 @@ Every `capture()` call SHALL include the `cli` property (anonymous UUID), the `c ### Requirement: CLI events use cli\_ prefix -CLI action events SHALL continue to use the `cli_` prefix, but the event taxonomy SHALL be reorganized as follows: - -- `cli_` — fired when an action command begins execution (e.g. `cli_rule_create`, `cli_rule_improve`, `cli_rule_delete`, `cli_check`, `cli_info`, `cli_init`, `cli_auth_login`, `cli_auth_logout`) -- `cli__completed` — fired when an action command finishes execution; event properties SHALL include `success: boolean`, `durationMs: number`, and `errorCode?: string` (when failure) -- `help_` — fired when the help command serves a specific topic (e.g. `help_rule_create`, `help_check`, `help_auth`); replaces previous `cli_help_` events -- `help_index` — fired when the help command is invoked with no arguments (probable agent confusion / routing failure) -- `help_unknown` — fired when the help command receives an unknown topic; event properties SHALL include `topic: string` (the attempted topic) - -The previous event names `cli_help`, `cli_help_auth`, `cli_help_check`, `cli_help_info`, `cli_help_init`, `cli_help_rule` SHALL be removed in this release. There is no dual-emit window — the rename is a hard cut. - -#### Scenario: Action command emits start and completion events - -- **WHEN** a user runs `taskless rule create --from req.json` -- **THEN** PostHog SHALL receive a `cli_rule_create` event when execution begins -- **AND** SHALL receive a `cli_rule_create_completed` event when execution finishes, with properties including `success`, `durationMs`, and (on failure) `errorCode` - -#### Scenario: Help fetch emits topic intent +CLI events SHALL use the `cli_` prefix, with the taxonomy organized as a +`cli_run` denominator plus concrete state-transition events: + +- `cli_run` — exactly one per invocation (see the dedicated requirement). This + replaces every previous `cli_` start event and `cli__completed` + event; the `success`/`durationMs`/`command` signal lives here. +- Concrete state-transition events, each fired at the point the state actually + changes, carrying counts/ids/booleans only (never rule content, prompts, or + matched source): + - `cli_rule_created`, `cli_rule_improved`, `cli_rule_deleted` + - `cli_authenticated`, `cli_logged_out` + - `cli_installed`, `cli_onboarded` + - `cli_check_completed` — error/warning counts only (e.g. `errorCount`, + `warningCount`, `filesScanned`) + - `cli_error` — a single failure event with `command` and `code` (a stable + `CliErrorCode`) +- `cli_help` — fired when the help command serves a request, with a `topic` + property (the served topic, or an index marker when invoked with no topic). + This replaces the previous `help_index`, `help_`, and `help_unknown` + events. + +Commands that carry no concrete state beyond the invocation (e.g. `info`, +`detect`, `update`, `auth status`, `rule verify`, `rule meta`) SHALL rely on +`cli_run` alone and SHALL NOT emit a bespoke event. The previous taxonomy +(`cli_`, `cli__completed`, `help_index`, `help_`, +`help_unknown`) SHALL be removed in this release; there is no dual-emit window. + +#### Scenario: Rule creation emits a concrete state event plus cli_run + +- **WHEN** a user runs `taskless rule create --from req.json` and a rule is written +- **THEN** PostHog SHALL receive one `cli_run` event with `command: "rule create"` +- **AND** SHALL receive a `cli_rule_created` event +- **AND** SHALL NOT receive `cli_rule_create` or `cli_rule_create_completed` + +#### Scenario: Help fetch emits cli_help with a topic - **WHEN** an agent runs `taskless help rule create` -- **THEN** PostHog SHALL receive a `help_rule_create` event +- **THEN** PostHog SHALL receive a `cli_help` event with `topic: "rule create"` +- **AND** SHALL NOT receive a `help_rule_create` event -#### Scenario: Help no-args emits index event +#### Scenario: Help with no topic emits cli_help with an index marker - **WHEN** an agent runs `taskless help` -- **THEN** PostHog SHALL receive a `help_index` event +- **THEN** PostHog SHALL receive a `cli_help` event whose `topic` marks the index + (no-argument) invocation +- **AND** SHALL NOT receive a `help_index` event -#### Scenario: Help unknown topic emits help_unknown +#### Scenario: A command failure emits cli_error -- **WHEN** an agent runs `taskless help nonexistent` -- **THEN** PostHog SHALL receive a `help_unknown` event with property `topic: "nonexistent"` +- **WHEN** a command fails with a known `CliErrorCode` +- **THEN** PostHog SHALL receive a `cli_error` event with `command` and `code` #### Scenario: Old event names are not emitted -- **WHEN** any CLI command runs in v0.7.0 -- **THEN** PostHog SHALL NOT receive any event named `cli_help`, `cli_help_`, or any other event under the previous taxonomy +- **WHEN** any CLI command runs in this release +- **THEN** PostHog SHALL NOT receive any event named `cli__completed`, + `help_index`, `help_`, or `help_unknown` ### Requirement: Wrong-topic re-routing is observable as a derivable funnel -The new event taxonomy is structured so that wrong-topic re-routing is a derivable funnel signal: +The taxonomy SHALL keep wrong-topic re-routing derivable as a funnel signal from +the new events: -- A `help_` event followed by no `cli_` event AND a subsequent `help_` event indicates the agent fetched the recipe for topic A, did not act on it, and re-routed to topic B -- A `help_index` event followed by a `help_` event indicates the agent consulted the index before picking a topic (expected behavior; baseline) -- A `help_` event with no subsequent `cli_` event AND no further `help_*` event indicates the agent abandoned the action +- A `cli_help { topic: A }` event not followed by the concrete event for topic A + (or by `cli_run` with the corresponding `command`), and then a subsequent + `cli_help { topic: B }`, indicates the agent fetched recipe A, did not act on + it, and re-routed to topic B. +- A `cli_help` index-marker event followed by a `cli_help { topic }` event + indicates the agent consulted the index before picking a topic (baseline). +- A `cli_help { topic }` event with no subsequent acting `cli_run` and no further + `cli_help` event indicates the agent abandoned the action. -No additional events SHALL be added to capture this signal directly — the funnel is derivable from the event sequence in PostHog. Dashboards SHOULD be created to surface re-routing rates per topic so wrong-topic confusion can be measured. +No additional events SHALL be added to capture this signal directly — it is +derivable from the `cli_help` / `cli_run` sequence. Dashboards SHOULD surface +re-routing rates per topic. #### Scenario: Funnel data supports wrong-topic detection - **WHEN** dashboards are constructed in PostHog -- **THEN** the events SHALL be sufficient to compute "rate of `help_` events not followed by a corresponding `cli_` event within N minutes" +- **THEN** the `cli_help` (with `topic`) and `cli_run` (with `command`) events + SHALL be sufficient to compute "rate of `cli_help { topic }` not followed by a + corresponding acting `cli_run` within N minutes" ### Requirement: Telemetry failures are silent @@ -218,3 +250,26 @@ Each command handler SHALL call `getTelemetry(cwd)` to lazily initialize the sin - **WHEN** the CLI exits without running a command (e.g. showing top-level help) - **THEN** `shutdownTelemetry()` SHALL be a no-op and no PostHog client SHALL be created + +### Requirement: Every invocation emits exactly one cli_run event + +The CLI SHALL emit exactly one `cli_run` event per invocation, from the top-level +runner rather than from individual commands. The event SHALL carry the properties +`command` (the resolved subcommand name, e.g. `"rule create"` or `"help"`), +`cli_version`, `success` (boolean), `durationMs` (number), `anonymous` (boolean), +and `loggedIn` (boolean). The event SHALL be emitted on both success and failure +(from a `finally`-equivalent path), and no command SHALL emit its own +"started" or "ran" event. + +#### Scenario: A successful command emits one cli_run + +- **WHEN** a user runs `taskless info` +- **THEN** PostHog SHALL receive exactly one `cli_run` event with + `command: "info"`, `success: true`, a numeric `durationMs`, and the + `cli_version`, `anonymous`, and `loggedIn` properties +- **AND** SHALL NOT receive a separate `cli_info` or `cli_info_completed` event + +#### Scenario: A failing command still emits cli_run + +- **WHEN** a command exits with an error +- **THEN** PostHog SHALL receive one `cli_run` event with `success: false` From ec1a1ac5a540ed2d00500e0470384287e6618d1a Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Sat, 13 Jun 2026 13:50:08 -0700 Subject: [PATCH 3/6] test(cli): Assert no legacy help_* event on served-topic and index paths PR #36 review: the served-topic and no-arg help-telemetry tests asserted cli_help was emitted but not that the implementation avoids dual-emitting a legacy help_* event. Both now map captured calls to their event names and assert none start with help_ (and specifically not help_index). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/test/help-telemetry.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/cli/test/help-telemetry.test.ts b/packages/cli/test/help-telemetry.test.ts index 1bc7f14a..c6ebc8ba 100644 --- a/packages/cli/test/help-telemetry.test.ts +++ b/packages/cli/test/help-telemetry.test.ts @@ -42,14 +42,21 @@ describe("help emits cli_help { topic }", () => { errorSpy.mockRestore(); }); - it("captures the served topic", async () => { + it("captures the served topic and no legacy help_* event", async () => { await runHelp(["help", "rule", "create"]); expect(capture).toHaveBeenCalledWith("cli_help", { topic: "rule create" }); + + const events = capture.mock.calls.map((call) => call[0] as string); + expect(events.every((event) => !event.startsWith("help_"))).toBe(true); }); - it("captures an index marker when invoked with no topic", async () => { + it("captures the index marker for no topic and no legacy help_index event", async () => { await runHelp(["help"]); expect(capture).toHaveBeenCalledWith("cli_help", { topic: "(index)" }); + + const events = capture.mock.calls.map((call) => call[0] as string); + expect(events).not.toContain("help_index"); + expect(events.every((event) => !event.startsWith("help_"))).toBe(true); }); it("captures the attempted topic for an unknown topic, and no legacy help_* event", async () => { From 102626136a33c7ea47fa61310a084943e723f362 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Sat, 13 Jun 2026 13:55:20 -0700 Subject: [PATCH 4/6] test(cli): Prove help_* removal once via a source scan, not per-test Per review feedback: instead of repeating a "no help_* event" assertion inside every behavioral help-telemetry test (over-testing), keep those tests purely behavioral (cli_help { topic }) and add a single source-scan test that asserts no help_* event-name literal remains anywhere under src/. That states the contract once, confidently. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/test/help-telemetry.test.ts | 45 ++++++++++++++++-------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/packages/cli/test/help-telemetry.test.ts b/packages/cli/test/help-telemetry.test.ts index c6ebc8ba..5e11937e 100644 --- a/packages/cli/test/help-telemetry.test.ts +++ b/packages/cli/test/help-telemetry.test.ts @@ -1,3 +1,6 @@ +import { readdirSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; // Spy on telemetry by mocking the module the help command imports. The factory @@ -42,30 +45,44 @@ describe("help emits cli_help { topic }", () => { errorSpy.mockRestore(); }); - it("captures the served topic and no legacy help_* event", async () => { + it("captures the served topic", async () => { await runHelp(["help", "rule", "create"]); expect(capture).toHaveBeenCalledWith("cli_help", { topic: "rule create" }); - - const events = capture.mock.calls.map((call) => call[0] as string); - expect(events.every((event) => !event.startsWith("help_"))).toBe(true); }); - it("captures the index marker for no topic and no legacy help_index event", async () => { + it("captures the index marker for no topic", async () => { await runHelp(["help"]); expect(capture).toHaveBeenCalledWith("cli_help", { topic: "(index)" }); - - const events = capture.mock.calls.map((call) => call[0] as string); - expect(events).not.toContain("help_index"); - expect(events.every((event) => !event.startsWith("help_"))).toBe(true); }); - it("captures the attempted topic for an unknown topic, and no legacy help_* event", async () => { + it("captures the attempted topic for an unknown topic", async () => { await runHelp(["help", "nope"]); expect(capture).toHaveBeenCalledWith("cli_help", { topic: "nope" }); + }); +}); + +// Rather than asserting "no help_* event" inside every behavioral test above, +// prove it once at the source: after this change lands, no legacy help_* event +// name is emitted anywhere in the CLI. +function collectSourceFiles(directory: string): string[] { + const files: string[] = []; + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const full = join(directory, entry.name); + if (entry.isDirectory()) files.push(...collectSourceFiles(full)); + else if (entry.name.endsWith(".ts")) files.push(full); + } + return files; +} - const events = capture.mock.calls.map((call) => call[0] as string); - expect(events).not.toContain("help_index"); - expect(events).not.toContain("help_unknown"); - expect(events.every((event) => !event.startsWith("help_"))).toBe(true); +describe("no legacy help_* event remains in the CLI source", () => { + it("emits no help_* event-name literal under src/", () => { + const sourceDirectory = resolve(import.meta.dirname, "../src"); + // Match a string/template literal that begins with help_ (e.g. "help_index", + // "help_unknown", or a `help_${...}` topic event). + const legacyHelpEvent = /["`]help_/; + const offenders = collectSourceFiles(sourceDirectory).filter((file) => + legacyHelpEvent.test(readFileSync(file, "utf8")) + ); + expect(offenders).toEqual([]); }); }); From 20d46d5aa5c2d59ab04c4a6958f634e857c7156b Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Sat, 13 Jun 2026 15:10:40 -0700 Subject: [PATCH 5/6] docs(openspec): Sync corrected check/help spec into the archived capability The archive commit synced the pre-review delta spec, so the capability spec carried the stale cli_check_completed{ filesScanned } and a fuzzy cli_help index-marker description. Reconcile the synced spec (and the archive design flow diagram) with the corrected contract: findings replaces filesScanned, and cli_help documents the exact literal "(index)" for the no-topic invocation. Co-Authored-By: Claude Opus 4.8 --- .../2026-06-13-restructure-cli-telemetry/design.md | 2 +- openspec/specs/analytics/spec.md | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md index 017512ee..254669b2 100644 --- a/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md +++ b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md @@ -55,7 +55,7 @@ auth login success → cli_authenticated { } auth logout success → cli_logged_out { } init/install success → cli_installed { targets? } onboard complete → cli_onboarded { } -check finishes → cli_check_completed{ errorCount, warningCount, filesScanned } +check finishes → cli_check_completed{ errorCount, warningCount, findings } any command fails → cli_error { command, code } help served → cli_help { topic } (topic = "(index)" for no-arg, the attempted topic otherwise) diff --git a/openspec/specs/analytics/spec.md b/openspec/specs/analytics/spec.md index dab92816..b8cc8f4b 100644 --- a/openspec/specs/analytics/spec.md +++ b/openspec/specs/analytics/spec.md @@ -147,13 +147,13 @@ CLI events SHALL use the `cli_` prefix, with the taxonomy organized as a - `cli_authenticated`, `cli_logged_out` - `cli_installed`, `cli_onboarded` - `cli_check_completed` — error/warning counts only (e.g. `errorCount`, - `warningCount`, `filesScanned`) + `warningCount`, `findings`) - `cli_error` — a single failure event with `command` and `code` (a stable `CliErrorCode`) - `cli_help` — fired when the help command serves a request, with a `topic` - property (the served topic, or an index marker when invoked with no topic). - This replaces the previous `help_index`, `help_`, and `help_unknown` - events. + property (the served topic; the exact literal `"(index)"` when invoked with no + topic; the attempted topic for an unknown request). This replaces the previous + `help_index`, `help_`, and `help_unknown` events. Commands that carry no concrete state beyond the invocation (e.g. `info`, `detect`, `update`, `auth status`, `rule verify`, `rule meta`) SHALL rely on @@ -174,11 +174,10 @@ Commands that carry no concrete state beyond the invocation (e.g. `info`, - **THEN** PostHog SHALL receive a `cli_help` event with `topic: "rule create"` - **AND** SHALL NOT receive a `help_rule_create` event -#### Scenario: Help with no topic emits cli_help with an index marker +#### Scenario: Help with no topic emits cli_help with the index marker - **WHEN** an agent runs `taskless help` -- **THEN** PostHog SHALL receive a `cli_help` event whose `topic` marks the index - (no-argument) invocation +- **THEN** PostHog SHALL receive a `cli_help` event with `topic: "(index)"` - **AND** SHALL NOT receive a `help_index` event #### Scenario: A command failure emits cli_error From b204084fdc76c1ea3bfa0a9c7ebde3486389746f Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Sat, 13 Jun 2026 16:41:17 -0700 Subject: [PATCH 6/6] docs(openspec): Uppercase CLIError/CLIErrorCode in the analytics spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match the source rename — the CLI acronym is uppercase in the CLIError class and CLIErrorCode type, so the spec prose and archived contract use the same casing. Co-Authored-By: Claude Opus 4.8 --- .../archive/2026-06-13-restructure-cli-telemetry/design.md | 4 ++-- .../specs/analytics/spec.md | 4 ++-- .../archive/2026-06-13-restructure-cli-telemetry/tasks.md | 2 +- openspec/specs/analytics/spec.md | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md index 254669b2..395760e4 100644 --- a/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md +++ b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md @@ -70,7 +70,7 @@ carries no concrete state beyond the invocation, so those are covered by ### D3 — `cli_error` is the single failure event Instead of `success:false` spread across each `_completed` event, failures emit -one `cli_error { command, code }` (code from the stable `CliErrorCode` set), and +one `cli_error { command, code }` (code from the stable `CLIErrorCode` set), and `cli_run` also records `success:false`. The runner emits `cli_error` from its catch path so no command has to remember to. @@ -86,7 +86,7 @@ Dashboards are rebuilt against the new names (the proposal calls this out). - **[Centralized `cli_run` can't see command-specific context]** → By design (D-non-goal). `loggedIn` covers the only cross-cutting dimension we need now. - **[`success` detection in the runner is imperfect]** → Derive from thrown - error and `process.exitCode`; commands already use `CliError` + exit codes + error and `process.exitCode`; commands already use `CLIError` + exit codes consistently, so this is reliable. ## Open Questions diff --git a/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/specs/analytics/spec.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/specs/analytics/spec.md index 5d325ce7..47eebac2 100644 --- a/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/specs/analytics/spec.md +++ b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/specs/analytics/spec.md @@ -44,7 +44,7 @@ CLI events SHALL use the `cli_` prefix, with the taxonomy organized as a - `cli_check_completed` — error/warning counts only (e.g. `errorCount`, `warningCount`, `findings`) - `cli_error` — a single failure event with `command` and `code` (a stable - `CliErrorCode`) + `CLIErrorCode`) - `cli_help` — fired when the help command serves a request, with a `topic` property. The `topic` SHALL be: the served topic for a known topic (e.g. `"rule create"`); the exact literal `"(index)"` when invoked with no topic; @@ -84,7 +84,7 @@ Commands that carry no concrete state beyond the invocation (e.g. `info`, #### Scenario: A command failure emits cli_error -- **WHEN** a command fails with a known `CliErrorCode` +- **WHEN** a command fails with a known `CLIErrorCode` - **THEN** PostHog SHALL receive a `cli_error` event with `command` and `code` #### Scenario: Old event names are not emitted diff --git a/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/tasks.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/tasks.md index 68041c98..aace17da 100644 --- a/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/tasks.md +++ b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/tasks.md @@ -27,7 +27,7 @@ keeps the suite green on its own. - [x] 1.1 In `packages/cli/src/index.ts`, wrap command execution so exactly one `cli_run` is emitted per invocation from a `finally`-equivalent path, with `{ command, cli_version, success, durationMs, anonymous, loggedIn }` - [x] 1.2 Resolve `command` from the matched citty subcommand (e.g. `"rule create"`, `"help"`); derive `success` from a thrown error / non-zero `process.exitCode`; measure `durationMs` from a start timestamp — extracted to a testable `telemetry-run.ts` (resolveCommandName/resolveCwd/emitRunEvents) so the entry module's side-effecting top level stays untested -- [x] 1.3 Emit `cli_error { command, code }` from the runner's catch path when the failure carries a stable `CliErrorCode` — added an optional `code` to `CliError`; falls back to `INTERNAL_ERROR` +- [x] 1.3 Emit `cli_error { command, code }` from the runner's catch path when the failure carries a stable `CLIErrorCode` — added an optional `code` to `CLIError`; falls back to `INTERNAL_ERROR` - [x] 1.4 Tests: one `cli_run` per invocation (success and failure), and `cli_error` on a known-code failure — `test/cli-run.test.ts` - [x] 1.5 typecheck + lint + suite green; commit; open PR 1 diff --git a/openspec/specs/analytics/spec.md b/openspec/specs/analytics/spec.md index b8cc8f4b..02dc1f4e 100644 --- a/openspec/specs/analytics/spec.md +++ b/openspec/specs/analytics/spec.md @@ -149,7 +149,7 @@ CLI events SHALL use the `cli_` prefix, with the taxonomy organized as a - `cli_check_completed` — error/warning counts only (e.g. `errorCount`, `warningCount`, `findings`) - `cli_error` — a single failure event with `command` and `code` (a stable - `CliErrorCode`) + `CLIErrorCode`) - `cli_help` — fired when the help command serves a request, with a `topic` property (the served topic; the exact literal `"(index)"` when invoked with no topic; the attempted topic for an unknown request). This replaces the previous @@ -182,7 +182,7 @@ Commands that carry no concrete state beyond the invocation (e.g. `info`, #### Scenario: A command failure emits cli_error -- **WHEN** a command fails with a known `CliErrorCode` +- **WHEN** a command fails with a known `CLIErrorCode` - **THEN** PostHog SHALL receive a `cli_error` event with `command` and `code` #### Scenario: Old event names are not emitted