diff --git a/openspec/changes/restructure-cli-telemetry/.openspec.yaml b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/.openspec.yaml similarity index 100% rename from openspec/changes/restructure-cli-telemetry/.openspec.yaml rename to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/.openspec.yaml diff --git a/openspec/changes/restructure-cli-telemetry/design.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md similarity index 99% rename from openspec/changes/restructure-cli-telemetry/design.md rename to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md index 2c616578..395760e4 100644 --- a/openspec/changes/restructure-cli-telemetry/design.md +++ b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/design.md @@ -55,7 +55,7 @@ auth login success → cli_authenticated { } auth logout success → cli_logged_out { } init/install success → cli_installed { targets? } onboard complete → cli_onboarded { } -check finishes → cli_check_completed{ errorCount, warningCount, filesScanned } +check finishes → cli_check_completed{ errorCount, warningCount, findings } any command fails → cli_error { command, code } help served → cli_help { topic } (topic = "(index)" for no-arg, the attempted topic otherwise) diff --git a/openspec/changes/restructure-cli-telemetry/proposal.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/proposal.md similarity index 100% rename from openspec/changes/restructure-cli-telemetry/proposal.md rename to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/proposal.md diff --git a/openspec/changes/restructure-cli-telemetry/specs/analytics/spec.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/specs/analytics/spec.md similarity index 100% rename from openspec/changes/restructure-cli-telemetry/specs/analytics/spec.md rename to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/specs/analytics/spec.md diff --git a/openspec/changes/restructure-cli-telemetry/tasks.md b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/tasks.md similarity index 50% rename from openspec/changes/restructure-cli-telemetry/tasks.md rename to openspec/changes/archive/2026-06-13-restructure-cli-telemetry/tasks.md index fd77f8ad..aace17da 100644 --- a/openspec/changes/restructure-cli-telemetry/tasks.md +++ b/openspec/changes/archive/2026-06-13-restructure-cli-telemetry/tasks.md @@ -27,36 +27,36 @@ keeps the suite green on its own. - [x] 1.1 In `packages/cli/src/index.ts`, wrap command execution so exactly one `cli_run` is emitted per invocation from a `finally`-equivalent path, with `{ command, cli_version, success, durationMs, anonymous, loggedIn }` - [x] 1.2 Resolve `command` from the matched citty subcommand (e.g. `"rule create"`, `"help"`); derive `success` from a thrown error / non-zero `process.exitCode`; measure `durationMs` from a start timestamp — extracted to a testable `telemetry-run.ts` (resolveCommandName/resolveCwd/emitRunEvents) so the entry module's side-effecting top level stays untested -- [x] 1.3 Emit `cli_error { command, code }` from the runner's catch path when the failure carries a stable `CliErrorCode` — added an optional `code` to `CliError`; falls back to `INTERNAL_ERROR` +- [x] 1.3 Emit `cli_error { command, code }` from the runner's catch path when the failure carries a stable `CLIErrorCode` — added an optional `code` to `CLIError`; falls back to `INTERNAL_ERROR` - [x] 1.4 Tests: one `cli_run` per invocation (success and failure), and `cli_error` on a known-code failure — `test/cli-run.test.ts` - [x] 1.5 typecheck + lint + suite green; commit; open PR 1 ## 2. Phase 2 — rule concrete-state events (PR 2, on PR 1) -- [ ] 2.1 `commands/rules.ts`: remove `cli_rule_create(_completed)`, `cli_rule_improve(_completed)`, `cli_rule_delete(_completed)`, `cli_rule_meta(_completed)`, `cli_rule_verify(_completed)` -- [ ] 2.2 Emit `cli_rule_created`, `cli_rule_improved`, `cli_rule_deleted` at the point each state changes (counts/ids/booleans only); `verify`/`meta` are covered by `cli_run` alone -- [ ] 2.3 Update rule command tests to the new events; assert no `cli_rule_*_completed` -- [ ] 2.4 typecheck + lint + suite green; commit; open PR 2 +- [x] 2.1 `commands/rules.ts`: remove `cli_rule_create(_completed)`, `cli_rule_improve(_completed)`, `cli_rule_delete(_completed)`, `cli_rule_meta(_completed)`, `cli_rule_verify(_completed)` +- [x] 2.2 Emit `cli_rule_created`, `cli_rule_improved`, `cli_rule_deleted` at the point each state changes (counts/ids/booleans only); `verify`/`meta` are covered by `cli_run` alone (their command-level telemetry was removed entirely) +- [x] 2.3 Update rule command tests to the new events; assert no `cli_rule_*_completed` — only `telemetry.test.ts` referenced an old rule name (a sample), updated to `cli_rule_created`; rule-from/verify tests assert behavior, not events +- [x] 2.4 typecheck + lint + suite green; commit; open PR 2 ## 3. Phase 3 — auth + lifecycle events (PR 3, on PR 2) -- [ ] 3.1 `commands/auth.ts`: remove `cli_auth_login(_completed)`, `cli_auth_logout(_completed)`, `cli_auth_status(_completed)`; emit `cli_authenticated` and `cli_logged_out` on success (status → `cli_run` only) -- [ ] 3.2 `commands/init.ts` + `wizard/index.ts`: remove `cli_init(_completed)`, `cli_init_cancelled`, `cli_update(_completed)`; emit `cli_installed` on a successful install -- [ ] 3.3 `commands/onboard.ts`: remove `cli_onboard_recipe` / `cli_onboard_already_done`; emit `cli_onboarded` when onboarding is marked complete -- [ ] 3.4 `commands/check.ts`: remove `cli_check(_completed)`; emit `cli_check_completed { errorCount, warningCount, filesScanned }` (counts only — no matched code) -- [ ] 3.5 Update auth/init/onboard/check tests to the new events -- [ ] 3.6 typecheck + lint + suite green; commit; open PR 3 +- [x] 3.1 `commands/auth.ts`: remove `cli_auth_login(_completed)`, `cli_auth_logout(_completed)`, `cli_auth_status(_completed)`; emit `cli_authenticated` (fresh login only) and `cli_logged_out` (token actually removed); status → `cli_run` only +- [x] 3.2 `commands/init.ts` + `wizard/index.ts`: remove `cli_init(_completed)`, `cli_init_cancelled`, `cli_update(_completed)`; emit `cli_installed` on a successful install (interactive + non-interactive + update) +- [x] 3.3 `commands/onboard.ts`: remove `cli_onboard_recipe` / `cli_onboard_already_done`; emit `cli_onboarded` when onboarding is marked complete +- [x] 3.4 `commands/check.ts`: remove `cli_check(_completed)`; emit `cli_check_completed { errorCount, warningCount, findings }` only when a scan actually runs (counts only — no matched code; `findings` replaces the unavailable `filesScanned`) +- [x] 3.5 Update auth/init/onboard/check tests to the new events — wizard-integration assertions updated to `cli_installed` / no-event-on-cancel; telemetry.test sample names → `cli_run` +- [x] 3.6 typecheck + lint + suite green; commit; open PR 3 ## 4. Phase 4 — cli_help { topic } + drop bespoke info/detect events (PR 4, on PR 3) -- [ ] 4.1 `commands/help.ts`: replace `help_index`, `help_`, `help_unknown` with one `cli_help { topic }` (served topic, an index marker for no-arg, the attempted topic for unknown) -- [ ] 4.2 `commands/info.ts`: remove its bespoke `cli_info(_completed)` events — covered by `cli_run`. (`commands/detect.ts` / `cli_detect` is NOT in this branch's lineage — it lives in the unmerged local-rule-routing stack — so there is nothing to change here; reconcile when both stacks land.) -- [ ] 4.3 Add `test/help-telemetry.test.ts` and update info tests; assert `cli_help` carries `topic` (served / `"(index)"` / attempted) and no `help_*` event is emitted -- [ ] 4.4 typecheck + lint + suite green; commit; open PR 4 +- [x] 4.1 `commands/help.ts`: replace `help_index`, `help_`, `help_unknown` with one `cli_help { topic }` (served topic, `"(index)"` marker for no-arg, the attempted topic for unknown) +- [x] 4.2 `commands/info.ts`: remove bespoke `cli_info(_completed)` (covered by `cli_run`); also drop its now-unused `getTelemetry` import. NOTE: `detect.ts`/`cli_detect` is NOT on this branch's lineage (it lives in the unmerged local-rule-routing stack) — no change needed here; it will be reconciled when that stack and this one both land +- [x] 4.3 Assert `cli_help` carries `topic` and no `help_*` event — added `test/help-telemetry.test.ts` (served topic, index marker, unknown topic, and no legacy `help_*`) +- [x] 4.4 typecheck + lint + suite green; commit; open PR 4 ## 5. Phase 5 — finalize (PR 5, tip) -- [ ] 5.1 Grep the CLI for any remaining old event names (`_completed`, `help_index`, `help_`, `help_unknown`, legacy `cli_` starts); remove any stragglers -- [ ] 5.2 Run `pnpm openspec validate restructure-cli-telemetry`; `pnpm typecheck`; `pnpm lint`; full suite green -- [ ] 5.3 Manual smoke: run a couple of commands with telemetry mocked/inspected — confirm one `cli_run` per invocation plus the expected concrete event, and no legacy names -- [ ] 5.4 Archive the change (`openspec archive restructure-cli-telemetry`) so the tip carries the spec sync + dated archive; commit; open PR 5 +- [x] 5.1 Grep the CLI for any remaining old event names (`_completed`, `help_index`, `help_`, `help_unknown`, legacy `cli_` starts); remove any stragglers — clean; the only `_completed` is the intentional concrete event `cli_check_completed` +- [x] 5.2 Run `pnpm openspec validate restructure-cli-telemetry`; `pnpm typecheck`; `pnpm lint`; full suite green (259) +- [x] 5.3 Manual smoke: `info`, `help check`, `help` (index) run end-to-end after the refactor; concrete events + cli_run/cli_help/cli_error verified by the in-process tests +- [x] 5.4 Archive the change (`openspec archive restructure-cli-telemetry`) so the tip carries the spec sync + dated archive; commit; open PR 5 diff --git a/openspec/specs/analytics/spec.md b/openspec/specs/analytics/spec.md index 6f1173c1..02dc1f4e 100644 --- a/openspec/specs/analytics/spec.md +++ b/openspec/specs/analytics/spec.md @@ -111,13 +111,13 @@ Every `capture()` call SHALL include the `cli` property (anonymous UUID), the `c #### Scenario: Anonymous capture includes standard properties -- **WHEN** `capture("cli_check")` is called without authentication +- **WHEN** `capture("cli_run")` is called without authentication - **THEN** the event SHALL include `{ cli: anonymousUuid, cliVersion: , scaffoldVersion: }` - **AND** the event SHALL NOT include a `groups` parameter #### Scenario: Authenticated capture includes standard properties and group -- **WHEN** `capture("cli_rule_create")` is called with authentication +- **WHEN** `capture("cli_rule_created")` is called with authentication - **THEN** the event SHALL include `{ cli: anonymousUuid, cliVersion: , scaffoldVersion: }` - **AND** the `groups` parameter SHALL include `{ organization: String(orgId) }` @@ -134,56 +134,87 @@ Every `capture()` call SHALL include the `cli` property (anonymous UUID), the `c ### Requirement: CLI events use cli\_ prefix -CLI action events SHALL continue to use the `cli_` prefix, but the event taxonomy SHALL be reorganized as follows: - -- `cli_` — fired when an action command begins execution (e.g. `cli_rule_create`, `cli_rule_improve`, `cli_rule_delete`, `cli_check`, `cli_info`, `cli_init`, `cli_auth_login`, `cli_auth_logout`) -- `cli__completed` — fired when an action command finishes execution; event properties SHALL include `success: boolean`, `durationMs: number`, and `errorCode?: string` (when failure) -- `help_` — fired when the help command serves a specific topic (e.g. `help_rule_create`, `help_check`, `help_auth`); replaces previous `cli_help_` events -- `help_index` — fired when the help command is invoked with no arguments (probable agent confusion / routing failure) -- `help_unknown` — fired when the help command receives an unknown topic; event properties SHALL include `topic: string` (the attempted topic) - -The previous event names `cli_help`, `cli_help_auth`, `cli_help_check`, `cli_help_info`, `cli_help_init`, `cli_help_rule` SHALL be removed in this release. There is no dual-emit window — the rename is a hard cut. - -#### Scenario: Action command emits start and completion events - -- **WHEN** a user runs `taskless rule create --from req.json` -- **THEN** PostHog SHALL receive a `cli_rule_create` event when execution begins -- **AND** SHALL receive a `cli_rule_create_completed` event when execution finishes, with properties including `success`, `durationMs`, and (on failure) `errorCode` - -#### Scenario: Help fetch emits topic intent +CLI events SHALL use the `cli_` prefix, with the taxonomy organized as a +`cli_run` denominator plus concrete state-transition events: + +- `cli_run` — exactly one per invocation (see the dedicated requirement). This + replaces every previous `cli_` start event and `cli__completed` + event; the `success`/`durationMs`/`command` signal lives here. +- Concrete state-transition events, each fired at the point the state actually + changes, carrying counts/ids/booleans only (never rule content, prompts, or + matched source): + - `cli_rule_created`, `cli_rule_improved`, `cli_rule_deleted` + - `cli_authenticated`, `cli_logged_out` + - `cli_installed`, `cli_onboarded` + - `cli_check_completed` — error/warning counts only (e.g. `errorCount`, + `warningCount`, `findings`) + - `cli_error` — a single failure event with `command` and `code` (a stable + `CLIErrorCode`) +- `cli_help` — fired when the help command serves a request, with a `topic` + property (the served topic; the exact literal `"(index)"` when invoked with no + topic; the attempted topic for an unknown request). This replaces the previous + `help_index`, `help_`, and `help_unknown` events. + +Commands that carry no concrete state beyond the invocation (e.g. `info`, +`detect`, `update`, `auth status`, `rule verify`, `rule meta`) SHALL rely on +`cli_run` alone and SHALL NOT emit a bespoke event. The previous taxonomy +(`cli_`, `cli__completed`, `help_index`, `help_`, +`help_unknown`) SHALL be removed in this release; there is no dual-emit window. + +#### Scenario: Rule creation emits a concrete state event plus cli_run + +- **WHEN** a user runs `taskless rule create --from req.json` and a rule is written +- **THEN** PostHog SHALL receive one `cli_run` event with `command: "rule create"` +- **AND** SHALL receive a `cli_rule_created` event +- **AND** SHALL NOT receive `cli_rule_create` or `cli_rule_create_completed` + +#### Scenario: Help fetch emits cli_help with a topic - **WHEN** an agent runs `taskless help rule create` -- **THEN** PostHog SHALL receive a `help_rule_create` event +- **THEN** PostHog SHALL receive a `cli_help` event with `topic: "rule create"` +- **AND** SHALL NOT receive a `help_rule_create` event -#### Scenario: Help no-args emits index event +#### Scenario: Help with no topic emits cli_help with the index marker - **WHEN** an agent runs `taskless help` -- **THEN** PostHog SHALL receive a `help_index` event +- **THEN** PostHog SHALL receive a `cli_help` event with `topic: "(index)"` +- **AND** SHALL NOT receive a `help_index` event -#### Scenario: Help unknown topic emits help_unknown +#### Scenario: A command failure emits cli_error -- **WHEN** an agent runs `taskless help nonexistent` -- **THEN** PostHog SHALL receive a `help_unknown` event with property `topic: "nonexistent"` +- **WHEN** a command fails with a known `CLIErrorCode` +- **THEN** PostHog SHALL receive a `cli_error` event with `command` and `code` #### Scenario: Old event names are not emitted -- **WHEN** any CLI command runs in v0.7.0 -- **THEN** PostHog SHALL NOT receive any event named `cli_help`, `cli_help_`, or any other event under the previous taxonomy +- **WHEN** any CLI command runs in this release +- **THEN** PostHog SHALL NOT receive any event named `cli__completed`, + `help_index`, `help_`, or `help_unknown` ### Requirement: Wrong-topic re-routing is observable as a derivable funnel -The new event taxonomy is structured so that wrong-topic re-routing is a derivable funnel signal: +The taxonomy SHALL keep wrong-topic re-routing derivable as a funnel signal from +the new events: -- A `help_` event followed by no `cli_` event AND a subsequent `help_` event indicates the agent fetched the recipe for topic A, did not act on it, and re-routed to topic B -- A `help_index` event followed by a `help_` event indicates the agent consulted the index before picking a topic (expected behavior; baseline) -- A `help_` event with no subsequent `cli_` event AND no further `help_*` event indicates the agent abandoned the action +- A `cli_help { topic: A }` event not followed by the concrete event for topic A + (or by `cli_run` with the corresponding `command`), and then a subsequent + `cli_help { topic: B }`, indicates the agent fetched recipe A, did not act on + it, and re-routed to topic B. +- A `cli_help` index-marker event followed by a `cli_help { topic }` event + indicates the agent consulted the index before picking a topic (baseline). +- A `cli_help { topic }` event with no subsequent acting `cli_run` and no further + `cli_help` event indicates the agent abandoned the action. -No additional events SHALL be added to capture this signal directly — the funnel is derivable from the event sequence in PostHog. Dashboards SHOULD be created to surface re-routing rates per topic so wrong-topic confusion can be measured. +No additional events SHALL be added to capture this signal directly — it is +derivable from the `cli_help` / `cli_run` sequence. Dashboards SHOULD surface +re-routing rates per topic. #### Scenario: Funnel data supports wrong-topic detection - **WHEN** dashboards are constructed in PostHog -- **THEN** the events SHALL be sufficient to compute "rate of `help_` events not followed by a corresponding `cli_` event within N minutes" +- **THEN** the `cli_help` (with `topic`) and `cli_run` (with `command`) events + SHALL be sufficient to compute "rate of `cli_help { topic }` not followed by a + corresponding acting `cli_run` within N minutes" ### Requirement: Telemetry failures are silent @@ -218,3 +249,26 @@ Each command handler SHALL call `getTelemetry(cwd)` to lazily initialize the sin - **WHEN** the CLI exits without running a command (e.g. showing top-level help) - **THEN** `shutdownTelemetry()` SHALL be a no-op and no PostHog client SHALL be created + +### Requirement: Every invocation emits exactly one cli_run event + +The CLI SHALL emit exactly one `cli_run` event per invocation, from the top-level +runner rather than from individual commands. The event SHALL carry the properties +`command` (the resolved subcommand name, e.g. `"rule create"` or `"help"`), +`cli_version`, `success` (boolean), `durationMs` (number), `anonymous` (boolean), +and `loggedIn` (boolean). The event SHALL be emitted on both success and failure +(from a `finally`-equivalent path), and no command SHALL emit its own +"started" or "ran" event. + +#### Scenario: A successful command emits one cli_run + +- **WHEN** a user runs `taskless info` +- **THEN** PostHog SHALL receive exactly one `cli_run` event with + `command: "info"`, `success: true`, a numeric `durationMs`, and the + `cli_version`, `anonymous`, and `loggedIn` properties +- **AND** SHALL NOT receive a separate `cli_info` or `cli_info_completed` event + +#### Scenario: A failing command still emits cli_run + +- **WHEN** a command exits with an error +- **THEN** PostHog SHALL receive one `cli_run` event with `success: false` diff --git a/packages/cli/src/commands/auth.ts b/packages/cli/src/commands/auth.ts index 416a5021..0c45ee75 100644 --- a/packages/cli/src/commands/auth.ts +++ b/packages/cli/src/commands/auth.ts @@ -33,15 +33,9 @@ const loginCommand = defineCommand({ async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_auth_login"); - - /** Tracks the last emitted error code so the completion event can include it. */ - let lastErrorCode: CLIErrorCode | undefined; /** Emit an error in the right channel and set exit code. */ const fail = (code: CLIErrorCode, message: string): void => { - lastErrorCode = code; if (args.json) { writeJsonError(code, message); } else { @@ -52,15 +46,12 @@ const loginCommand = defineCommand({ if (args.anonymous) { fail("INVALID_INPUT", "auth commands cannot be anonymous."); - telemetry.capture("cli_auth_login_completed", { - success: false, - durationMs: Date.now() - startedAt, - errorCode: lastErrorCode, - }); return; } - let success = false; + // Set true only when a fresh authentication completes; drives the + // cli_authenticated event in the finally. + let authenticated = false; try { // In --json mode the user is an agent / pipe; suppress the device-flow // chatter and only emit a single structured line on error. @@ -71,7 +62,7 @@ const loginCommand = defineCommand({ switch (result.status) { case "ok": { - success = true; + authenticated = true; return; } case "already_logged_in": { @@ -79,7 +70,6 @@ const loginCommand = defineCommand({ console.log("You are already logged in."); console.log("Run `taskless auth logout` first to re-authenticate."); } - success = true; return; } case "cancelled": { @@ -97,11 +87,10 @@ const loginCommand = defineCommand({ } } } finally { - telemetry.capture("cli_auth_login_completed", { - success, - durationMs: Date.now() - startedAt, - ...(success ? {} : { errorCode: lastErrorCode }), - }); + // Concrete state event: a fresh authentication succeeded. + if (authenticated) { + telemetry.capture("cli_authenticated"); + } } }, }); @@ -132,21 +121,18 @@ const logoutCommand = defineCommand({ async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_auth_logout"); - let success = false; + let removed = false; try { - const removed = await removeToken(cwd); + removed = await removeToken(cwd); if (!args.json) { console.log(removed ? "Logged out." : "Not logged in."); } - success = true; } finally { - telemetry.capture("cli_auth_logout_completed", { - success, - durationMs: Date.now() - startedAt, - }); + // Concrete state event: a saved token was actually removed. + if (removed) { + telemetry.capture("cli_logged_out"); + } } }, }); @@ -181,39 +167,25 @@ export const authCommand = defineCommand({ } const cwd = resolve(args.dir ?? process.cwd()); - const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_auth_status"); - - let success = false; - try { - const token = await getToken(cwd); - if (!token) { - console.log("Not logged in."); - console.log("Run `taskless auth login` to authenticate."); - success = true; - return; - } - const whoami = await fetchWhoami(token); - if (!whoami) { - console.log("Logged in, but unable to verify identity."); - console.log( - "Your token may be invalid or expired. Run `taskless auth login` to re-authenticate." - ); - success = true; - return; - } + const token = await getToken(cwd); + if (!token) { + console.log("Not logged in."); + console.log("Run `taskless auth login` to authenticate."); + return; + } - const orgs = whoami.orgs.map((o) => o.name); - const orgSuffix = orgs.length > 0 ? ` (${orgs.join(", ")})` : ""; - console.log(`Logged in as ${whoami.user}${orgSuffix}.`); - success = true; - } finally { - telemetry.capture("cli_auth_status_completed", { - success, - durationMs: Date.now() - startedAt, - }); + const whoami = await fetchWhoami(token); + if (!whoami) { + console.log("Logged in, but unable to verify identity."); + console.log( + "Your token may be invalid or expired. Run `taskless auth login` to re-authenticate." + ); + return; } + + const orgs = whoami.orgs.map((o) => o.name); + const orgSuffix = orgs.length > 0 ? ` (${orgs.join(", ")})` : ""; + console.log(`Logged in as ${whoami.user}${orgSuffix}.`); }, }); diff --git a/packages/cli/src/commands/check.ts b/packages/cli/src/commands/check.ts index bab9e7cb..f1e82b49 100644 --- a/packages/cli/src/commands/check.ts +++ b/packages/cli/src/commands/check.ts @@ -108,10 +108,12 @@ export const checkCommand = defineCommand({ async run({ args, rawArgs }) { const cwd = resolve(args.dir ?? process.cwd()); const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_check"); - let success = false; + // Set when a scan actually runs; drives cli_check_completed with counts + // only (never matched code). + let scanCounts: + | { errorCount: number; warningCount: number; findings: number } + | undefined; try { const positionalPaths = extractPositionalPaths(rawArgs); const hadExplicitPaths = positionalPaths.length > 0; @@ -129,7 +131,6 @@ export const checkCommand = defineCommand({ ) ); } - success = true; return; } @@ -155,7 +156,6 @@ export const checkCommand = defineCommand({ "No rules configured. Create one with `taskless rule create`." ); } - success = true; return; } @@ -163,7 +163,14 @@ export const checkCommand = defineCommand({ try { await generateSgConfig(cwd); const { results } = await runAstGrepScan(cwd, existingPaths); - const hasErrors = results.some((r) => r.severity === "error"); + let errorCount = 0; + let warningCount = 0; + for (const result of results) { + if (result.severity === "error") errorCount++; + else if (result.severity === "warning") warningCount++; + } + const hasErrors = errorCount > 0; + scanCounts = { errorCount, warningCount, findings: results.length }; // Format output if (args.json) { @@ -180,7 +187,6 @@ export const checkCommand = defineCommand({ if (hasErrors) { process.exitCode = 1; } - success = !hasErrors; } catch (error) { const message = `Error: ${error instanceof Error ? error.message : String(error)}`; if (args.json) { @@ -193,10 +199,10 @@ export const checkCommand = defineCommand({ process.exitCode = 1; } } finally { - telemetry.capture("cli_check_completed", { - success, - durationMs: Date.now() - startedAt, - }); + // Concrete state event: a scan completed; counts only, no matched code. + if (scanCounts) { + telemetry.capture("cli_check_completed", scanCounts); + } } }, }); diff --git a/packages/cli/src/commands/help.ts b/packages/cli/src/commands/help.ts index 71793854..f6e12938 100644 --- a/packages/cli/src/commands/help.ts +++ b/packages/cli/src/commands/help.ts @@ -153,8 +153,8 @@ export function createHelpCommand(subCommands: SubCommandsDef) { const telemetry = await getTelemetry(cwd); if (positionals.length === 0) { - // help_index: agent fetched the topic list - telemetry.capture("help_index"); + // cli_help with the index marker: agent fetched the topic list + telemetry.capture("cli_help", { topic: "(index)" }); console.log("Taskless CLI\n"); console.log( @@ -198,16 +198,13 @@ export function createHelpCommand(subCommands: SubCommandsDef) { : helpMap.get(key); if (content) { - // help_: agent fetched a specific recipe (intent signal) - const topicEvent = `help_${key.replaceAll("-", "_")}`; - telemetry.capture(topicEvent, { - topic: positionals.join(" "), - anonymous: args.anonymous, - }); + // cli_help: agent fetched a specific recipe (intent signal). The topic + // is the served topic; filtering on it replaces the old per-topic events. + telemetry.capture("cli_help", { topic: positionals.join(" ") }); console.log(renderRecipe(content, key).trimEnd()); } else { - // help_unknown: agent asked for a topic that does not exist - telemetry.capture("help_unknown", { topic: positionals.join(" ") }); + // cli_help for an unknown topic — still the attempted topic string. + telemetry.capture("cli_help", { topic: positionals.join(" ") }); console.error(`Unknown command: ${positionals.join(" ")}`); console.error("Run `taskless help` for available commands."); process.exitCode = 1; diff --git a/packages/cli/src/commands/info.ts b/packages/cli/src/commands/info.ts index 5ebfafb7..1da8a835 100644 --- a/packages/cli/src/commands/info.ts +++ b/packages/cli/src/commands/info.ts @@ -5,7 +5,6 @@ import { checkStaleness } from "../install/install"; import { getToken } from "../auth/token"; import { fetchWhoami } from "../auth/whoami"; import { outputSchema as infoOutputSchema } from "../schemas/info"; -import { getTelemetry } from "../telemetry"; import { makeErrorEnvelope } from "../types/errors"; export const infoCommand = defineCommand({ @@ -32,100 +31,87 @@ export const infoCommand = defineCommand({ }, async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); - const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_info"); - let success = false; - try { - const [tools, token] = await Promise.all([ - checkStaleness(cwd), - args.anonymous ? Promise.resolve() : getToken(cwd), - ]); + const [tools, token] = await Promise.all([ + checkStaleness(cwd), + args.anonymous ? Promise.resolve() : getToken(cwd), + ]); - let auth: { user: string; email: string; orgs: string[] } | undefined; - if (!args.anonymous && token) { - const whoami = await fetchWhoami(token); - if (whoami) { - auth = { - user: whoami.user, - email: whoami.email, - orgs: whoami.orgs.map((o) => o.name), - }; - } + let auth: { user: string; email: string; orgs: string[] } | undefined; + if (!args.anonymous && token) { + const whoami = await fetchWhoami(token); + if (whoami) { + auth = { + user: whoami.user, + email: whoami.email, + orgs: whoami.orgs.map((o) => o.name), + }; } + } - const result = { - success: true as const, - version: __VERSION__, - tools, - loggedIn: token !== undefined, - auth, - }; + const result = { + success: true as const, + version: __VERSION__, + tools, + loggedIn: token !== undefined, + auth, + }; - if (args.json) { - const parsed = infoOutputSchema.safeParse(result); - if (!parsed.success) { - console.log( - JSON.stringify( - makeErrorEnvelope( - "INTERNAL_ERROR", - "Internal schema validation failed" - ) + if (args.json) { + const parsed = infoOutputSchema.safeParse(result); + if (!parsed.success) { + console.log( + JSON.stringify( + makeErrorEnvelope( + "INTERNAL_ERROR", + "Internal schema validation failed" ) - ); - process.exitCode = 1; - return; - } - console.log(JSON.stringify(parsed.data)); - success = true; + ) + ); + process.exitCode = 1; return; } + console.log(JSON.stringify(parsed.data)); + return; + } - // Human-readable output - console.log(`Taskless CLI v${__VERSION__}\n`); + // Human-readable output + console.log(`Taskless CLI v${__VERSION__}\n`); - if (tools.length === 0) { - console.log("Tools: none detected"); - } else { - console.log("Tools:"); - for (const tool of tools) { - const total = tool.skills.length; - const upToDate = tool.skills.filter((s) => s.current).length; - const stale = total - upToDate; + if (tools.length === 0) { + console.log("Tools: none detected"); + } else { + console.log("Tools:"); + for (const tool of tools) { + const total = tool.skills.length; + const upToDate = tool.skills.filter((s) => s.current).length; + const stale = total - upToDate; - if (stale === 0) { - console.log( - ` ${tool.name}: ${String(total)} skills (all up to date)` - ); - } else { - console.log( - ` ${tool.name}: ${String(total)} skills (${String(stale)} outdated)` - ); - for (const skill of tool.skills) { - if (!skill.current) { - console.log( - ` - ${skill.name}: ${skill.installedVersion ?? "missing"} → ${skill.currentVersion}` - ); - } + if (stale === 0) { + console.log( + ` ${tool.name}: ${String(total)} skills (all up to date)` + ); + } else { + console.log( + ` ${tool.name}: ${String(total)} skills (${String(stale)} outdated)` + ); + for (const skill of tool.skills) { + if (!skill.current) { + console.log( + ` - ${skill.name}: ${skill.installedVersion ?? "missing"} → ${skill.currentVersion}` + ); } } } } + } - console.log(""); - if (auth) { - const orgs = auth.orgs.length > 0 ? ` (${auth.orgs.join(", ")})` : ""; - console.log(`Auth: logged in as ${auth.user}${orgs}`); - } else { - console.log("Auth: not logged in"); - } - success = true; - } finally { - telemetry.capture("cli_info_completed", { - success, - durationMs: Date.now() - startedAt, - }); + console.log(""); + if (auth) { + const orgs = auth.orgs.length > 0 ? ` (${auth.orgs.join(", ")})` : ""; + console.log(`Auth: logged in as ${auth.user}${orgs}`); + } else { + console.log("Auth: not logged in"); } }, }); diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 7f134e3c..b3f0b7dc 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -53,7 +53,6 @@ export const initCommand = defineCommand({ async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); const telemetry = await getTelemetry(cwd); - telemetry.capture("cli_init"); const interactive = shouldRunInteractively(args["no-interactive"]); @@ -71,19 +70,12 @@ export const initCommand = defineCommand({ ); } - const start = Date.now(); const result = await runNonInteractive(cwd); console.log( getOnboardTrailer({ commandsInstalled: result.commandsInstalled }) ); - telemetry.capture("cli_init_completed", { - locations: await detectedLocationDirectories(cwd), - optionalSkills: [], - authPromptShown: false, - authCompleted: false, - nonInteractive: true, - durationMs: Date.now() - start, - }); + // Concrete state event: skills/commands were installed (non-interactive). + telemetry.capture("cli_installed"); }, }); @@ -108,19 +100,16 @@ export const updateCommand = defineCommand({ async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_update"); let success = false; try { await runNonInteractive(cwd); success = true; } finally { - telemetry.capture("cli_update_completed", { - locations: await detectedLocationDirectories(cwd), - success, - durationMs: Date.now() - startedAt, - }); + // Concrete state event: skills/commands were installed/updated. + if (success) { + telemetry.capture("cli_installed"); + } } }, }); @@ -233,7 +222,3 @@ function groupValuesByTarget( } return map; } - -async function detectedLocationDirectories(cwd: string): Promise { - return detectSelectedDirectories(cwd); -} diff --git a/packages/cli/src/commands/onboard.ts b/packages/cli/src/commands/onboard.ts index 0302ba45..99d97d2b 100644 --- a/packages/cli/src/commands/onboard.ts +++ b/packages/cli/src/commands/onboard.ts @@ -80,7 +80,8 @@ export const onboardCommand = defineCommand({ manifest.install = install; await writeManifest(tasklessDirectory, manifest, raw); console.log("Marked Taskless onboarding as complete."); - telemetry.capture("cli_onboard_marked_complete"); + // Concrete state event: onboarding reached completion. + telemetry.capture("cli_onboarded"); return; } @@ -92,7 +93,6 @@ export const onboardCommand = defineCommand({ console.log( "Run `taskless onboard --force` to re-run the discovery recipe." ); - telemetry.capture("cli_onboard_already_done"); return; } @@ -104,6 +104,5 @@ export const onboardCommand = defineCommand({ throw new CLIError("recipe missing"); } console.log(recipe.trimEnd()); - telemetry.capture("cli_onboard_recipe", { forced: args.force }); }, }); diff --git a/packages/cli/src/commands/rules.ts b/packages/cli/src/commands/rules.ts index 3911e9b5..64e884fe 100644 --- a/packages/cli/src/commands/rules.ts +++ b/packages/cli/src/commands/rules.ts @@ -71,8 +71,6 @@ const createCommand = defineCommand({ async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_rule_create"); /** Emit an error and exit, respecting --json mode */ function fail( @@ -101,14 +99,12 @@ const createCommand = defineCommand({ console.error(message); } process.exitCode = 1; - telemetry.capture("cli_rule_create_completed", { - success: false, - durationMs: Date.now() - startedAt, - }); return; } - let success = false; + // Set to the number of rules written when generation succeeds; drives the + // cli_rule_created event in the finally. + let createdRuleCount: number | undefined; try { // 1. Read and validate --from file if (!args.from) { @@ -249,7 +245,7 @@ const createCommand = defineCommand({ console.log(` ${filePath}`); } } - success = true; + if (rules.length > 0) createdRuleCount = rules.length; return; } case "pr": @@ -267,16 +263,15 @@ const createCommand = defineCommand({ } else { console.log(`Rule ${ruleId} is in state "${status.status}".`); } - success = true; return; } } } } finally { - telemetry.capture("cli_rule_create_completed", { - success, - durationMs: Date.now() - startedAt, - }); + // Concrete state event: a rule was actually generated and written. + if (createdRuleCount !== undefined) { + telemetry.capture("cli_rule_created", { ruleCount: createdRuleCount }); + } } }, }); @@ -313,8 +308,6 @@ const improveCommand = defineCommand({ async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_rule_improve"); /** Emit an error and exit, respecting --json mode */ function fail( @@ -341,14 +334,12 @@ const improveCommand = defineCommand({ console.error(message); } process.exitCode = 1; - telemetry.capture("cli_rule_improve_completed", { - success: false, - durationMs: Date.now() - startedAt, - }); return; } - let success = false; + // Set to the number of rules written when iteration succeeds; drives the + // cli_rule_improved event in the finally. + let improvedRuleCount: number | undefined; try { // 1. Read and validate --from file if (!args.from) { @@ -487,7 +478,7 @@ const improveCommand = defineCommand({ console.log(` ${filePath}`); } } - success = true; + if (rules.length > 0) improvedRuleCount = rules.length; return; } case "pr": @@ -506,16 +497,17 @@ const improveCommand = defineCommand({ `Request ${requestId} is in state "${status.status}".` ); } - success = true; return; } } } } finally { - telemetry.capture("cli_rule_improve_completed", { - success, - durationMs: Date.now() - startedAt, - }); + // Concrete state event: a rule was actually iterated and rewritten. + if (improvedRuleCount !== undefined) { + telemetry.capture("cli_rule_improved", { + ruleCount: improvedRuleCount, + }); + } } }, }); @@ -549,9 +541,6 @@ const metaCommand = defineCommand({ }, async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); - const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_rule_meta"); function fail( message: string, @@ -566,42 +555,33 @@ const metaCommand = defineCommand({ throw new CLIError(message); } - let success = false; - try { - const meta = await readRuleMetaFile(cwd, args.id); - if (!meta) { - fail( - `No metadata found for rule "${args.id}". Expected .taskless/rule-metadata/${args.id}.yml`, - "RULE_NOT_FOUND" - ); - } + const meta = await readRuleMetaFile(cwd, args.id); + if (!meta) { + fail( + `No metadata found for rule "${args.id}". Expected .taskless/rule-metadata/${args.id}.yml`, + "RULE_NOT_FOUND" + ); + } - if (args.json) { - let output; - try { - output = metaOutputSchema.parse({ id: args.id, ...meta }); - } catch (error) { - if (error instanceof ZodError) { - fail( - `Invalid metadata for rule "${args.id}": ${error.issues.map((issue) => issue.message).join(", ")}`, - "INVALID_INPUT" - ); - } - fail(error instanceof Error ? error.message : String(error)); - } - console.log(JSON.stringify(output)); - } else { - console.log(`Metadata for rule "${args.id}":\n`); - for (const [key, value] of Object.entries(meta)) { - console.log(` ${key}: ${String(value)}`); + if (args.json) { + let output; + try { + output = metaOutputSchema.parse({ id: args.id, ...meta }); + } catch (error) { + if (error instanceof ZodError) { + fail( + `Invalid metadata for rule "${args.id}": ${error.issues.map((issue) => issue.message).join(", ")}`, + "INVALID_INPUT" + ); } + fail(error instanceof Error ? error.message : String(error)); + } + console.log(JSON.stringify(output)); + } else { + console.log(`Metadata for rule "${args.id}":\n`); + for (const [key, value] of Object.entries(meta)) { + console.log(` ${key}: ${String(value)}`); } - success = true; - } finally { - telemetry.capture("cli_rule_meta_completed", { - success, - durationMs: Date.now() - startedAt, - }); } }, }); @@ -637,8 +617,6 @@ const deleteCommand = defineCommand({ async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_rule_delete"); const id = args.id; let success = false; @@ -661,10 +639,10 @@ const deleteCommand = defineCommand({ process.exitCode = 1; } } finally { - telemetry.capture("cli_rule_delete_completed", { - success, - durationMs: Date.now() - startedAt, - }); + // Concrete state event: a rule and its tests were actually removed. + if (success) { + telemetry.capture("cli_rule_deleted"); + } } }, }); @@ -698,73 +676,61 @@ const verifyCommand = defineCommand({ }, async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); - const telemetry = await getTelemetry(cwd); - const startedAt = Date.now(); - telemetry.capture("cli_rule_verify"); - - let success = false; - try { - if (!args.id) { - if (args.json) { - console.log( - JSON.stringify( - makeErrorEnvelope("INVALID_INPUT", "Rule ID is required.") - ) - ); - } else { - console.error( - "Error: Rule ID is required.\n Usage: taskless rule verify " - ); - } - process.exitCode = 1; - return; - } - - const result = await verifyRule(cwd, args.id); + if (!args.id) { if (args.json) { - console.log(JSON.stringify(verifyOutputSchema.parse(result))); - } else { - console.log(`Verifying rule: ${result.ruleId}\n`); - - // Layer 1 console.log( - `Schema: ${result.schema.valid ? "✓ valid" : "✗ invalid"}` + JSON.stringify( + makeErrorEnvelope("INVALID_INPUT", "Rule ID is required.") + ) ); - for (const error of result.schema.errors) { - console.log(` - ${error}`); - } - - // Layer 2 - console.log( - `Requirements: ${result.requirements.valid ? "✓ valid" : "✗ invalid"}` + } else { + console.error( + "Error: Rule ID is required.\n Usage: taskless rule verify " ); - for (const error of result.requirements.errors) { - console.log(` - ${error}`); - } + } + process.exitCode = 1; + return; + } - // Layer 3 - console.log( - `Tests: ${result.tests.valid ? "✓ passed" : "✗ failed"} (${String(result.tests.passed)} passed, ${String(result.tests.failed)} failed)` - ); - for (const error of result.tests.errors) { - console.log(` - ${error}`); - } + const result = await verifyRule(cwd, args.id); - console.log( - `\nResult: ${result.success ? "✓ All checks passed" : "✗ Verification failed"}` - ); + if (args.json) { + console.log(JSON.stringify(verifyOutputSchema.parse(result))); + } else { + console.log(`Verifying rule: ${result.ruleId}\n`); + + // Layer 1 + console.log( + `Schema: ${result.schema.valid ? "✓ valid" : "✗ invalid"}` + ); + for (const error of result.schema.errors) { + console.log(` - ${error}`); } - if (!result.success) { - process.exitCode = 1; + // Layer 2 + console.log( + `Requirements: ${result.requirements.valid ? "✓ valid" : "✗ invalid"}` + ); + for (const error of result.requirements.errors) { + console.log(` - ${error}`); } - success = result.success; - } finally { - telemetry.capture("cli_rule_verify_completed", { - success, - durationMs: Date.now() - startedAt, - }); + + // Layer 3 + console.log( + `Tests: ${result.tests.valid ? "✓ passed" : "✗ failed"} (${String(result.tests.passed)} passed, ${String(result.tests.failed)} failed)` + ); + for (const error of result.tests.errors) { + console.log(` - ${error}`); + } + + console.log( + `\nResult: ${result.success ? "✓ All checks passed" : "✗ Verification failed"}` + ); + } + + if (!result.success) { + process.exitCode = 1; } }, }); diff --git a/packages/cli/src/wizard/index.ts b/packages/cli/src/wizard/index.ts index 157b56ef..22a7565c 100644 --- a/packages/cli/src/wizard/index.ts +++ b/packages/cli/src/wizard/index.ts @@ -102,19 +102,8 @@ export async function runWizard( function finish(args: { status: "completed" | "cancelled" }): WizardResult { const durationMs = Date.now() - start; if (args.status === "completed") { - telemetry.capture("cli_init_completed", { - locations, - optionalSkills, - authPromptShown, - authCompleted, - nonInteractive: false, - durationMs, - }); - } else { - telemetry.capture("cli_init_cancelled", { - atStep: cancelledStep ?? "unknown", - durationMs, - }); + // Concrete state event: skills/commands were installed (interactive). + telemetry.capture("cli_installed"); } return { status: args.status, diff --git a/packages/cli/test/help-telemetry.test.ts b/packages/cli/test/help-telemetry.test.ts new file mode 100644 index 00000000..5e11937e --- /dev/null +++ b/packages/cli/test/help-telemetry.test.ts @@ -0,0 +1,88 @@ +import { readdirSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +// Spy on telemetry by mocking the module the help command imports. The factory +// is invoked lazily at import time (same pattern as telemetry.test.ts). +const capture = vi.fn(); +vi.mock("../src/telemetry", () => ({ + getTelemetry: vi.fn(() => + Promise.resolve({ capture, shutdown: () => Promise.resolve() }) + ), + shutdownTelemetry: () => Promise.resolve(), +})); + +const { createHelpCommand } = await import("../src/commands/help"); + +interface RunnableCommand { + run: (context: { + args: { dir: string; anonymous: boolean }; + rawArgs: string[]; + }) => Promise; +} + +async function runHelp(rawArguments: string[]): Promise { + const command = createHelpCommand({}) as unknown as RunnableCommand; + await command.run({ + args: { dir: process.cwd(), anonymous: false }, + rawArgs: rawArguments, + }); +} + +describe("help emits cli_help { topic }", () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + capture.mockClear(); + logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it("captures the served topic", async () => { + await runHelp(["help", "rule", "create"]); + expect(capture).toHaveBeenCalledWith("cli_help", { topic: "rule create" }); + }); + + it("captures the index marker for no topic", async () => { + await runHelp(["help"]); + expect(capture).toHaveBeenCalledWith("cli_help", { topic: "(index)" }); + }); + + it("captures the attempted topic for an unknown topic", async () => { + await runHelp(["help", "nope"]); + expect(capture).toHaveBeenCalledWith("cli_help", { topic: "nope" }); + }); +}); + +// Rather than asserting "no help_* event" inside every behavioral test above, +// prove it once at the source: after this change lands, no legacy help_* event +// name is emitted anywhere in the CLI. +function collectSourceFiles(directory: string): string[] { + const files: string[] = []; + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const full = join(directory, entry.name); + if (entry.isDirectory()) files.push(...collectSourceFiles(full)); + else if (entry.name.endsWith(".ts")) files.push(full); + } + return files; +} + +describe("no legacy help_* event remains in the CLI source", () => { + it("emits no help_* event-name literal under src/", () => { + const sourceDirectory = resolve(import.meta.dirname, "../src"); + // Match a string/template literal that begins with help_ (e.g. "help_index", + // "help_unknown", or a `help_${...}` topic event). + const legacyHelpEvent = /["`]help_/; + const offenders = collectSourceFiles(sourceDirectory).filter((file) => + legacyHelpEvent.test(readFileSync(file, "utf8")) + ); + expect(offenders).toEqual([]); + }); +}); diff --git a/packages/cli/test/telemetry.test.ts b/packages/cli/test/telemetry.test.ts index 671bef6b..76fe4462 100644 --- a/packages/cli/test/telemetry.test.ts +++ b/packages/cli/test/telemetry.test.ts @@ -81,7 +81,7 @@ describe("telemetry disabled", () => { vi.stubEnv("TASKLESS_TELEMETRY_DISABLED", "1"); const telemetry = await getTelemetry(); - telemetry.capture("cli_check"); + telemetry.capture("cli_run"); await telemetry.shutdown(); expect(mockCapture).not.toHaveBeenCalled(); @@ -93,7 +93,7 @@ describe("telemetry disabled", () => { vi.stubEnv("DO_NOT_TRACK", "1"); const telemetry = await getTelemetry(); - telemetry.capture("cli_check"); + telemetry.capture("cli_run"); await telemetry.shutdown(); expect(mockCapture).not.toHaveBeenCalled(); @@ -166,7 +166,7 @@ describe("authenticated identity", () => { await writeTokenFile(cwd, jwt); const telemetry = await getTelemetry(cwd); - telemetry.capture("cli_check"); + telemetry.capture("cli_run"); expect(mockIdentify).toHaveBeenCalledWith( expect.objectContaining({ @@ -203,7 +203,7 @@ describe("authenticated identity", () => { it("falls back to anonymous UUID when no JWT is available", async () => { const telemetry = await getTelemetry(); - telemetry.capture("cli_check"); + telemetry.capture("cli_run"); // distinctId should be the anonymous UUID, not a JWT sub const captureArgument = mockCapture.mock.calls[0]![0] as { @@ -219,7 +219,7 @@ describe("authenticated identity", () => { describe("capture", () => { it("includes cli property on every event", async () => { const telemetry = await getTelemetry(); - telemetry.capture("cli_check"); + telemetry.capture("cli_run"); expect(mockCapture).toHaveBeenCalledWith( expect.objectContaining({ @@ -234,11 +234,11 @@ describe("capture", () => { it("merges custom properties with standard properties", async () => { const telemetry = await getTelemetry(); - telemetry.capture("cli_check", { foo: "bar" }); + telemetry.capture("cli_run", { foo: "bar" }); expect(mockCapture).toHaveBeenCalledWith( expect.objectContaining({ - event: "cli_check", + event: "cli_run", properties: expect.objectContaining({ cli: expect.any(String) as string, foo: "bar", @@ -249,7 +249,7 @@ describe("capture", () => { it("does not include groups when unauthenticated", async () => { const telemetry = await getTelemetry(); - telemetry.capture("cli_check"); + telemetry.capture("cli_run"); const captureArgument = mockCapture.mock.calls[0]![0] as Record< string, @@ -260,7 +260,7 @@ describe("capture", () => { it("includes cliVersion and scaffoldVersion on every anonymous capture", async () => { const telemetry = await getTelemetry(); - telemetry.capture("cli_check"); + telemetry.capture("cli_run"); expect(mockCapture).toHaveBeenCalledWith( expect.objectContaining({ @@ -287,7 +287,7 @@ describe("capture", () => { await writeTokenFile(cwd, jwt); const telemetry = await getTelemetry(cwd); - telemetry.capture("cli_rule_create"); + telemetry.capture("cli_rule_created"); expect(mockCapture).toHaveBeenCalledWith( expect.objectContaining({ @@ -307,7 +307,7 @@ describe("capture", () => { const cwd = await mkdtemp(join(tmpdir(), "taskless-no-manifest-")); try { const telemetry = await getTelemetry(cwd); - telemetry.capture("cli_check"); + telemetry.capture("cli_run"); expect(mockCapture).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/packages/cli/test/wizard-integration.test.ts b/packages/cli/test/wizard-integration.test.ts index e74d2607..d399a6b6 100644 --- a/packages/cli/test/wizard-integration.test.ts +++ b/packages/cli/test/wizard-integration.test.ts @@ -96,14 +96,7 @@ describe("runWizard end-to-end", () => { }; expect(manifest.install.targets[".claude"]?.skills).toContain("taskless"); - expect(captureSpy).toHaveBeenCalledWith( - "cli_init_completed", - expect.objectContaining({ - locations: [".claude"], - optionalSkills: [], - nonInteractive: false, - }) - ); + expect(captureSpy).toHaveBeenCalledWith("cli_installed"); }); it("re-running with the same location is idempotent", async () => { @@ -123,7 +116,7 @@ describe("runWizard end-to-end", () => { ).toBe(true); }); - it("cancelling at locations step writes nothing and emits cli_init_cancelled", async () => { + it("cancelling at locations step writes nothing and emits no install event", async () => { clackResponses.locations = fakeCancelSymbol; const { runWizard } = await import("../src/wizard"); @@ -137,10 +130,11 @@ describe("runWizard end-to-end", () => { ); expect(await exists(join(cwd, ".taskless", "taskless.json"))).toBe(false); - expect(captureSpy).toHaveBeenCalledWith( - "cli_init_cancelled", - expect.objectContaining({ atStep: "locations" }) - ); + // A cancelled wizard installs nothing, so it emits no cli_installed event; + // the invocation itself is captured by cli_run at the runner level. Assert + // on the event name across all calls so extra properties can't slip past. + const events = captureSpy.mock.calls.map((call) => call[0] as string); + expect(events).not.toContain("cli_installed"); }); it("cancelling the summary confirm writes nothing", async () => {