diff --git a/openspec/changes/local-rule-routing/tasks.md b/openspec/changes/local-rule-routing/tasks.md index ecc20967..44631a8c 100644 --- a/openspec/changes/local-rule-routing/tasks.md +++ b/openspec/changes/local-rule-routing/tasks.md @@ -9,19 +9,19 @@ ## 2. Routing recipes (cli-rule-routing) -- [ ] 2.1 Author `packages/cli/src/help/route.txt`: run `detect`; require the agent to WRITE its rationale first (detect signals, existing-linter coverage, ast-grep expressibility, local-solvability confidence) and name a destination only as a conclusion of that rationale; commit to the believed-correct path on REASONABLE confidence (existing/static/remote), route remote directly when not reasonably confident; when multiple paths fit, ask the user and explain trade-offs (note `remote` consumes a generation + needs login); never use a deliberate fail-first probe to select `remote` -- [ ] 2.1a In `route.txt`, specify the try-verify-escalate FALLBACK: when a believed-local `static` path fails verification, inform the user the local rule couldn't capture the cases and PROMPT-AND-CONFIRM before calling the service — never silently fall through to `remote` -- [ ] 2.2 Author `packages/cli/src/help/existing.txt`: author in the detected linter's dialect; repo-first knowledge sourcing then WebFetch; explicit author-only (user's toolchain runs it; `taskless check` does not run the external linter) -- [ ] 2.3 Author `packages/cli/src/help/static.txt`: local ast-grep authoring with verification against success/failure cases; canonical on-disk shape and paths; working candidate written under `.taskless/.tmp-*` with guaranteed cleanup on BOTH success and failure (mirror the `rule create` pattern); on verification failure, hand back to the `route` prompt-and-confirm fallback rather than escalating directly -- [ ] 2.4 Author `packages/cli/src/help/remote.txt`: collect inputs, require auth, invoke the existing `rule create` backend; service decides static vs runtime; never decide that locally -- [ ] 2.5 Ensure all four recipes follow the embedded help-text format (header, sprintf escaping) and reference `detect`/`route` consistently +- [x] 2.1 Author `packages/cli/src/help/route.txt`: run `detect`; require the agent to WRITE its rationale first (detect signals, existing-linter coverage, ast-grep expressibility, local-solvability confidence) and name a destination only as a conclusion of that rationale; commit to the believed-correct path on REASONABLE confidence (existing/static/remote), route remote directly when not reasonably confident; when multiple paths fit, ask the user and explain trade-offs (note `remote` consumes a generation + needs login); never use a deliberate fail-first probe to select `remote` +- [x] 2.1a In `route.txt`, specify the try-verify-escalate FALLBACK: when a believed-local `static` path fails verification, inform the user the local rule couldn't capture the cases and PROMPT-AND-CONFIRM before calling the service — never silently fall through to `remote` +- [x] 2.2 Author `packages/cli/src/help/existing.txt`: author in the detected linter's dialect; repo-first knowledge sourcing then WebFetch; explicit author-only (user's toolchain runs it; `taskless check` does not run the external linter) +- [x] 2.3 Author `packages/cli/src/help/static.txt`: local ast-grep authoring with verification against success/failure cases; canonical on-disk shape and paths (`.taskless/rules/.yml` + rule-tests, so `rule verify ` can read it); on verification failure into the escalation fallback, guarantee cleanup of the abandoned candidate rule + test files (mirrors the `rule create` cleanup intent); hand back to the `route` prompt-and-confirm fallback rather than escalating directly +- [x] 2.4 Author `packages/cli/src/help/remote.txt`: collect inputs, require auth, invoke the existing `rule create` backend; service decides static vs runtime; never decide that locally +- [x] 2.5 Ensure all four recipes follow the embedded help-text format (header, sprintf escaping) and reference `detect`/`route` consistently ## 3. Help registration + telemetry (cli-help) -- [ ] 3.1 Confirm the four `.txt` recipes are picked up by the `import.meta.glob` embedding and resolve via `taskless help ` -- [ ] 3.2 Ensure `route`, `existing`, `static`, `remote` appear in the `taskless help` (no-arg) topic index -- [ ] 3.3 Verify `help_` intent telemetry fires for each routing topic -- [ ] 3.4 Add tests for topic resolution, index listing, and telemetry capture +- [x] 3.1 Confirm the four `.txt` recipes are picked up by the `import.meta.glob` embedding and resolve via `taskless help ` +- [x] 3.2 Ensure `route`, `existing`, `static`, `remote` appear in the `taskless help` (no-arg) topic index +- [x] 3.3 Verify `help_` intent telemetry fires for each routing topic +- [x] 3.4 Add tests for topic resolution, index listing, and telemetry capture ## 4. Skill routing posture (skill-taskless) diff --git a/packages/cli/src/commands/help.ts b/packages/cli/src/commands/help.ts index f6e12938..edc80db5 100644 --- a/packages/cli/src/commands/help.ts +++ b/packages/cli/src/commands/help.ts @@ -50,6 +50,16 @@ function buildHelpMaps(): { const { helpMap, anonymousMap } = buildHelpMaps(); +// Help-only recipe topics (no backing subcommand) that should still be +// discoverable from the `taskless help` index. The rule-authoring front +// door (`route`) and its destinations live here so an agent can find them. +const RECIPE_TOPICS: ReadonlyArray<[string, string]> = [ + ["route", "Decide where to author a rule (existing/static/remote)"], + ["existing", "Author a rule in a linter the repo already uses"], + ["static", "Author a local ast-grep rule on this machine (no login)"], + ["remote", "Generate a rule via the Taskless service (login)"], +]; + // Topic → Zod input schema. When a recipe contains the %(INPUT_SCHEMA)s // placeholder, the help command substitutes the JSON Schema rendered // from this Zod source. @@ -173,11 +183,21 @@ export function createHelpCommand(subCommands: SubCommandsDef) { entries.push([name, description]); } - const maxLength = Math.max(...entries.map(([name]) => name.length)); + // Pad commands and recipe topics against a shared width so the two + // sections line up. + const maxLength = Math.max( + ...entries.map(([name]) => name.length), + ...RECIPE_TOPICS.map(([name]) => name.length) + ); for (const [name, description] of entries) { console.log(` ${name.padEnd(maxLength + 2)}${description}`); } + console.log("\nAuthoring recipes:"); + for (const [name, description] of RECIPE_TOPICS) { + console.log(` ${name.padEnd(maxLength + 2)}${description}`); + } + console.log( "\nAppend `--anonymous` to any rule/check command to skip the Taskless API" ); diff --git a/packages/cli/src/help/existing.txt b/packages/cli/src/help/existing.txt new file mode 100644 index 00000000..efc00e53 --- /dev/null +++ b/packages/cli/src/help/existing.txt @@ -0,0 +1,57 @@ +# Topic: existing (CLI v%(CLI_VERSION)s / topic v1) + +## Goal +Author a rule in a linter the repository ALREADY uses, expressed in that +tool's own dialect (an ESLint rule, a Ruff rule selection, a RuboCop cop, +a Stylelint rule, etc.). Taskless does not maintain a catalog of linter +rules — you source the knowledge from the repo first and the web second, +then write the rule where that tool expects it. + +## Preconditions +- The repo has a detected linter (confirm via `taskless detect --json`). +- The agent can read/write files and fetch web pages. +- No auth required. + +## Steps + +1. **Confirm the target tool.** Use `taskless detect --json` to identify + which linter is configured. If more than one could host this rule, + ask the user which tool should own it. + +2. **Mine the repo's own rules first.** This is the highest-signal + source for house style. Look at how the repo already writes rules of + this kind: + - existing config (e.g. `.eslintrc*`/`eslint.config.*`, `ruff.toml` + or `[tool.ruff]`, `.rubocop.yml`, `.stylelintrc*`); + - any custom/local rules the repo authored (the `detect` output's + rule styles point at these); + - the conventions, severity choices, and naming they use. + Match that style. + +3. **Fall back to the web only if the repo signal is thin.** If the repo + doesn't show how to express this rule, fetch the linter's CURRENT + documentation (WebFetch/WebSearch). Prefer the latest version's docs + and, when present, an `llms.txt` on the tool's docs site. Confirm the + installed version where it matters so you target the right syntax. + +4. **Author the rule in the tool's dialect.** Write or extend the + linter's config / custom-rule file the way that tool expects. Keep it + consistent with the repo's existing entries from step 2. + +5. **Report, and be explicit about who runs it.** Show the file(s) you + changed. Make clear that the user's OWN toolchain runs this rule — + `taskless check` does NOT execute external linters. Tell the user how + to run their linter to see it fire (e.g. their existing lint script). + +## Important Notes + +- Do not invent linter rules from memory — verify against the repo's + usage and the tool's current docs. +- This path is author-only. Taskless does not aggregate or run external + linters; it writes the rule in the tool's dialect and hands off. + +## See Also + +- `taskless help route` — re-decide the destination if this no longer fits +- `taskless help static` — author a local ast-grep rule instead +- `taskless help remote` — generate via the Taskless service (login) diff --git a/packages/cli/src/help/remote.txt b/packages/cli/src/help/remote.txt new file mode 100644 index 00000000..823cffc2 --- /dev/null +++ b/packages/cli/src/help/remote.txt @@ -0,0 +1,60 @@ +# Topic: remote (CLI v%(CLI_VERSION)s / topic v1) + +## Goal +Generate a rule using the Taskless service. Your job on this path is to +gather the inputs and hand off to `rule create`; the service generates +the rule and writes the standard rule files. This path consumes a +generation and requires login. Reach it when a rule is not reasonably +solvable locally, or when a believed-local attempt failed and the user +confirmed. + +## Preconditions +- `.taskless/` directory exists. +- The user is logged in (this path requires auth). +- The request has been routed here (see `taskless help route`), not + reached by skipping the local-first decision. + +## Steps + +1. **Confirm this is the right path.** You should be here because the + request is not reasonably solvable locally, OR a local `static` + attempt failed and the user confirmed spending a generation. If the + user has not confirmed an escalation from a failed local attempt, get + that confirmation first — the service costs a generation and login. + +2. **Confirm auth.** Run: + ``` + npx @taskless/cli info --json + ``` + Check `loggedIn`. If false, fetch `taskless help auth` and follow the + login recipe before continuing. + +3. **Gather the request.** Collect the rule description plus concrete + success and failure cases. If you arrived here from `static`, reuse + the cases you already gathered. The service uses these to generate the + rule. + +4. **Delegate to the generation backend.** Fetch `taskless help rule + create` and follow it. That recipe builds the request payload and + invokes `rule create`, which submits to the service and writes the + result. Do not re-implement the submission here. + +5. **Report results.** The service writes the generated rule to the same + on-disk paths and shape as a locally authored rule, so `check`, + `improve`, and `verify` treat it identically. Show the user the file + paths and suggest `taskless help check` to validate. + +## Important Notes + +- Do NOT pre-build the rule yourself on this path — submit the request + and let the service generate it. +- The service owns rule-type selection. Today it generates ast-grep + rules written under `.taskless/rules/`, the same shape the local + `static` path produces. + +## See Also + +- `taskless help route` — the local-first routing decision +- `taskless help rule create` — the generation backend this path uses +- `taskless help auth` — log in before generating +- `taskless help check` — validate the generated rule diff --git a/packages/cli/src/help/route.txt b/packages/cli/src/help/route.txt new file mode 100644 index 00000000..b1ece4ed --- /dev/null +++ b/packages/cli/src/help/route.txt @@ -0,0 +1,95 @@ +# Topic: route (CLI v%(CLI_VERSION)s / topic v1) + +## Goal +Decide where a rule-authoring request should be built: in a linter the +repo ALREADY uses (`existing`), as a local ast-grep rule on this machine +(`static`), or by the Taskless service (`remote`). This is the front +door for "author/write/create a rule" requests. You stay local whenever +you can reasonably build the rule on-device; you only send the user to +the login-gated service when you are not reasonably confident the rule +is locally solvable — or when a believed-local attempt has genuinely +failed and the user confirms. + +## Preconditions +- A working repository the agent can read. +- No auth required to route. (The `remote` destination requires login; + the `existing` and `static` destinations do not.) + +## Steps + +1. **Scan the repo.** Run: + ``` + npx @taskless/cli detect --json + ``` + This returns the configured linters, languages, and the repo's own + rule styles. It is deterministic and offline — use it as ground truth + instead of guessing the repo's tooling. The scan is monorepo-aware, so + evidence may carry a sub-package path. The output shape: + ```json + { + "success": true, + "linters": [{ "name": "eslint", "evidence": ["packages/api/.eslintrc.json"] }], + "languages": ["JavaScript", "TypeScript"], + "ruleStyles": [ + { "source": ".taskless/rules", "description": "Existing Taskless ast-grep rules." } + ] + } + ``` + +2. **Write your reasoning BEFORE naming a destination.** Do not pick a + route first and justify it after. Write a short rationale covering: + - what the `detect` signals show (linters, languages, repo rule styles); + - whether an existing linter plausibly already covers this request; + - whether the pattern is expressible as a simple static ast-grep rule; + - your resulting confidence that the request is locally solvable. + The destination you choose in step 3 MUST follow from this rationale. + +3. **Choose the destination as a conclusion of the rationale:** + + - **Both an existing linter AND a local static rule fit** → do NOT + silently pick one. Present both to the user with trade-offs and let + them choose. Note that `remote` consumes a Taskless generation and + requires login, so it is for what cannot be solved locally — not a + default. + - **An existing linter clearly fits** (the repo uses it and it can + express this) → fetch `taskless help existing`. + - **You are reasonably confident it is a simple static ast-grep + pattern** → fetch `taskless help static`. Reasonable confidence is + enough here; you do not need certainty, because step 4 backstops a + wrong-but-reasonable bet. + - **You reasonably believe it CANNOT be solved locally** (a positive + judgment — e.g. it needs cross-file or semantic checks ast-grep can't + express) → fetch `taskless help remote`. Do not manufacture a + deliberate local failure to reach this; route here directly. + + These are three distinct states. If you genuinely cannot tell whether + the request is local or remote — mere uncertainty, not a belief that it + needs the service — ASK the user a clarifying question. Uncertainty is a + reason to ask, never a reason to spend a generation on `remote`. + +4. **Failure fallback (try-verify-escalate).** If you committed to a + local `static` rule on reasonable confidence and it then FAILS + verification against the user's success/failure cases: + - Tell the user the local rule could not capture the cases. + - State that generating via the Taskless service uses a generation + and requires login. + - Call the service only after the user confirms. On yes, fetch + `taskless help remote`. + Never silently fall through from a failed local attempt to a service + call — the confirmation step is required. + +## Important Notes + +- Reason first, route second. Do not name a destination before the + rationale is written. +- Stay local when you reasonably can. `remote` is the path for what + cannot be solved on-device, and it costs a generation plus login. +- A developer who watches a local attempt fail reads it as a Taskless + failure. Only attempt local when you reasonably believe it will work; + otherwise route `remote` upfront. + +## See Also + +- `taskless help existing` — author in a linter the repo already uses +- `taskless help static` — author a local ast-grep rule (no login) +- `taskless help remote` — generate via the Taskless service (login) diff --git a/packages/cli/src/help/static.txt b/packages/cli/src/help/static.txt new file mode 100644 index 00000000..2332e570 --- /dev/null +++ b/packages/cli/src/help/static.txt @@ -0,0 +1,74 @@ +# Topic: static (CLI v%(CLI_VERSION)s / topic v1) + +## Goal +Author a Taskless ast-grep rule **locally**, on this machine, without +contacting the Taskless service. You derive the rule yourself, write it +in the canonical on-disk shape, and validate it with `rule verify` in a +feedback loop. The files you produce match exactly what the service +writes, so `check`, `improve`, and `verify` treat them identically. + +## Preconditions +- `.taskless/` directory exists. +- The agent can read/write files and run shell commands. +- No auth required. + +## Steps + +1. **Learn the ast-grep rule format.** Consult the ast-grep rule + reference at https://ast-grep.github.io/guide/rule-config.html for + valid fields and operators (`pattern`, `kind`, `regex`, + `any`/`all`/`has`/`inside`/`not`) and meta-variable syntax. + +2. **Gather and confirm the pattern.** Make sure you have concrete + success cases (code that should pass) and failure cases (code that + should be flagged), the target language, and any exceptions. Search + the codebase for real instances and confirm exclusions with the user. + +3. **Author the rule in the canonical shape.** Write the rule to + `.taskless/rules/.yml` with at minimum `id` (kebab-case), + `language`, `severity` (`error`/`warning`/`info`/`hint`), `message`, + and the `rule` object. Write tests to + `.taskless/rule-tests/-YYYYMMDD-test.yml` with the matching `id` + field plus `valid` and `invalid` arrays (at least two of each). The + `id` must match the rule's `id` so ast-grep test filtering pairs them. + These paths and shape are the same ones the service writes — do not + invent a different layout. + +4. **Run the verify feedback loop.** Run: + ``` + npx @taskless/cli rule verify --json + ``` + - `success: true` → the rule passes. Go to step 5. + - `success: false` → read the per-layer errors (`schema`, + `requirements`, `tests`), fix the rule or tests, and re-run. Repeat + up to 3 times. + +5. **On success, report.** Show the rule and test file paths and a + one-line summary of what the rule detects. Suggest `taskless help + check` to validate against the broader codebase. + +6. **On failure, escalate via the route fallback — with confirmation.** + If after the feedback loop the rule still cannot capture the user's + cases, this is the try-verify-escalate fallback: + - Delete the candidate `.taskless/rules/.yml` and its test file so + the repo is not left with a broken rule (guaranteed cleanup of the + abandoned candidate). + - Tell the user the local rule could not capture the cases, and that + generating via the Taskless service uses a generation and requires + login. + - Only after the user confirms, fetch `taskless help remote` and + follow it. Do not call the service silently. + +## Important Notes + +- Do NOT make any HTTP requests to taskless.io on this path. +- Do NOT write to `.taskless/rule-metadata/` — local rules have no + metadata sidecar; they iterate via file edits. +- The verify loop is the quality gate. A clean failure is a legitimate + reason to escalate, but only with the user's confirmation (step 6). + +## See Also + +- `taskless help route` — re-decide the destination +- `taskless help remote` — generate via the Taskless service (login) +- `taskless help check` — validate the new rule against the codebase diff --git a/packages/cli/test/help-extensions.test.ts b/packages/cli/test/help-extensions.test.ts index 998dd553..10688669 100644 --- a/packages/cli/test/help-extensions.test.ts +++ b/packages/cli/test/help-extensions.test.ts @@ -58,6 +58,37 @@ describe("taskless help (no args)", () => { const result = await runCli(["help", "-d", cwd]); expect(result.stdout).toContain("--anonymous"); }); + + it("lists the routing recipe topics under Authoring recipes", async () => { + const result = await runCli(["help", "-d", cwd]); + expect(result.stdout).toContain("Authoring recipes:"); + for (const topic of ["route", "existing", "static", "remote"]) { + expect(result.stdout).toContain(topic); + } + }); +}); + +describe("taskless help ", () => { + let cwd: string; + + beforeEach(async () => { + cwd = await mkdtemp(join(tmpdir(), "taskless-help-routing-")); + }); + + afterEach(async () => { + await rm(cwd, { recursive: true, force: true }); + }); + + it.each(["route", "existing", "static", "remote"])( + "resolves the %s recipe without an unknown-topic error", + async (topic) => { + const result = await runCli(["help", topic, "-d", cwd]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(`# Topic: ${topic}`); + expect(result.stdout).toContain("## Goal"); + expect(result.stderr).not.toContain("Unknown command"); + } + ); }); describe("taskless help ", () => { diff --git a/packages/cli/test/help-routing-telemetry.test.ts b/packages/cli/test/help-routing-telemetry.test.ts new file mode 100644 index 00000000..ece49ffe --- /dev/null +++ b/packages/cli/test/help-routing-telemetry.test.ts @@ -0,0 +1,55 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +// Spy on the telemetry capture by mocking the telemetry module the help +// command imports. The factory is invoked lazily at import time, so the +// closure over `capture` resolves after initialization (same pattern as +// telemetry.test.ts mocking posthog-node). +const capture = vi.fn(); +vi.mock("../src/telemetry", () => ({ + getTelemetry: vi.fn(() => + Promise.resolve({ + capture, + shutdown: () => Promise.resolve(), + }) + ), + shutdownTelemetry: () => Promise.resolve(), +})); + +const { createHelpCommand } = await import("../src/commands/help"); + +interface RunnableCommand { + run: (context: { + args: { dir: string; anonymous: boolean }; + rawArgs: string[]; + }) => Promise; +} + +describe("help routing topics emit cli_help intent telemetry", () => { + let logSpy: ReturnType; + + beforeEach(() => { + capture.mockClear(); + // Suppress the recipe text the command prints to stdout. + logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + }); + + it.each(["route", "existing", "static", "remote"])( + "captures cli_help for %s", + async (topic) => { + const command = createHelpCommand({}) as unknown as RunnableCommand; + await command.run({ + args: { dir: process.cwd(), anonymous: false }, + rawArgs: ["help", topic], + }); + + expect(capture).toHaveBeenCalledWith( + "cli_help", + expect.objectContaining({ topic }) + ); + } + ); +});