From 7a295874f23f26210278dd3633c8dac0687c8f53 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 11 Jun 2026 16:34:13 -0700 Subject: [PATCH 1/9] feat(skill): Engage routing when a linter is named Reverse the named-tool suppression: naming a linter (eslint, ruff, biome, ast-grep) now ENGAGES the skill's routing flow via `taskless help route` instead of quieting it. Replace the "Quiet suggestion" section with a route-through-route body that forbids fetching `rule create` directly and adds no linter knowledge, and relax the install gate so the `existing` path (which only reads the repo) is not blocked when `.taskless/` is absent. Description measures 835/1024 chars. Skill version is build-locked to the package version, so a minor changeset drives the bump rather than a hand-edited mismatch. Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/local-rule-routing.md | 10 +++++ openspec/changes/local-rule-routing/tasks.md | 10 ++--- skills/taskless/SKILL.md | 46 ++++++++++---------- 3 files changed, 37 insertions(+), 29 deletions(-) create mode 100644 .changeset/local-rule-routing.md diff --git a/.changeset/local-rule-routing.md b/.changeset/local-rule-routing.md new file mode 100644 index 00000000..e123ec41 --- /dev/null +++ b/.changeset/local-rule-routing.md @@ -0,0 +1,10 @@ +--- +"@taskless/skills": minor +--- + +Add a local-first rule-routing layer. A new deterministic `taskless detect` +command plus `route`/`existing`/`static`/`remote` recipes let the agent author +rules in an existing linter or as a local ast-grep rule on-device, only +escalating to the login-gated service (with confirmation) when a rule cannot be +built locally. The skill now engages this routing flow when a user names a +linter instead of suppressing itself. diff --git a/openspec/changes/local-rule-routing/tasks.md b/openspec/changes/local-rule-routing/tasks.md index 4c464a02..f2662c3a 100644 --- a/openspec/changes/local-rule-routing/tasks.md +++ b/openspec/changes/local-rule-routing/tasks.md @@ -25,11 +25,11 @@ ## 4. Skill routing posture (skill-taskless) -- [ ] 4.1 Update `skills/taskless/SKILL.md` `description`: replace the named-tool suppression clause so naming a linter engages routing via `taskless help route`; tighten (reword shorter) rather than append trigger text -- [ ] 4.1a Measure the resulting `description` length and assert it is ≤ 1024 chars (Agent Skills ceiling); treat overflow as a blocking failure and trim trigger wording until it fits -- [ ] 4.2 Update the skill body to route authoring requests through `taskless help route` (not `rule create` directly); remove the "quiet suggestion" suppression path; keep the skill a thin router with no linter knowledge -- [ ] 4.3 Bump the skill `metadata.version` per the file conventions -- [ ] 4.4 Verify the skill change against the updated `skill-taskless` scenarios (routing on named tool, no suppression wording, local-first before login) +- [x] 4.1 Update `skills/taskless/SKILL.md` `description`: replace the named-tool suppression clause so naming a linter engages routing via `taskless help route`; tighten (reword shorter) rather than append trigger text +- [x] 4.1a Measure the resulting `description` length and assert it is ≤ 1024 chars (Agent Skills ceiling); treat overflow as a blocking failure and trim trigger wording until it fits — measured 835 chars +- [x] 4.2 Update the skill body to route authoring requests through `taskless help route` (not `rule create` directly); remove the "quiet suggestion" suppression path; keep the skill a thin router with no linter knowledge +- [x] 4.3 Bump the skill `metadata.version` per the file conventions — version is build-locked to package version (assertSkillVersions); added a `minor` changeset so the release path bumps both in lockstep rather than hand-editing a mismatch +- [x] 4.4 Verify the skill change against the updated `skill-taskless` scenarios (routing on named tool, no suppression wording, local-first before login) ## 5. Validation + quality gate diff --git a/skills/taskless/SKILL.md b/skills/taskless/SKILL.md index b5e56174..08b17c16 100644 --- a/skills/taskless/SKILL.md +++ b/skills/taskless/SKILL.md @@ -14,12 +14,10 @@ description: | - "add taskless to CI", "wire taskless into github actions" - "onboard with taskless", "set up taskless for this project" - Also trigger when the user asks to add/write/create a rule and has NOT - named a specific lint/format/static-analysis tool. Examples that suppress - this trigger (illustrative — any named tool of this kind suppresses): - eslint, ruff, biome, ast-grep. On this trigger, surface a quiet - single-line offer rather than launching a recipe; see the body's quiet - suggestion section. + Also trigger on any request to add/write/create a lint or code rule, + including ones that name a specific tool (eslint, ruff, biome, stylelint, + ast-grep). Naming a tool ENGAGES this skill's routing flow via + `taskless help route`; it does NOT suppress the skill. metadata: author: taskless version: 0.8.1 @@ -34,20 +32,33 @@ canonical recipes live behind `npx @taskless/cli help `. Always fetch the recipe first; do not improvise from prior knowledge — recipes change with each CLI version. -## First step: confirm Taskless is installed here +## Authoring a rule: always start at route -If the working directory does not contain a `.taskless/` directory, ask the -user to confirm they meant Taskless (vs. ESLint or another tool). If they -confirm, offer to run `npx @taskless/cli` to install. Otherwise, stop. +For any request to add/write/create a rule — whether or not the user names a +tool (eslint, ruff, biome, ast-grep) — fetch `npx @taskless/cli help route` +and follow it. Do NOT fetch `rule create` directly, and do NOT author from +your own linter knowledge. `route` runs `detect`, reasons about the request, +and decides whether the rule is built in an existing linter (`existing`), as a +local ast-grep rule (`static`), or via the Taskless service (`remote`) — and it +keeps the work local before any login. This skill is a thin router: all +authoring judgment lives in the fetched recipes. + +## Confirm Taskless is installed when a path needs it + +`route` and the `existing` path only read the repo, so they need no install. If +routing lands on a local Taskless rule (`static`) or the service (`remote`) and +the working directory has no `.taskless/` directory, offer to run +`npx @taskless/cli` to install. If the user only wanted help with their own +linter, the `existing` path needs nothing installed. ## Topics | User wants | Topic | | -------------------------- | ------------------------------------- | +| Author/create a rule | `npx @taskless/cli help route` | | First-time install | tell user to run `npx @taskless/cli` | | Update existing install | `npx @taskless/cli update` | | Discover candidate rules | `npx @taskless/cli help onboard` | -| Create a new rule | `npx @taskless/cli help rule create` | | Improve an existing rule | `npx @taskless/cli help rule improve` | | Delete a rule | `npx @taskless/cli help rule delete` | | Check code against rules | `npx @taskless/cli help check` | @@ -58,19 +69,6 @@ If the user's intent is ambiguous between two topics, run `npx @taskless/cli help` (no args) to see the disambiguation table, or ask the user. -## Quiet suggestion - -When you trigger because the user asked to add/write/create a rule and did -NOT name a specific tool (per the description's named-tool clause), DO NOT -launch into a full recipe. Instead surface a one-line offer such as: "I can -capture this as a Taskless rule if you want — say so, or I'll proceed with -." If the user accepts, fetch -`npx @taskless/cli help rule create` and follow it. If the user declines or -ignores the offer, proceed with what you would have done without this skill, -and DO NOT re-offer Taskless in the same conversation. The decline is sticky -within the conversation only — do NOT write any persistent decline state to -disk or to `.taskless/taskless.json`. - ## --anonymous Any rule/check command accepts `--anonymous` to skip the Taskless API and From 1ecf2a5b01127d026613b409968233c8fa166d9c Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 11 Jun 2026 16:40:22 -0700 Subject: [PATCH 2/9] test(cli): Add route honesty eval fixtures and finalize change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a labeled request→route calibration dataset for the route recipe covering both failure directions: over-claim (Taskless grabbing a packaged/formatter job that should stay `existing`) and over-escalate (a locally-solvable request wrongly sent to the login-gated service that should stay `static`), plus genuine `remote` cases. The route decision is agent-made by following help/route.txt, so the coverage test guards the dataset's balance across routes and traps rather than running a code classifier. Closes out the change: openspec validate passes, full CLI suite green, detect + routing recipes smoke-tested end-to-end. Co-Authored-By: Claude Opus 4.8 (1M context) --- openspec/changes/local-rule-routing/tasks.md | 8 +- packages/cli/test/fixtures/route-eval.json | 79 ++++++++++++++++++++ packages/cli/test/route-eval.test.ts | 79 ++++++++++++++++++++ 3 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 packages/cli/test/fixtures/route-eval.json create mode 100644 packages/cli/test/route-eval.test.ts diff --git a/openspec/changes/local-rule-routing/tasks.md b/openspec/changes/local-rule-routing/tasks.md index f2662c3a..2ea9f27e 100644 --- a/openspec/changes/local-rule-routing/tasks.md +++ b/openspec/changes/local-rule-routing/tasks.md @@ -33,7 +33,7 @@ ## 5. Validation + quality gate -- [ ] 5.1 Run `pnpm openspec validate local-rule-routing` and resolve any issues -- [ ] 5.2 Add/curate the honesty eval fixtures (labeled request → expected route) and assert the `route` heuristic against both failure directions: under-confident (escalating a locally-solvable request to login) and over-confident (claiming local for a request that needs the service). Use the fixtures to calibrate the "confident enough for local" threshold -- [ ] 5.3 Run `pnpm typecheck` and `pnpm lint`; fix all failures -- [ ] 5.4 Manual smoke: `taskless detect --json`, then `taskless help route`/`existing`/`static`/`remote` resolve and read coherently end-to-end +- [x] 5.1 Run `pnpm openspec validate local-rule-routing` and resolve any issues +- [x] 5.2 Add/curate the honesty eval fixtures (labeled request → expected route) and assert the `route` heuristic against both failure directions: under-confident (escalating a locally-solvable request to login) and over-confident (claiming local for a request that needs the service). Use the fixtures to calibrate the "confident enough for local" threshold — dataset at `test/fixtures/route-eval.json` with a coverage test; route decision is agent-made (recipe-followed), so the test guards dataset balance across routes + both traps rather than running a code classifier +- [x] 5.3 Run `pnpm typecheck` and `pnpm lint`; fix all failures +- [x] 5.4 Manual smoke: `taskless detect --json`, then `taskless help route`/`existing`/`static`/`remote` resolve and read coherently end-to-end diff --git a/packages/cli/test/fixtures/route-eval.json b/packages/cli/test/fixtures/route-eval.json new file mode 100644 index 00000000..38813b2c --- /dev/null +++ b/packages/cli/test/fixtures/route-eval.json @@ -0,0 +1,79 @@ +{ + "description": "Honesty eval fixtures for the `route` recipe. Each case is a labeled request with the destination route SHOULD reach. `trap` names the failure direction the case guards against (or null for a clean case). These are a calibration dataset for evaluating the route recipe's decisions; the route decision itself is made by an agent following help/route.txt, so these are not asserted against a code classifier — the coverage test only checks the dataset stays balanced across routes and both failure directions.", + "routes": ["existing", "static", "remote"], + "traps": ["over-claim", "over-escalate", "under-engage"], + "cases": [ + { + "request": "warn on console.log", + "expected": "existing", + "trap": "over-claim", + "reason": "ESLint's packaged no-console already covers this; claiming it for a Taskless rule erodes trust where Taskless would never win." + }, + { + "request": "sort our imports", + "expected": "existing", + "trap": "over-claim", + "reason": "Formatter / eslint-plugin-import territory, not a semantic rule." + }, + { + "request": "disallow `any` in TypeScript", + "expected": "existing", + "trap": "over-claim", + "reason": "@typescript-eslint/no-explicit-any is a packaged rule; enable it rather than authoring." + }, + { + "request": "write a Ruff rule for X (repo has ruff.toml)", + "expected": "existing", + "trap": "under-engage", + "reason": "Naming a tool must ENGAGE routing into that tool, not suppress the skill." + }, + { + "request": "add an ESLint rule banning direct process.env reads outside config/", + "expected": "existing", + "trap": "under-engage", + "reason": "User named ESLint and wants a custom rule; author it in ESLint's dialect, mined from the repo's own rules." + }, + { + "request": "controllers must not import the db module directly", + "expected": "static", + "trap": null, + "reason": "Bespoke syntactic import ban, no packaged rule, cleanly expressible as a local ast-grep rule." + }, + { + "request": "never call fetch directly — use our http wrapper", + "expected": "static", + "trap": null, + "reason": "Call/import ban; classic local ast-grep pattern." + }, + { + "request": "enforce our feature-flag naming convention ff_*", + "expected": "static", + "trap": "over-escalate", + "reason": "Repo-specific identifier pattern is a simple local ast-grep rule; escalating to the service here wastes a generation and a login." + }, + { + "request": "flag string literals that look like API keys in source files", + "expected": "static", + "trap": "over-escalate", + "reason": "A regex-shaped syntactic match is locally solvable; do not route to remote just because it sounds security-ish." + }, + { + "request": "every API handler must parse its request body through a Zod schema before using it", + "expected": "remote", + "trap": null, + "reason": "Cross-statement intent (parse-before-use) that a single ast-grep pattern strains to express; let the service classify static vs runtime." + }, + { + "request": "flag any PR that adds a new env var without documenting it in the README", + "expected": "remote", + "trap": null, + "reason": "Cross-file, config-vs-docs consistency — needs a runtime check.ts the service generates." + }, + { + "request": "ensure every exported function in a route file has a matching test in __tests__", + "expected": "remote", + "trap": null, + "reason": "Cross-file correlation no static syntactic matcher expresses; service/runtime territory." + } + ] +} diff --git a/packages/cli/test/route-eval.test.ts b/packages/cli/test/route-eval.test.ts new file mode 100644 index 00000000..50466b87 --- /dev/null +++ b/packages/cli/test/route-eval.test.ts @@ -0,0 +1,79 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +// The route decision is made by an agent following help/route.txt, so this +// dataset is not run against a code classifier. The test guards the dataset +// itself: it must stay structurally valid and balanced across every route and +// both failure directions, so it remains a usable calibration set. + +interface EvalCase { + request: string; + expected: string; + trap: string | null; + reason: string; +} + +interface EvalFixtures { + routes: string[]; + traps: string[]; + cases: EvalCase[]; +} + +const fixtures = JSON.parse( + readFileSync(resolve(import.meta.dirname, "fixtures/route-eval.json"), "utf8") +) as EvalFixtures; + +describe("route honesty eval fixtures", () => { + it("declares the three routes and both failure-direction traps", () => { + expect(fixtures.routes).toEqual(["existing", "static", "remote"]); + // Over-claim = Taskless grabbing a packaged/formatter job; over-escalate = + // sending a locally-solvable request to the login-gated service. + expect(fixtures.traps).toEqual( + expect.arrayContaining(["over-claim", "over-escalate", "under-engage"]) + ); + }); + + it("every case is well-formed and uses a declared route/trap", () => { + expect(fixtures.cases.length).toBeGreaterThanOrEqual(10); + for (const c of fixtures.cases) { + expect(c.request.length).toBeGreaterThan(0); + expect(c.reason.length).toBeGreaterThan(0); + expect(fixtures.routes).toContain(c.expected); + if (c.trap !== null) { + expect(fixtures.traps).toContain(c.trap); + } + } + }); + + it("covers every route at least twice", () => { + for (const route of fixtures.routes) { + const count = fixtures.cases.filter((c) => c.expected === route).length; + expect(count, `route ${route} needs >= 2 cases`).toBeGreaterThanOrEqual( + 2 + ); + } + }); + + it("guards both failure directions: over-claim and over-escalate", () => { + const overClaim = fixtures.cases.filter((c) => c.trap === "over-claim"); + const overEscalate = fixtures.cases.filter( + (c) => c.trap === "over-escalate" + ); + // Over-claim cases must expect a non-`static` local route (Taskless should + // not have grabbed them); over-escalate cases must expect a local route + // (they should not have gone to remote). + expect(overClaim.length).toBeGreaterThan(0); + expect(overEscalate.length).toBeGreaterThan(0); + for (const c of overClaim) expect(c.expected).toBe("existing"); + for (const c of overEscalate) expect(c.expected).toBe("static"); + }); + + it("includes genuine remote cases that are not locally solvable", () => { + const remote = fixtures.cases.filter((c) => c.expected === "remote"); + expect(remote.length).toBeGreaterThanOrEqual(2); + // Genuine remote cases are clean (no trap) — they legitimately need the + // service, not a misroute being corrected. + expect(remote.every((c) => c.trap === null)).toBe(true); + }); +}); From 546f9887f98dbe57142746a77b0a1ce243944712 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 11 Jun 2026 18:31:12 -0700 Subject: [PATCH 3/9] fix(skill): Target @taskless/cli in the changeset; align tool lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #27 review: - The changeset targeted the private, unpublished root `@taskless/skills` package, which produces no release and is overwritten by sync-skill-versions. Target `@taskless/cli` — the published package and the version source of truth that skill metadata.version is locked to. - Add `stylelint` to the skill body's example tool list so it matches the description's list. - Update the tasks.md 4.3 note to describe the real release mechanism (@taskless/cli version drives the skill version via sync-skill-versions). Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/local-rule-routing.md | 2 +- openspec/changes/local-rule-routing/tasks.md | 2 +- skills/taskless/SKILL.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/local-rule-routing.md b/.changeset/local-rule-routing.md index e123ec41..ac842d84 100644 --- a/.changeset/local-rule-routing.md +++ b/.changeset/local-rule-routing.md @@ -1,5 +1,5 @@ --- -"@taskless/skills": minor +"@taskless/cli": minor --- Add a local-first rule-routing layer. A new deterministic `taskless detect` diff --git a/openspec/changes/local-rule-routing/tasks.md b/openspec/changes/local-rule-routing/tasks.md index f2662c3a..143224d2 100644 --- a/openspec/changes/local-rule-routing/tasks.md +++ b/openspec/changes/local-rule-routing/tasks.md @@ -28,7 +28,7 @@ - [x] 4.1 Update `skills/taskless/SKILL.md` `description`: replace the named-tool suppression clause so naming a linter engages routing via `taskless help route`; tighten (reword shorter) rather than append trigger text - [x] 4.1a Measure the resulting `description` length and assert it is ≤ 1024 chars (Agent Skills ceiling); treat overflow as a blocking failure and trim trigger wording until it fits — measured 835 chars - [x] 4.2 Update the skill body to route authoring requests through `taskless help route` (not `rule create` directly); remove the "quiet suggestion" suppression path; keep the skill a thin router with no linter knowledge -- [x] 4.3 Bump the skill `metadata.version` per the file conventions — version is build-locked to package version (assertSkillVersions); added a `minor` changeset so the release path bumps both in lockstep rather than hand-editing a mismatch +- [x] 4.3 Bump the skill `metadata.version` per the file conventions — skill version is build-locked to the `@taskless/cli` package version (assertSkillVersions), which `scripts/sync-skill-versions.ts` treats as the source of truth; added a `minor` changeset targeting `@taskless/cli` (the root `@taskless/skills` package is private/unpublished) so the release path bumps the CLI version and syncs the skill version in lockstep, rather than hand-editing a mismatch - [x] 4.4 Verify the skill change against the updated `skill-taskless` scenarios (routing on named tool, no suppression wording, local-first before login) ## 5. Validation + quality gate diff --git a/skills/taskless/SKILL.md b/skills/taskless/SKILL.md index 08b17c16..2b26955c 100644 --- a/skills/taskless/SKILL.md +++ b/skills/taskless/SKILL.md @@ -35,7 +35,7 @@ each CLI version. ## Authoring a rule: always start at route For any request to add/write/create a rule — whether or not the user names a -tool (eslint, ruff, biome, ast-grep) — fetch `npx @taskless/cli help route` +tool (eslint, ruff, biome, stylelint, ast-grep) — fetch `npx @taskless/cli help route` and follow it. Do NOT fetch `rule create` directly, and do NOT author from your own linter knowledge. `route` runs `detect`, reasons about the request, and decides whether the rule is built in an existing linter (`existing`), as a From 7d9f4ce515840e3c0d05920ed5ccf21c8dafeefc Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 11 Jun 2026 18:44:04 -0700 Subject: [PATCH 4/9] test(cli): Cover every eval trap and fix the fixtures path PR #28 review: - The trap-coverage test asserted the traps were declared but never verified each trap has at least one case, so the dataset could silently stop covering a failure direction (e.g. under-engage) while the test stayed green. Now every declared trap must have >= 1 case, and under-engage cases are asserted to route to `existing`. - Fix the tasks.md note to point at the real fixture/test paths under packages/cli/test/. Co-Authored-By: Claude Opus 4.8 (1M context) --- openspec/changes/local-rule-routing/tasks.md | 2 +- packages/cli/test/route-eval.test.ts | 35 +++++++++++++------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/openspec/changes/local-rule-routing/tasks.md b/openspec/changes/local-rule-routing/tasks.md index 18417a5b..bf03b6e3 100644 --- a/openspec/changes/local-rule-routing/tasks.md +++ b/openspec/changes/local-rule-routing/tasks.md @@ -34,6 +34,6 @@ ## 5. Validation + quality gate - [x] 5.1 Run `pnpm openspec validate local-rule-routing` and resolve any issues -- [x] 5.2 Add/curate the honesty eval fixtures (labeled request → expected route) and assert the `route` heuristic against both failure directions: under-confident (escalating a locally-solvable request to login) and over-confident (claiming local for a request that needs the service). Use the fixtures to calibrate the "confident enough for local" threshold — dataset at `test/fixtures/route-eval.json` with a coverage test; route decision is agent-made (recipe-followed), so the test guards dataset balance across routes + both traps rather than running a code classifier +- [x] 5.2 Add/curate the honesty eval fixtures (labeled request → expected route) and assert the `route` heuristic against both failure directions: under-confident (escalating a locally-solvable request to login) and over-confident (claiming local for a request that needs the service). Use the fixtures to calibrate the "confident enough for local" threshold — dataset at `packages/cli/test/fixtures/route-eval.json` with a coverage test (`packages/cli/test/route-eval.test.ts`); route decision is agent-made (recipe-followed), so the test guards dataset balance across routes + every declared trap rather than running a code classifier - [x] 5.3 Run `pnpm typecheck` and `pnpm lint`; fix all failures - [x] 5.4 Manual smoke: `taskless detect --json`, then `taskless help route`/`existing`/`static`/`remote` resolve and read coherently end-to-end diff --git a/packages/cli/test/route-eval.test.ts b/packages/cli/test/route-eval.test.ts index 50466b87..99f5256d 100644 --- a/packages/cli/test/route-eval.test.ts +++ b/packages/cli/test/route-eval.test.ts @@ -24,6 +24,9 @@ const fixtures = JSON.parse( readFileSync(resolve(import.meta.dirname, "fixtures/route-eval.json"), "utf8") ) as EvalFixtures; +const casesForTrap = (trap: string): EvalCase[] => + fixtures.cases.filter((c) => c.trap === trap); + describe("route honesty eval fixtures", () => { it("declares the three routes and both failure-direction traps", () => { expect(fixtures.routes).toEqual(["existing", "static", "remote"]); @@ -55,18 +58,26 @@ describe("route honesty eval fixtures", () => { } }); - it("guards both failure directions: over-claim and over-escalate", () => { - const overClaim = fixtures.cases.filter((c) => c.trap === "over-claim"); - const overEscalate = fixtures.cases.filter( - (c) => c.trap === "over-escalate" - ); - // Over-claim cases must expect a non-`static` local route (Taskless should - // not have grabbed them); over-escalate cases must expect a local route - // (they should not have gone to remote). - expect(overClaim.length).toBeGreaterThan(0); - expect(overEscalate.length).toBeGreaterThan(0); - for (const c of overClaim) expect(c.expected).toBe("existing"); - for (const c of overEscalate) expect(c.expected).toBe("static"); + it("guards every declared trap with at least one correctly-routed case", () => { + // Each declared trap must have at least one case, so the dataset can't + // silently stop covering a failure direction while the test still passes. + for (const trap of fixtures.traps) { + expect( + casesForTrap(trap).length, + `trap ${trap} needs >= 1 case` + ).toBeGreaterThan(0); + } + + // Over-claim: Taskless should not have grabbed it → expect `existing`. + for (const c of casesForTrap("over-claim")) + expect(c.expected).toBe("existing"); + // Over-escalate: locally solvable → expect `static`, not remote. + for (const c of casesForTrap("over-escalate")) + expect(c.expected).toBe("static"); + // Under-engage: naming a tool must engage routing into that linter, not + // suppress → expect `existing`. + for (const c of casesForTrap("under-engage")) + expect(c.expected).toBe("existing"); }); it("includes genuine remote cases that are not locally solvable", () => { From 1b9ef6558e205b4a10655c67581c46bc91d3f7b9 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 11 Jun 2026 18:51:52 -0700 Subject: [PATCH 5/9] chore(openspec): Archive local-rule-routing and sync specs All 25 tasks are complete, so finalize the change on the tip of the stack: apply the delta specs into the main specs (new cli-detect and cli-rule-routing capabilities; cli-help and skill-taskless updates) and move the change to openspec/changes/archive/2026-06-12-local-rule-routing/. With no unarchived change remaining, the PR OpenSpec Archive Check passes for the final merged state. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../.openspec.yaml | 0 .../2026-06-12-local-rule-routing}/design.md | 0 .../proposal.md | 0 .../specs/cli-detect/spec.md | 0 .../specs/cli-help/spec.md | 0 .../specs/cli-rule-routing/spec.md | 0 .../specs/skill-taskless/spec.md | 0 .../2026-06-12-local-rule-routing}/tasks.md | 0 openspec/specs/cli-detect/spec.md | 78 +++++++ openspec/specs/cli-help/spec.md | 31 +++ openspec/specs/cli-rule-routing/spec.md | 209 ++++++++++++++++++ openspec/specs/skill-taskless/spec.md | 40 +++- 12 files changed, 351 insertions(+), 7 deletions(-) rename openspec/changes/{local-rule-routing => archive/2026-06-12-local-rule-routing}/.openspec.yaml (100%) rename openspec/changes/{local-rule-routing => archive/2026-06-12-local-rule-routing}/design.md (100%) rename openspec/changes/{local-rule-routing => archive/2026-06-12-local-rule-routing}/proposal.md (100%) rename openspec/changes/{local-rule-routing => archive/2026-06-12-local-rule-routing}/specs/cli-detect/spec.md (100%) rename openspec/changes/{local-rule-routing => archive/2026-06-12-local-rule-routing}/specs/cli-help/spec.md (100%) rename openspec/changes/{local-rule-routing => archive/2026-06-12-local-rule-routing}/specs/cli-rule-routing/spec.md (100%) rename openspec/changes/{local-rule-routing => archive/2026-06-12-local-rule-routing}/specs/skill-taskless/spec.md (100%) rename openspec/changes/{local-rule-routing => archive/2026-06-12-local-rule-routing}/tasks.md (100%) create mode 100644 openspec/specs/cli-detect/spec.md create mode 100644 openspec/specs/cli-rule-routing/spec.md diff --git a/openspec/changes/local-rule-routing/.openspec.yaml b/openspec/changes/archive/2026-06-12-local-rule-routing/.openspec.yaml similarity index 100% rename from openspec/changes/local-rule-routing/.openspec.yaml rename to openspec/changes/archive/2026-06-12-local-rule-routing/.openspec.yaml diff --git a/openspec/changes/local-rule-routing/design.md b/openspec/changes/archive/2026-06-12-local-rule-routing/design.md similarity index 100% rename from openspec/changes/local-rule-routing/design.md rename to openspec/changes/archive/2026-06-12-local-rule-routing/design.md diff --git a/openspec/changes/local-rule-routing/proposal.md b/openspec/changes/archive/2026-06-12-local-rule-routing/proposal.md similarity index 100% rename from openspec/changes/local-rule-routing/proposal.md rename to openspec/changes/archive/2026-06-12-local-rule-routing/proposal.md diff --git a/openspec/changes/local-rule-routing/specs/cli-detect/spec.md b/openspec/changes/archive/2026-06-12-local-rule-routing/specs/cli-detect/spec.md similarity index 100% rename from openspec/changes/local-rule-routing/specs/cli-detect/spec.md rename to openspec/changes/archive/2026-06-12-local-rule-routing/specs/cli-detect/spec.md diff --git a/openspec/changes/local-rule-routing/specs/cli-help/spec.md b/openspec/changes/archive/2026-06-12-local-rule-routing/specs/cli-help/spec.md similarity index 100% rename from openspec/changes/local-rule-routing/specs/cli-help/spec.md rename to openspec/changes/archive/2026-06-12-local-rule-routing/specs/cli-help/spec.md diff --git a/openspec/changes/local-rule-routing/specs/cli-rule-routing/spec.md b/openspec/changes/archive/2026-06-12-local-rule-routing/specs/cli-rule-routing/spec.md similarity index 100% rename from openspec/changes/local-rule-routing/specs/cli-rule-routing/spec.md rename to openspec/changes/archive/2026-06-12-local-rule-routing/specs/cli-rule-routing/spec.md diff --git a/openspec/changes/local-rule-routing/specs/skill-taskless/spec.md b/openspec/changes/archive/2026-06-12-local-rule-routing/specs/skill-taskless/spec.md similarity index 100% rename from openspec/changes/local-rule-routing/specs/skill-taskless/spec.md rename to openspec/changes/archive/2026-06-12-local-rule-routing/specs/skill-taskless/spec.md diff --git a/openspec/changes/local-rule-routing/tasks.md b/openspec/changes/archive/2026-06-12-local-rule-routing/tasks.md similarity index 100% rename from openspec/changes/local-rule-routing/tasks.md rename to openspec/changes/archive/2026-06-12-local-rule-routing/tasks.md diff --git a/openspec/specs/cli-detect/spec.md b/openspec/specs/cli-detect/spec.md new file mode 100644 index 00000000..a9b33f3f --- /dev/null +++ b/openspec/specs/cli-detect/spec.md @@ -0,0 +1,78 @@ +# cli-detect Specification + +## Purpose + +TBD - created by archiving change local-rule-routing. Update Purpose after archive. + +## Requirements + +### Requirement: Detect subcommand exists + +The CLI SHALL provide a `taskless detect` subcommand registered in the top-level +command list, with a `--json` flag and the standard `--dir`/`-d` working-directory +flag. + +#### Scenario: Detect is registered + +- **WHEN** `taskless detect --help` is run +- **THEN** the command SHALL be recognized and print its usage +- **AND** the command SHALL accept `--json` and `--dir`/`-d` + +### Requirement: Detect scans deterministic repo signals only + +The `detect` command SHALL emit only deterministic signals derived from files on +disk: configured linters, detected languages/frameworks, and the styles of the +repo's own existing rules. It SHALL NOT perform any LLM inference and SHALL NOT +match the request against any catalog of known packaged linter rules. + +#### Scenario: Linter configs are detected from disk + +- **WHEN** the working directory contains a recognized linter config (for + example `.eslintrc*`, `eslint.config.js`, `ruff.toml`, a `[tool.ruff]` block in + `pyproject.toml`, `.rubocop.yml`, `biome.json`, or `stylelint` config) +- **THEN** `detect --json` SHALL report each configured linter it found + +#### Scenario: Languages and frameworks are reported + +- **WHEN** `detect --json` runs in a repository +- **THEN** the output SHALL include the languages and frameworks inferred from + manifest and source signals present on disk + +#### Scenario: The repo's own rule styles are surfaced + +- **WHEN** the working directory contains existing rule definitions (for example + custom linter rules or `.taskless/rules/`) +- **THEN** `detect --json` SHALL surface a description of those existing rule + styles for downstream authoring + +#### Scenario: No packaged-rule catalog matching + +- **WHEN** `detect --json` runs +- **THEN** the output SHALL NOT claim a request maps to a specific named packaged + rule (such matching is left to the authoring recipe, not the command) + +### Requirement: Detect runs offline with no network or auth + +The `detect` command SHALL complete without network access and without +authentication. + +#### Scenario: Detect works without login or network + +- **WHEN** `detect --json` runs while logged out and offline +- **THEN** it SHALL produce its signal output successfully +- **AND** it SHALL NOT require or prompt for authentication + +### Requirement: Detect emits a stable JSON shape + +When `--json` is set, `detect` SHALL emit a single structured JSON object whose +shape is validated internally against a stable Zod output schema before being +printed, consistent with how other `--json` commands in the CLI (e.g. `info`, +`check`) validate their output. The schema is an internal contract, not a +published artifact, and `detect` does not expose a `--schema` mode. + +#### Scenario: JSON output validates against the internal schema + +- **WHEN** `detect --json` succeeds +- **THEN** stdout SHALL be a single JSON object that the command has validated + against its internal output schema (linters, languages/frameworks, existing + rule styles) diff --git a/openspec/specs/cli-help/spec.md b/openspec/specs/cli-help/spec.md index f9ba03bc..f9fb6190 100644 --- a/openspec/specs/cli-help/spec.md +++ b/openspec/specs/cli-help/spec.md @@ -155,6 +155,37 @@ The help command's existing intent-telemetry requirement SHALL extend naturally - **WHEN** an agent runs `taskless help onboard` - **THEN** PostHog SHALL receive a `help_onboard` event +### Requirement: Routing topics are registered in the help system + +The help system SHALL register the routing recipes `route`, `existing`, `static`, +and `remote` as embedded help topics, retrievable via `taskless help ` and +listed in the help index, consistent with the existing topic embedding and format +requirements. + +#### Scenario: Routing topics resolve + +- **WHEN** `taskless help route`, `taskless help existing`, + `taskless help static`, or `taskless help remote` is run +- **THEN** the corresponding recipe text SHALL be returned +- **AND** an unknown-topic error SHALL NOT be raised for any of the four + +#### Scenario: Routing topics appear in the index + +- **WHEN** `taskless help` (no arguments) is run +- **THEN** the topic index SHALL include the routing topics so an agent can + discover the authoring front door + +### Requirement: Routing topics emit intent telemetry + +Fetching a routing recipe SHALL emit a per-topic intent telemetry event, +consistent with the existing `help_` telemetry convention. + +#### Scenario: Help topic intent is captured for routing recipes + +- **WHEN** the agent fetches `route`, `existing`, `static`, or `remote` +- **THEN** the help command SHALL capture the corresponding `help_` intent + event with the topic name + ## Goal diff --git a/openspec/specs/cli-rule-routing/spec.md b/openspec/specs/cli-rule-routing/spec.md new file mode 100644 index 00000000..27cca600 --- /dev/null +++ b/openspec/specs/cli-rule-routing/spec.md @@ -0,0 +1,209 @@ +# cli-rule-routing Specification + +## Purpose + +TBD - created by archiving change local-rule-routing. Update Purpose after archive. + +## Requirements + +### Requirement: Route is the local authoring classifier + +The CLI SHALL provide a `route` help recipe that instructs the agent to classify +a rule-authoring request into one of three destinations — `existing`, `static`, +or `remote` — using `taskless detect --json` signals plus the user's intent. The +`route` recipe SHALL be biased to stay local: it SHALL prefer `existing` or +`static` and SHALL treat `remote` as the escalation of last resort. + +#### Scenario: Route fetches detection before classifying + +- **WHEN** the agent fetches the `route` recipe to author a rule +- **THEN** the recipe SHALL direct the agent to run `taskless detect --json` and + use its signals as input to the classification + +#### Scenario: Route classifies into one of three destinations + +- **WHEN** the agent follows `route` +- **THEN** it SHALL select exactly one of `existing`, `static`, or `remote` +- **AND** it SHALL fetch the corresponding recipe to perform the authoring + +### Requirement: Route states reasoning before naming a destination + +The `route` recipe SHALL require the agent to write an explicit rationale before +naming a destination. The rationale SHALL cover what the `detect` signals show, +whether an existing linter plausibly already covers the request, whether the +pattern is expressible as a simple static ast-grep rule, and the resulting +confidence that the request is locally solvable. The destination SHALL be emitted +only after this rationale, and SHALL follow from it. + +#### Scenario: Rationale precedes the route decision + +- **WHEN** the agent follows `route` to classify a request +- **THEN** it SHALL produce a written rationale covering the detection signals, + existing-linter coverage, ast-grep expressibility, and local-solvability + confidence +- **AND** it SHALL name the destination (`existing`, `static`, or `remote`) only + after that rationale + +#### Scenario: Route is not named before reasoning + +- **WHEN** the agent has not yet articulated its reasoning +- **THEN** the recipe SHALL NOT permit committing to a destination +- **AND** the destination SHALL be a conclusion of the rationale, not asserted + ahead of it + +### Requirement: Route commits to the believed-correct path on reasonable confidence + +The `route` recipe SHALL determine the destination upfront from `detect` signals +and the user's intent, committing to the path it believes is correct. The bar to +commit to a local path SHALL be **reasonable confidence**, not certainty. Routing +distinguishes three states: reasonable confidence the request IS locally solvable +selects a local path; reasonable belief the request is NOT locally solvable selects +`remote` directly, without first attempting a local rule; genuine inability to +judge either way is uncertainty, which SHALL be resolved by asking the user (see +the clarifying-question scenario) and SHALL NOT by itself select `remote`. The +recipe SHALL NOT use a deliberate local attempt-and-fail with no genuine belief of +success as the mechanism for choosing `remote`. + +#### Scenario: Reasonably-confident-local commits locally without a justification probe + +- **WHEN** `route` is reasonably confident the request fits an existing linter or + a simple static ast-grep pattern +- **THEN** it SHALL select `existing` or `static` and proceed locally +- **AND** it SHALL NOT run a throwaway local attempt whose only purpose is to + justify the choice + +#### Scenario: Believed-not-local routes remote upfront + +- **WHEN** `route` reasonably believes the request cannot be solved locally — a + positive judgment, not mere inability to tell +- **THEN** it SHALL select `remote` directly +- **AND** it SHALL NOT manufacture a deliberate local failure to reach that + decision + +#### Scenario: Uncertainty biases toward asking, not toward login + +- **WHEN** `route` cannot reasonably place a request as local or remote +- **THEN** it SHALL prefer clarifying with the user over defaulting to `remote` +- **AND** uncertainty alone SHALL NOT be treated as a reason to consume a + generation via `remote` + +### Requirement: A believed-local path that fails escalates only after confirmation + +The `route` recipe SHALL treat try-verify-escalate as a legitimate failure +fallback: when it committed to a local path on reasonable confidence and the +authored rule then fails verification against the user's success/failure cases, it +SHALL surface the failure and SHALL obtain explicit user confirmation before +calling the Taskless service. The recipe SHALL NOT silently fall through from a +failed local attempt to a service call. + +#### Scenario: Failed local attempt prompts before spending a generation + +- **WHEN** a `static` rule the agent committed to fails verification +- **THEN** the recipe SHALL inform the user the local rule could not capture the + cases +- **AND** SHALL state that generating via the Taskless service uses a generation + and requires login +- **AND** SHALL call the service only after the user confirms + +#### Scenario: No silent fall-through to the service + +- **WHEN** a believed-local attempt fails +- **THEN** the recipe SHALL NOT invoke `remote` / the service without an explicit + user confirmation step + +### Requirement: Route asks the user when multiple paths fit + +The `route` recipe SHALL present the viable options to the user with their +trade-offs, rather than silently selecting one, whenever more than one destination +genuinely fits a request (most commonly both `existing` and `static`). The +trade-off framing SHALL note that `remote` consumes a generation and requires +login, so it is appropriate when a request cannot be solved locally rather than as +a default. + +#### Scenario: Both local paths viable surfaces a choice + +- **WHEN** the repository has a detected linter that fits AND the pattern is a + clean local static ast-grep rule +- **THEN** `route` SHALL present both `existing` and `static` with their + trade-offs and let the user choose + +#### Scenario: Trade-off framing names the generation cost of remote + +- **WHEN** `route` presents options that include `remote` +- **THEN** it SHALL state that `remote` consumes a generation and requires login +- **AND** SHALL frame `remote` as the path for what cannot be solved locally + +### Requirement: Existing recipe authors in the detected linter's dialect + +The CLI SHALL provide an `existing` help recipe that instructs the agent to +author a rule in a linter already detected in the repository, expressed in that +tool's own dialect. The recipe SHALL direct the agent to source authoring +knowledge first from the repository's own existing rules and only then from the +agent's own web research. The recipe SHALL NOT embed or rely on a Taskless- +maintained catalog of linter rules. + +#### Scenario: Repo-first knowledge sourcing + +- **WHEN** the agent follows `existing` for a detected linter +- **THEN** it SHALL first mine the repository's existing rules of that kind for + house style +- **AND** SHALL fall back to web research (WebFetch/WebSearch) only when the + repository signal is insufficient + +#### Scenario: Existing path is author-only + +- **WHEN** the agent authors a rule via `existing` +- **THEN** the recipe SHALL make clear the user's own toolchain runs the rule and + that `taskless check` does not execute the external linter + +### Requirement: Static recipe authors a verified local ast-grep rule + +The CLI SHALL provide a `static` help recipe that instructs the agent to author a +local ast-grep rule on-device, without calling the Taskless service, and to +verify it against the user's success and failure cases before reporting success. +The recipe SHALL produce the canonical on-disk rule shape and paths used by remote +generation so that `check`, `improve`, and `verify` see a single dialect. + +#### Scenario: Local authoring without the service + +- **WHEN** the agent follows `static` +- **THEN** it SHALL write the rule on-device without requiring login or the + Taskless API + +#### Scenario: Verification gates success + +- **WHEN** the agent authors a static rule +- **THEN** it SHALL verify the rule against the provided success/failure cases + before reporting the rule as complete + +#### Scenario: Canonical output shape + +- **WHEN** the agent writes a static rule to disk +- **THEN** the files, paths, and shape SHALL match those produced by remote + generation + +### Requirement: Remote recipe collects inputs and delegates to the service + +The CLI SHALL provide a `remote` help recipe that instructs the agent to gather +the inputs required to call the Taskless service and to invoke the existing rule +generation backend, which runs the service-side classifier and returns either a +static or a runtime rule. The `remote` recipe SHALL require authentication and +SHALL NOT itself decide static versus runtime. + +#### Scenario: Remote requires authentication + +- **WHEN** the agent follows `remote` while logged out +- **THEN** the recipe SHALL direct the agent to the authentication flow before + submitting the request + +#### Scenario: Static-versus-runtime is decided by the service + +- **WHEN** the agent submits an authored request via `remote` +- **THEN** the recipe SHALL rely on the service to classify static versus runtime +- **AND** SHALL NOT make that determination locally + +#### Scenario: Remote output matches local on-disk shape + +- **WHEN** the service returns a generated rule via `remote` +- **THEN** the written files and paths SHALL match the shape produced by the + local `static` path diff --git a/openspec/specs/skill-taskless/spec.md b/openspec/specs/skill-taskless/spec.md index 09d776de..651c3664 100644 --- a/openspec/specs/skill-taskless/spec.md +++ b/openspec/specs/skill-taskless/spec.md @@ -32,22 +32,27 @@ The consolidated skill's `description` frontmatter field SHALL anchor triggers o 1. An explicit reference to "Taskless" in the user's message, OR 2. A reference to the `.taskless/` directory or files within it (rules, rule-tests, rule-metadata), OR -3. A request to add/write/create a rule where the user has NOT named a specific lint/format/static-analysis tool. The description SHALL include four illustrative example tools whose presence in the user's message suppresses this trigger: eslint, ruff, biome, ast-grep. The wording SHALL make clear the list is illustrative — any named lint/format/static-analysis tool suppresses the trigger. +3. A request to add/write/create a rule for code, including requests that name a specific lint/format/static-analysis tool (for example eslint, ruff, biome, stylelint, ast-grep). Naming such a tool SHALL engage the skill's routing flow rather than suppress it; the skill routes the request toward the appropriate authoring destination via `npx @taskless/cli help route`. -The description SHALL NOT contain a blanket "do NOT trigger on generic linting" instruction; that prior carve-out is replaced by the named-tool suppression in clause 3. +The description SHALL NOT instruct the agent to suppress or quiet itself merely because a lint/format/static-analysis tool is named, and SHALL NOT contain a blanket "do NOT trigger on generic linting" instruction. #### Scenario: Description includes anchored trigger phrases - **WHEN** the skill `description` field is read - **THEN** it SHALL include trigger phrases such as "create/add/write a taskless rule", "improve/fix/iterate on this taskless rule", "run taskless", "taskless login", "add taskless to CI" -- **AND** SHALL include the unspecified-tool clause covering "add/write/create a rule" with no tool named -- **AND** SHALL list at least the four illustrative suppressing tool names: eslint, ruff, biome, ast-grep +- **AND** SHALL include a rule-authoring clause covering "add/write/create a rule" for code -#### Scenario: Description omits the prior blanket carve-out +#### Scenario: Naming a linter engages routing rather than suppressing + +- **WHEN** the user asks to add/write/create a rule and names a specific lint/format/static-analysis tool (for example "write an eslint rule for X") +- **THEN** the skill SHALL trigger and route the request via `npx @taskless/cli help route` +- **AND** SHALL NOT quiet itself to a one-line offer on the basis of the named tool + +#### Scenario: Description omits suppression and the prior blanket carve-out - **WHEN** the skill `description` field is read -- **THEN** it SHALL NOT contain wording instructing the agent to never trigger on generic ESLint/linting requests -- **AND** any suppression wording SHALL be expressed via the named-tool clause +- **THEN** it SHALL NOT contain wording instructing the agent to suppress on a named lint/format/static-analysis tool +- **AND** it SHALL NOT contain wording instructing the agent to never trigger on generic ESLint/linting requests #### Scenario: Description is at most 1024 characters @@ -129,3 +134,24 @@ The consolidated skill's frontmatter SHALL include `metadata.commandName: tskl` - **WHEN** a user invokes `/tskl` with no arguments - **THEN** the command body SHALL instruct the agent to ask the user what they want to do with Taskless before proceeding + +### Requirement: Skill routes authoring requests through the routing front door + +The skill body SHALL route rule-authoring requests through `npx @taskless/cli help route` +as the authoring front door, rather than fetching `rule create` directly. The +skill SHALL add no linter knowledge of its own and SHALL remain a thin router that +defers all authoring judgment to the fetched recipes. + +#### Scenario: Authoring requests are routed via route + +- **WHEN** the user asks to author a rule (with or without naming a tool) +- **THEN** the skill body SHALL direct the agent to fetch `npx @taskless/cli help route` + before any login-gated path +- **AND** SHALL NOT embed linter-specific rule knowledge in the skill body + +#### Scenario: Local-first before login + +- **WHEN** the routing recipe has not yet demonstrated that a rule cannot be built + locally +- **THEN** the skill SHALL NOT direct the agent toward a login-gated authoring + path From 3d838a9f2bd4c3b94bae415bdd5cc5c227e27bd0 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Sat, 13 Jun 2026 21:01:56 -0700 Subject: [PATCH 6/9] docs(cli): Sync cli-detect spec to new shape; document test layers - Mirror the detect contract changes into the synced cli-detect capability spec: drop frameworks, add the languages-only / per-language-manifest / monorepo scenarios. - Add a README testing section clarifying the two kinds of test: the local vitest suite (deterministic, no agent) versus the route-honesty dataset, whose automated test only guards the fixture structure while the actual agent evaluation is a separate manual calibration step. Co-Authored-By: Claude Opus 4.8 --- openspec/specs/cli-detect/spec.md | 38 ++++++++++++++++++++++++------- packages/cli/README.md | 24 +++++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/openspec/specs/cli-detect/spec.md b/openspec/specs/cli-detect/spec.md index a9b33f3f..8fef84e5 100644 --- a/openspec/specs/cli-detect/spec.md +++ b/openspec/specs/cli-detect/spec.md @@ -21,9 +21,16 @@ flag. ### Requirement: Detect scans deterministic repo signals only The `detect` command SHALL emit only deterministic signals derived from files on -disk: configured linters, detected languages/frameworks, and the styles of the -repo's own existing rules. It SHALL NOT perform any LLM inference and SHALL NOT -match the request against any catalog of known packaged linter rules. +disk: configured linters, detected languages, and the styles of the repo's own +existing rules. It SHALL NOT perform any LLM inference and SHALL NOT match the +request against any catalog of known packaged linter rules. + +Detection follows a languages → linters flow: languages are inferred first, and +a linter's dependency evidence is then read from the manifest of that linter's +own language (a node dependency from `package.json`, a Python dependency from +`pyproject.toml`/`requirements.txt`) rather than conflating ecosystems. A +recognized linter config file on disk is honored regardless of the languages +inferred. #### Scenario: Linter configs are detected from disk @@ -32,11 +39,27 @@ match the request against any catalog of known packaged linter rules. `pyproject.toml`, `.rubocop.yml`, `biome.json`, or `stylelint` config) - **THEN** `detect --json` SHALL report each configured linter it found -#### Scenario: Languages and frameworks are reported +#### Scenario: Languages are reported - **WHEN** `detect --json` runs in a repository -- **THEN** the output SHALL include the languages and frameworks inferred from - manifest and source signals present on disk +- **THEN** the output SHALL include the languages inferred from manifest and + marker files present on disk and from the linters detected + +#### Scenario: A linter dependency is sourced from its own language's manifest + +- **WHEN** a dependency-evidenced linter (for example `ruff`) is named only in a + manifest belonging to a different language (for example `package.json`) +- **THEN** `detect --json` SHALL NOT report that linter from the mismatched + manifest + +#### Scenario: Configs in monorepo sub-packages are detected + +- **WHEN** a linter config or language manifest lives in a sub-package rather + than the repository root (for example `packages/api/.eslintrc.json`) +- **THEN** `detect --json` SHALL detect it and SHALL carry the path it was found + at in the linter's evidence +- **AND** the scan SHALL prune a curated set of ignored directories (for example + `node_modules`, `.git`, build output) and SHALL bound traversal depth #### Scenario: The repo's own rule styles are surfaced @@ -74,5 +97,4 @@ published artifact, and `detect` does not expose a `--schema` mode. - **WHEN** `detect --json` succeeds - **THEN** stdout SHALL be a single JSON object that the command has validated - against its internal output schema (linters, languages/frameworks, existing - rule styles) + against its internal output schema (linters, languages, existing rule styles) diff --git a/packages/cli/README.md b/packages/cli/README.md index 15e05f6c..db8635e2 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -176,6 +176,30 @@ All commands output structured JSON to stdout by default. Parse with `JSON.parse ## Developing +### Testing + +```bash +pnpm --filter @taskless/cli test # run the suite once +pnpm --filter @taskless/cli exec vitest # watch mode +``` + +The suite runs entirely locally under vitest — no network, no auth, no agent. +Integration tests that exercise the built binary (for example `detect`) run +against `dist/`, so run `pnpm --filter @taskless/cli build` first (or after any +source change) before invoking them directly. + +**Two kinds of test, one of which is not fully automatable.** Most tests are +deterministic unit/integration checks. The route-honesty dataset +(`test/fixtures/route-eval.json`) is different: the actual routing decision is +made by an _agent_ following `help/route.txt`, so it cannot be asserted by a +code classifier. The automated test (`test/route-eval.test.ts`) therefore only +**guards the dataset** — that it stays structurally valid and balanced across +every route and both failure directions (over-claim / over-escalate). Running +the dataset _as an evaluation_ — feeding each case to an agent and scoring its +chosen destination — is a separate, manual calibration step with more setup; it +is not part of `pnpm test`. Keep the two distinct: the suite proves the fixtures +are well-formed; an agent run proves the recipe routes honestly. + ### API base URL The CLI resolves the API base URL in this order: From d30341c47c10468ba21c5d275c6be9dd074c38a6 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Sat, 13 Jun 2026 21:28:45 -0700 Subject: [PATCH 7/9] chore(cli): Anchor the telemetry-taxonomy reconciliation in code When the restructure-cli-telemetry change lands, detect/help must conform to the cli_run + cli_help taxonomy. Mark both sites with a shared TODO(telemetry-taxonomy) tag: drop the silent cli_detect capture in detect.ts (the easy-to-miss half) and convert the help_ assertions (which fail on rebase, so they self-surface). Co-Authored-By: Claude Opus 4.8 --- packages/cli/src/commands/detect.ts | 6 ++++++ packages/cli/test/help-routing-telemetry.test.ts | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/packages/cli/src/commands/detect.ts b/packages/cli/src/commands/detect.ts index c32756fe..7a0ff294 100644 --- a/packages/cli/src/commands/detect.ts +++ b/packages/cli/src/commands/detect.ts @@ -28,6 +28,12 @@ export const detectCommand = defineCommand({ async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); const telemetry = await getTelemetry(cwd); + // TODO(telemetry-taxonomy): once the restructure-cli-telemetry change lands + // on main, drop this bespoke cli_detect capture. detect is a read-only + // command with no state transition, so it rides on the cli_run denominator + // alone (the new taxonomy lists info/detect among cli_run-only commands). + // This emit is silent under the new taxonomy — nothing fails if it lingers — + // so it is the reconciliation's easy-to-miss half. See the telemetry stack. telemetry.capture("cli_detect"); const result = { diff --git a/packages/cli/test/help-routing-telemetry.test.ts b/packages/cli/test/help-routing-telemetry.test.ts index 79437851..29414b17 100644 --- a/packages/cli/test/help-routing-telemetry.test.ts +++ b/packages/cli/test/help-routing-telemetry.test.ts @@ -1,5 +1,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +// TODO(telemetry-taxonomy): once the restructure-cli-telemetry change lands on +// main, the per-topic `help_` events collapse into a single +// `cli_help { topic }`. These assertions then move to `cli_help` with a topic +// property, and the help command stops emitting bespoke per-topic names. This +// half is self-surfacing: against the new help command this suite fails, so the +// rebase can't silently skip it. See the telemetry stack's analytics spec. +// // Spy on the telemetry capture by mocking the telemetry module the help // command imports. The factory is invoked lazily at import time, so the // closure over `capture` resolves after initialization (same pattern as From 26c7ced592980cc0e1bccbabfea0fee853b1c148 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Sat, 13 Jun 2026 21:32:40 -0700 Subject: [PATCH 8/9] chore(cli): Point the telemetry-taxonomy TODOs at issue #39 Reference the tracking issue (#39) from both reconciliation anchors so the context lives next to the code. Co-Authored-By: Claude Opus 4.8 --- packages/cli/src/commands/detect.ts | 4 ++-- packages/cli/test/help-routing-telemetry.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/detect.ts b/packages/cli/src/commands/detect.ts index 7a0ff294..6b4fb13f 100644 --- a/packages/cli/src/commands/detect.ts +++ b/packages/cli/src/commands/detect.ts @@ -28,8 +28,8 @@ export const detectCommand = defineCommand({ async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); const telemetry = await getTelemetry(cwd); - // TODO(telemetry-taxonomy): once the restructure-cli-telemetry change lands - // on main, drop this bespoke cli_detect capture. detect is a read-only + // TODO(#39): once the restructure-cli-telemetry change lands on main, drop + // this bespoke cli_detect capture. detect is a read-only // command with no state transition, so it rides on the cli_run denominator // alone (the new taxonomy lists info/detect among cli_run-only commands). // This emit is silent under the new taxonomy — nothing fails if it lingers — diff --git a/packages/cli/test/help-routing-telemetry.test.ts b/packages/cli/test/help-routing-telemetry.test.ts index 29414b17..de6e7c20 100644 --- a/packages/cli/test/help-routing-telemetry.test.ts +++ b/packages/cli/test/help-routing-telemetry.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -// TODO(telemetry-taxonomy): once the restructure-cli-telemetry change lands on -// main, the per-topic `help_` events collapse into a single +// TODO(#39): once the restructure-cli-telemetry change lands on main, the +// per-topic `help_` events collapse into a single // `cli_help { topic }`. These assertions then move to `cli_help` with a topic // property, and the help command stops emitting bespoke per-topic names. This // half is self-surfacing: against the new help command this suite fails, so the From d60e74fcb478edde686054a08d5f015ed990e06e Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Sat, 13 Jun 2026 22:06:47 -0700 Subject: [PATCH 9/9] =?UTF-8?q?chore(cli):=20Resolve=20#39=20=E2=80=94=20c?= =?UTF-8?q?onform=20detect/help=20to=20the=20cli=5F=20taxonomy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The telemetry restructure has landed on main, so the detect stack now adopts the new event taxonomy: - detect.ts: drop the bespoke cli_detect capture. detect is read-only, so the per-invocation cli_run denominator (from the runner) covers it. Removes the now-unused getTelemetry import. - help-routing-telemetry.test.ts: the per-topic help_ events collapsed into cli_help { topic }; assert cli_help for the routing topics. Clears both TODO(#39) anchors. Closes #39. Co-Authored-By: Claude Opus 4.8 --- packages/cli/src/commands/detect.ts | 12 +++--------- packages/cli/test/help-routing-telemetry.test.ts | 13 +++---------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/packages/cli/src/commands/detect.ts b/packages/cli/src/commands/detect.ts index 6b4fb13f..f0be6b32 100644 --- a/packages/cli/src/commands/detect.ts +++ b/packages/cli/src/commands/detect.ts @@ -4,7 +4,6 @@ import { defineCommand } from "citty"; import { detectRepository } from "../detect/scan"; import { outputSchema as detectOutputSchema } from "../schemas/detect"; -import { getTelemetry } from "../telemetry"; import { makeErrorEnvelope } from "../types/errors"; export const detectCommand = defineCommand({ @@ -27,14 +26,9 @@ export const detectCommand = defineCommand({ }, async run({ args }) { const cwd = resolve(args.dir ?? process.cwd()); - const telemetry = await getTelemetry(cwd); - // TODO(#39): once the restructure-cli-telemetry change lands on main, drop - // this bespoke cli_detect capture. detect is a read-only - // command with no state transition, so it rides on the cli_run denominator - // alone (the new taxonomy lists info/detect among cli_run-only commands). - // This emit is silent under the new taxonomy — nothing fails if it lingers — - // so it is the reconciliation's easy-to-miss half. See the telemetry stack. - telemetry.capture("cli_detect"); + // detect is read-only with no state transition, so it emits no bespoke + // event — the per-invocation cli_run denominator (emitted by the runner) + // covers it, consistent with info under the cli_ telemetry taxonomy. const result = { success: true as const, diff --git a/packages/cli/test/help-routing-telemetry.test.ts b/packages/cli/test/help-routing-telemetry.test.ts index de6e7c20..ece49ffe 100644 --- a/packages/cli/test/help-routing-telemetry.test.ts +++ b/packages/cli/test/help-routing-telemetry.test.ts @@ -1,12 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -// TODO(#39): once the restructure-cli-telemetry change lands on main, the -// per-topic `help_` events collapse into a single -// `cli_help { topic }`. These assertions then move to `cli_help` with a topic -// property, and the help command stops emitting bespoke per-topic names. This -// half is self-surfacing: against the new help command this suite fails, so the -// rebase can't silently skip it. See the telemetry stack's analytics spec. -// // Spy on the telemetry capture by mocking the telemetry module the help // command imports. The factory is invoked lazily at import time, so the // closure over `capture` resolves after initialization (same pattern as @@ -31,7 +24,7 @@ interface RunnableCommand { }) => Promise; } -describe("help routing topics emit help_ intent telemetry", () => { +describe("help routing topics emit cli_help intent telemetry", () => { let logSpy: ReturnType; beforeEach(() => { @@ -45,7 +38,7 @@ describe("help routing topics emit help_ intent telemetry", () => { }); it.each(["route", "existing", "static", "remote"])( - "captures help_%s", + "captures cli_help for %s", async (topic) => { const command = createHelpCommand({}) as unknown as RunnableCommand; await command.run({ @@ -54,7 +47,7 @@ describe("help routing topics emit help_ intent telemetry", () => { }); expect(capture).toHaveBeenCalledWith( - `help_${topic}`, + "cli_help", expect.objectContaining({ topic }) ); }