From 1b62c987a1329d91176d7a66319a5272706bdcf1 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 21:15:01 +0200 Subject: [PATCH 1/2] docs: the capture layout is the interface, on disk or mirrored from a bucket README and DOCS.md still said the SDK-CLI interface is files under .evalshift/captures/. Since captures.store (1.3.0) the CLI also mirrors that layout from a bucket, which AGENTS.md already said. Co-Authored-By: Claude Fable 5.1 --- DOCS.md | 2 +- README.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/DOCS.md b/DOCS.md index 28d95474..d1ec88e3 100644 --- a/DOCS.md +++ b/DOCS.md @@ -93,7 +93,7 @@ EvalShift is four pieces. Each is released and documented independently; each ow | **GitHub Action** | `evalshift/evalshift-action@v0` | Runs the pipeline on PRs, pushes the run, maintains one PR comment, sets the `evalshift/regression` status. | [docs/github-action.md](docs/github-action.md), [action repo](https://github.com/evalshift/evalshift-action) | | | **Hosted server** | service — API `https://api.evalshift.dev`, web app `https://evalshift.dev` | Stores pushed run bundles, diffs runs across branches, serves the web app, drives PR comments and gating. | [docs/hosted.md](docs/hosted.md) | covered by the CLI reference (`push`/`bundle` contract) | -Data flow is one-directional: **SDK captures → CLI runs and bundles → server stores and diffs → web app displays.** The SDK and CLI never call each other — the interface is files under `.evalshift/captures/`. The CLI (import `evalshift_cli`) depends on the SDK (import `evalshift`), so one environment holds both. +Data flow is one-directional: **SDK captures → CLI runs and bundles → server stores and diffs → web app displays.** The SDK and CLI never call each other — the interface is the capture layout under `.evalshift/captures/`, on disk or mirrored from a bucket you own (`captures.store`). The CLI (import `evalshift_cli`) depends on the SDK (import `evalshift`), so one environment holds both. The CLI reference is generated from [llms-full.txt](llms-full.txt) at this repo's root — edit that file when CLI behaviour changes. The SDK and Action references are owned by their own repos; the copies served from `evalshift.dev` are synced from there. diff --git a/README.md b/README.md index 36259ee2..3b6790d8 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,9 @@ Four pieces, released and documented independently: | **GitHub Action** — `evalshift/evalshift-action@v0` | Runs the pipeline on pull requests, pushes the run, posts one PR comment, sets the `evalshift/regression` status. | [docs/github-action.md](docs/github-action.md) | | **Hosted server** — `api.evalshift.dev`, web app at `evalshift.dev` | Optional. Stores pushed run bundles, diffs them across branches, drives PR comments and gating. | [docs/hosted.md](docs/hosted.md) | -The SDK and the CLI never call each other — the interface is files under -`.evalshift/captures/`, so either works without the other. The CLI (import +The SDK and the CLI never call each other — the interface is the capture layout +under `.evalshift/captures/`, on disk or mirrored from a bucket you own +(`captures.store`), so either works without the other. The CLI (import package `evalshift_cli`) depends on the SDK (import name `evalshift`), so one environment holds both: `pip install evalshift` brings the SDK with it, and a production agent that only records captures installs `evalshift-sdk` alone. From d096c7ccfc9d6a0a7c5c994714a6661458d377ff Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 21:15:01 +0200 Subject: [PATCH 2/2] docs(init): the scaffolded EVALSHIFT.md names capture fetch The capture row listed five subcommands; fetch shipped in 1.3.0 without joining it. A docs-currency test now reads the Typer group and fails when any registered capture subcommand is missing from that row. Co-Authored-By: Claude Fable 5.1 --- src/evalshift_cli/cli/commands/_agents.py | 2 +- tests/unit/test_docs_currency.py | 26 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/evalshift_cli/cli/commands/_agents.py b/src/evalshift_cli/cli/commands/_agents.py index 0e10be73..2146fd70 100644 --- a/src/evalshift_cli/cli/commands/_agents.py +++ b/src/evalshift_cli/cli/commands/_agents.py @@ -152,7 +152,7 @@ | `evalshift report` | Render the single-file HTML report. | | `evalshift compare` | Run doctor -> run -> evaluate -> analyze -> report end to end on one suite (formerly `all`). | | `evalshift inspect` | Inspect a run / artifact. | -| `evalshift capture ...` | `list` / `promote` / `clean` / `diff` / `sync` captures into suites. | +| `evalshift capture ...` | `list` / `fetch` / `promote` / `clean` / `diff` / `sync` captures into suites. | | `evalshift diff case` | Diff a single case between runs. | | `evalshift replay case` | Replay a single case. | | `evalshift traces import` | Import external traces. | diff --git a/tests/unit/test_docs_currency.py b/tests/unit/test_docs_currency.py index 42228931..fe0afa7e 100644 --- a/tests/unit/test_docs_currency.py +++ b/tests/unit/test_docs_currency.py @@ -19,6 +19,8 @@ import pytest import evalshift_cli +from evalshift_cli.cli.commands._agents import AGENT_INSTRUCTIONS +from evalshift_cli.cli.commands.capture import capture_app from evalshift_cli.cli.commands.init import PROVIDERS from evalshift_cli.models.registry import PROVIDER_ENV_VARS @@ -227,3 +229,27 @@ def _whitespace_normalized(name: str) -> str: def test_copy_does_not_describe_the_retired_hosted_plans(term: str) -> None: offenders = [name for name in _PLAN_COPY_FILES if term in _whitespace_normalized(name)] assert offenders == [], f"{term!r} still appears in {offenders}" + + +def _registered_capture_subcommands() -> list[str]: + """Every subcommand `evalshift capture --help` lists, straight from the Typer group.""" + return sorted(str(cmd.name) for cmd in capture_app.registered_commands) + + +@pytest.mark.parametrize("name", _registered_capture_subcommands()) +def test_scaffolded_agent_guide_names_every_capture_subcommand(name: str) -> None: + """`init` writes EVALSHIFT.md into user repos, and its `evalshift capture ...` + row is the one place a coding agent learns which subcommands exist. + + `capture fetch` shipped in 1.3.0 without being added to that row, so every + scaffolded guide told agents the group had five subcommands. The row is + matched by its leading cell so a bare word like `list` elsewhere in the + guide cannot satisfy the check. + """ + row = next( + line + for line in AGENT_INSTRUCTIONS.splitlines() + if line.startswith("| `evalshift capture ...`") + ) + + assert f"`{name}`" in row, f"EVALSHIFT.md's capture row does not name `{name}`"