From 2ff25b6823f36e3685b184fc1df70d68d18ebb23 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 19:08:50 +0200 Subject: [PATCH 01/17] feat(captures): store URI grammar shared with the SDK Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/store_uri.py | 92 +++++++++++++++++++++++++ tests/unit/test_captures_store_uri.py | 65 +++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 src/evalshift_cli/captures/store_uri.py create mode 100644 tests/unit/test_captures_store_uri.py diff --git a/src/evalshift_cli/captures/store_uri.py b/src/evalshift_cli/captures/store_uri.py new file mode 100644 index 00000000..8a714779 --- /dev/null +++ b/src/evalshift_cli/captures/store_uri.py @@ -0,0 +1,92 @@ +"""Store URI grammar, shared verbatim with the SDK (``evalshift.stores.uri``). + +Three forms are accepted:: + + s3:/// + gs:/// + az://// + +```` is optional and may be empty; a trailing slash is stripped. Credentials never +belong in a URI: any ``@`` or ``?`` is rejected so nobody is tempted to embed a key or a +query parameter, and every adapter uses its provider's default credential chain instead. + +This module deliberately imports nothing from the project so :mod:`evalshift_cli.config.models` +can validate ``captures.store`` with it without an import cycle. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal, cast +from urllib.parse import urlsplit + +Scheme = Literal["s3", "gs", "az"] + +#: The three accepted forms, quoted in every grammar error. +STORE_URI_FORMS = ( + "s3:///, gs:///, az:////" +) + +_EXTRA_FOR_SCHEME: dict[str, str] = {"s3": "s3", "gs": "gcs", "az": "azure"} + + +@dataclass(frozen=True, slots=True) +class StoreURI: + """A parsed ``captures.store`` URI. + + Attributes: + scheme: ``"s3"``, ``"gs"`` or ``"az"``. + bucket: The S3 / GCS bucket, or the Azure storage *account*. + container: The Azure blob container; ``None`` for S3 and GCS. + prefix: Key prefix under which ``captures/`` and ``toolsets/`` live. May be ``""``. + """ + + scheme: Scheme + bucket: str + container: str | None + prefix: str + + @property + def extra(self) -> str: + """The pip extra (``evalshift[]``) that installs this scheme's client.""" + return _EXTRA_FOR_SCHEME[self.scheme] + + +def parse_store_uri(uri: str) -> StoreURI: + """Parse ``uri`` against the grammar above. + + Raises: + ValueError: for an unknown scheme, a missing bucket or container, or a URI carrying + credentials (``@``) or query parameters (``?``). The message names the accepted + forms so a config error is self-explanatory; the ``@`` and ``?`` messages never + echo the URI, since that is exactly where a key or signature would sit. + """ + text = uri.strip() + if "@" in text: + raise ValueError( + "credentials are not accepted in a store URI; use the provider's credential chain" + ) + if "?" in text: + # Never echo the URI: a query string is where SAS tokens and signatures live. + raise ValueError( + "query parameters are not accepted in a store URI; use the provider's credential chain" + ) + parts = urlsplit(text) + if parts.scheme not in _EXTRA_FOR_SCHEME: + raise ValueError(f"unsupported store URI {text!r}; accepted forms: {STORE_URI_FORMS}") + if not parts.netloc: + raise ValueError(f"store URI {text!r} names no bucket; accepted forms: {STORE_URI_FORMS}") + scheme = cast(Scheme, parts.scheme) + path = parts.path.strip("/") + if scheme == "az": + container, _, prefix = path.partition("/") + if not container: + raise ValueError( + f"Azure store URI {text!r} names no container; " + "expected az:////" + ) + return StoreURI(scheme, parts.netloc, container, prefix.strip("/")) + return StoreURI(scheme, parts.netloc, None, path) + + +__all__ = ["STORE_URI_FORMS", "Scheme", "StoreURI", "parse_store_uri"] diff --git a/tests/unit/test_captures_store_uri.py b/tests/unit/test_captures_store_uri.py new file mode 100644 index 00000000..ce1319d9 --- /dev/null +++ b/tests/unit/test_captures_store_uri.py @@ -0,0 +1,65 @@ +"""The store URI grammar — shared verbatim with the SDK; these vectors are the contract.""" + +from __future__ import annotations + +import pytest + +from evalshift_cli.captures.store_uri import StoreURI, parse_store_uri + + +@pytest.mark.parametrize( + ("uri", "expected"), + [ + ("s3://acme-evals/support-agent", StoreURI("s3", "acme-evals", None, "support-agent")), + ("s3://acme-evals/support-agent/", StoreURI("s3", "acme-evals", None, "support-agent")), + ("s3://acme-evals", StoreURI("s3", "acme-evals", None, "")), + ("s3://acme-evals/", StoreURI("s3", "acme-evals", None, "")), + ("s3://acme-evals/a/b/c", StoreURI("s3", "acme-evals", None, "a/b/c")), + ("gs://acme-evals/support-agent", StoreURI("gs", "acme-evals", None, "support-agent")), + ("az://acmeprod/evals/support-agent", StoreURI("az", "acmeprod", "evals", "support-agent")), + ("az://acmeprod/evals", StoreURI("az", "acmeprod", "evals", "")), + ("az://acmeprod/evals/a/b/", StoreURI("az", "acmeprod", "evals", "a/b")), + (" s3://acme-evals/x ", StoreURI("s3", "acme-evals", None, "x")), + ], +) +def test_accepted_forms(uri: str, expected: StoreURI) -> None: + assert parse_store_uri(uri) == expected + + +@pytest.mark.parametrize( + ("uri", "fragment"), + [ + ("ftp://bucket/prefix", "accepted forms"), + ("http://bucket/prefix", "accepted forms"), + ("bucket/prefix", "accepted forms"), + ("", "accepted forms"), + ("s3:///prefix", "names no bucket"), + ("az://acmeprod", "names no container"), + ("az://acmeprod/", "names no container"), + ("s3://key:secret@bucket/prefix", "credential"), + ("s3://bucket/prefix?region=eu", "query"), + ], +) +def test_rejected_forms(uri: str, fragment: str) -> None: + with pytest.raises(ValueError, match=fragment): + parse_store_uri(uri) + + +def test_extra_name_per_scheme() -> None: + assert parse_store_uri("s3://b").extra == "s3" + assert parse_store_uri("gs://b").extra == "gcs" + assert parse_store_uri("az://a/c").extra == "azure" + + +@pytest.mark.parametrize( + ("uri", "secret"), + [ + ("az://acct/c?sv=2024&sig=SECRETSIG", "SECRETSIG"), + ("s3://AKIAKEY:SECRET@bucket/p", "SECRET"), + ("s3://AKIAKEY:SECRET@bucket/p", "AKIAKEY"), + ], +) +def test_credential_rejection_does_not_echo_secret(uri: str, secret: str) -> None: + with pytest.raises(ValueError, match="credential chain") as excinfo: + parse_store_uri(uri) + assert secret not in str(excinfo.value) From 569a5eb8ae669ab658f0bcb5939453ce94ac7aa3 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 19:12:20 +0200 Subject: [PATCH 02/17] feat(config): captures.store names the object store captures are mirrored from Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 46 +++++++++++++++ src/evalshift_cli/config/models.py | 39 ++++++++++++- tests/unit/test_config_captures.py | 93 ++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_config_captures.py diff --git a/docs/configuration.md b/docs/configuration.md index 66766079..9a73ec72 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,6 +21,7 @@ evaluators: {...} # optional (but at least one is needed for `evaluate`) suites: {...} # optional named suites for `run --suite-name`, each with # its own optional `evaluators:` block retention: {...} # optional run-history pruning policy +captures: {...} # optional, see below ``` Unknown keys are rejected (`extra: forbid` everywhere) so typos fail @@ -1013,6 +1014,46 @@ evalshift runs clean --suite main_chat # restrict to one suite `--keep` beats `EVALSHIFT_MAX_RUNS`, which beats the config value. `runs clean` works even without a valid `evalshift.yaml` (it falls back to the defaults), so it's always available for disk cleanup. +## `captures` + +Where the capture SDK's recordings live when they are not on this machine. Omit the block +entirely for the default: captures are read from `.evalshift/captures/` (or `EVALSHIFT_DIR`) +on local disk and nothing here is needed. + +Production agents on Fargate, Lambda or Kubernetes lose their disk when they stop, so the +SDK can ship captures to an object store you own instead (`EVALSHIFT_SINK=` in the +agent's environment — see [Capture SDK](sdk.md)). Name the same store here and the CLI +pulls new captures down before it promotes them, with no flags to remember: + +```yaml +captures: + store: s3://acme-evals/support-agent +``` + +| Field | Type | Default | Description | +| ------- | ------------- | ------- | ----------- | +| `store` | string (URI) | (none) | Object store the SDK writes to. `capture sync` and `capture list` fetch new captures and toolset sidecars from it into the local directory first; `capture fetch` does only that step. | + +Accepted URI forms — the same grammar the SDK uses: + +| Form | Store | Install | Credentials | +| --- | --- | --- | --- | +| `s3:///` | Amazon S3; MinIO, Cloudflare R2, Backblaze B2 via boto3's `AWS_ENDPOINT_URL` | `pip install "evalshift[s3]"` | IAM role in CI (OIDC), `aws sso login` locally | +| `gs:///` | Google Cloud Storage | `pip install "evalshift[gcs]"` | Workload Identity in CI, `gcloud auth application-default login` locally | +| `az:////` | Azure Blob Storage | `pip install "evalshift[azure]"` | Managed Identity / federated credential in CI, `az login` locally | + +`` is optional. Credentials never go in the URI: a value containing `@` or `?` fails +to load, naming the accepted forms. Under the prefix the layout is exactly the local one — +`captures//cap_.json` and `toolsets/.json` — so the bucket is a mirror of +`.evalshift/`, not a different format. + +What the fetch does: it lists the bucket, downloads only objects missing locally (captures +are immutable and toolsets content-addressed, so "exists" means "current"), skips captures +already promoted into a suite (so `capture clean` never causes a re-download), and writes +each file atomically. `--since 24h` (or `7d`, `30m`, an ISO date) limits a fetch to recent +objects; `--offline` skips it and works with what is already local. Retention in the bucket +is yours to set with a lifecycle rule; the SDK's `max_captures` does not apply there. + ## Capture lifecycle The companion `evalshift-sdk` package records real agent runs to @@ -1021,11 +1062,16 @@ the base). The CLI consumes them via the `capture` commands: ```shell evalshift capture list # see what the SDK recorded +evalshift capture fetch # (captures.store only) mirror new captures locally evalshift capture sync --input-var query # promote every capture + wire suites: evalshift run --suite-name support_agent --yes # score a candidate model against it evalshift capture clean # prune already-promoted captures ``` +With [`captures.store`](#captures) configured, `list` and `sync` run the same fetch first +and print one summary line (`fetched 12 capture(s) and 1 toolset(s) from s3://…`); pass +`--offline` to skip it. `promote`, `diff` and `clean` operate on the local mirror only. + `capture clean []` deletes promoted captures by default (`--promoted`); `--all` deletes every capture, promoted or not. It never touches promoted suites, then offers to sweep toolset sidecars no surviving capture or suite diff --git a/src/evalshift_cli/config/models.py b/src/evalshift_cli/config/models.py index a46de0a9..6b0fe877 100644 --- a/src/evalshift_cli/config/models.py +++ b/src/evalshift_cli/config/models.py @@ -19,8 +19,9 @@ from jsonschema import Draft202012Validator from jsonschema.exceptions import SchemaError -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from evalshift_cli.captures.store_uri import STORE_URI_FORMS, parse_store_uri from evalshift_cli.suite.tags import RESERVED_SLICE_NAME # Default judge model used across the config when none is specified. @@ -815,6 +816,41 @@ class Retention(_StrictModel): run_ttl_days: int | None = Field(default=None, ge=1) +class CapturesConfig(_StrictModel): + """Where the capture SDK's recordings live, beyond the local ``.evalshift/`` directory. + + By default captures are read from ``/captures/`` on this machine and nothing here + is needed. Production hosts whose disk does not outlive them (Fargate, Lambda, pods) can + ship captures to an object store instead (``EVALSHIFT_SINK`` on the SDK side); naming the + same store here makes ``capture sync`` and ``capture list`` mirror new captures into the + local directory first, with no flags. + + Attributes: + store: ``s3:///``, ``gs:///`` or + ``az:////``. Credentials come from the provider's + default chain, never from this file; a URI carrying ``@`` or ``?`` is rejected. + ``None`` (the default) means local disk only. + """ + + store: str | None = None + + @field_validator("store") + @classmethod + def _validate_store(cls, value: str | None) -> str | None: + if value is None: + return None + try: + parse_store_uri(value) + except ValueError as exc: + # The parser's ``@`` / ``?`` messages deliberately omit both the URI (a secret + # would sit there) and the accepted forms; a config error must still name them. + message = str(exc) + if "accepted forms" not in message: + message = f"{message}; accepted forms: {STORE_URI_FORMS}" + raise ValueError(message) from None + return value.strip() + + class EvalShiftConfig(_StrictModel): """Top-level configuration loaded from ``evalshift.yaml``.""" @@ -828,6 +864,7 @@ class EvalShiftConfig(_StrictModel): # Empty by default so every pre-existing config stays valid. suites: dict[str, SuiteSource] = Field(default_factory=dict) retention: Retention = Field(default_factory=Retention) + captures: CapturesConfig = Field(default_factory=CapturesConfig) def evaluators_for(self, suite_name: str | None) -> EvaluatorsConfig: """Resolve the evaluator set a given named suite is scored with. diff --git a/tests/unit/test_config_captures.py b/tests/unit/test_config_captures.py new file mode 100644 index 00000000..1cfa65dc --- /dev/null +++ b/tests/unit/test_config_captures.py @@ -0,0 +1,93 @@ +"""The ``captures:`` block of ``evalshift.yaml`` — where the SDK's recordings live.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from pydantic import ValidationError +from rich.console import Console + +from evalshift_cli.config.loader import ConfigError, load_config +from evalshift_cli.config.models import CapturesConfig, EvalShiftConfig + +_PROMPT = {"id": "p", "detection": "manual", "content": "hello {name}"} + + +def test_default_is_no_store() -> None: + cfg = EvalShiftConfig(prompts=[_PROMPT]) + assert cfg.captures == CapturesConfig() + assert cfg.captures.store is None + + +@pytest.mark.parametrize( + "uri", + ["s3://acme-evals/support-agent", "gs://acme-evals/x", "az://acmeprod/evals/x"], +) +def test_store_accepts_each_scheme(uri: str) -> None: + cfg = EvalShiftConfig(prompts=[_PROMPT], captures={"store": uri}) + assert cfg.captures.store == uri + + +def test_store_is_stripped() -> None: + cfg = EvalShiftConfig(prompts=[_PROMPT], captures={"store": " s3://b/p "}) + assert cfg.captures.store == "s3://b/p" + + +def test_store_rejects_unknown_scheme_naming_forms() -> None: + with pytest.raises(ValidationError, match="accepted forms"): + EvalShiftConfig(prompts=[_PROMPT], captures={"store": "ftp://b/p"}) + + +@pytest.mark.parametrize("uri", ["s3://bucket/prefix?x=1", "s3://user@bucket/p", "az://acct"]) +def test_store_rejects_credentials_and_query(uri: str) -> None: + with pytest.raises(ValidationError, match="accepted forms"): + EvalShiftConfig(prompts=[_PROMPT], captures={"store": uri}) + + +def test_unknown_key_in_captures_is_rejected() -> None: + with pytest.raises(ValidationError, match="Extra inputs are not permitted"): + EvalShiftConfig(prompts=[_PROMPT], captures={"store": "s3://b/p", "bucket": "b"}) + + +def test_loader_reports_store_error_at_its_location(tmp_path: Path) -> None: + path = tmp_path / "evalshift.yaml" + path.write_text( + "prompts:\n - id: p\n detection: manual\n content: hi\n" + "captures:\n store: ftp://bucket/prefix\n", + encoding="utf-8", + ) + with pytest.raises(ConfigError) as info: + load_config(path) + [detail] = info.value.details + assert detail.location == "captures.store" + assert "accepted forms" in detail.message + + +@pytest.mark.parametrize( + ("store", "secrets"), + [ + ("s3://AKIAKEY:SECRET@b/p", ["AKIAKEY", "SECRET"]), + ("az://acct/c?sv=1&sig=SECRETSIG", ["SECRETSIG", "sig="]), + ], +) +def test_loader_error_never_echoes_a_secret_store( + tmp_path: Path, store: str, secrets: list[str] +) -> None: + path = tmp_path / "evalshift.yaml" + path.write_text( + "prompts:\n - id: p\n detection: manual\n content: hi\n" + f"captures:\n store: '{store}'\n", + encoding="utf-8", + ) + with pytest.raises(ConfigError) as info: + load_config(path) + console = Console(record=True, width=200) + console.print(info.value.format_rich()) + rendered = console.export_text() + plain = str(info.value) + assert "accepted forms" in plain + assert "accepted forms" in rendered + for secret in secrets: + assert secret not in plain + assert secret not in rendered From 9028285e6e3d5875a70e79d92e68af3b8ef68f7f Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 19:18:03 +0200 Subject: [PATCH 03/17] feat(captures): incremental fetch from a remote object store into the local mirror Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/__init__.py | 5 +- src/evalshift_cli/captures/remote.py | 326 +++++++++++++++++++++++++ tests/unit/fake_remote_store.py | 110 +++++++++ tests/unit/test_captures_remote.py | 256 +++++++++++++++++++ 4 files changed, 695 insertions(+), 2 deletions(-) create mode 100644 src/evalshift_cli/captures/remote.py create mode 100644 tests/unit/fake_remote_store.py create mode 100644 tests/unit/test_captures_remote.py diff --git a/src/evalshift_cli/captures/__init__.py b/src/evalshift_cli/captures/__init__.py index 227ee506..186c4f3b 100644 --- a/src/evalshift_cli/captures/__init__.py +++ b/src/evalshift_cli/captures/__init__.py @@ -7,8 +7,9 @@ *promotes* a capture into a golden suite case (:mod:`evalshift_cli.captures.promote`) that ``evalshift run`` can evaluate. -The SDK and CLI never call each other — the only contract is the on-disk -capture envelope, frozen at SDK schema version ``1.0.0``. +The SDK and CLI never call each other — the only contract is the capture envelope and its +layout (``captures//.json`` + ``toolsets/.json``), on local disk or +mirrored from an object store by :mod:`evalshift_cli.captures.remote`. """ from __future__ import annotations diff --git a/src/evalshift_cli/captures/remote.py b/src/evalshift_cli/captures/remote.py new file mode 100644 index 00000000..e813cfa9 --- /dev/null +++ b/src/evalshift_cli/captures/remote.py @@ -0,0 +1,326 @@ +"""Mirror captures from a user-owned object store into the local capture directory. + +The SDK's ``ObjectStoreSink`` writes the *local layout* under a bucket prefix -- +``captures//cap_.json`` and ``toolsets/.json`` -- so the CLI never reads a +bucket directly. It fetches what is missing into ``/captures`` and ``/toolsets`` +and every other command runs on the local mirror unchanged. Two properties make that cheap: +captures are immutable (a file that exists locally is current) and toolsets are +content-addressed (same name, same bytes), so no ETag or mtime is ever compared. + +Keys read from a bucket are untrusted. Only basenames shaped like the SDK's are accepted +(:data:`_CAPTURE_NAME`, :data:`_TOOLSET_NAME`), a suite segment must be one the SDK's +``_safe_segment`` could have produced (:func:`_is_safe_suite_segment` -- so ``Support Agent`` +or ``ünïcode`` pass, while ``..``, ``.hidden`` or a backslash do not), and anything else is +counted as malformed and never written, so no key can place a file outside ````. Local +writes are atomic (temp file beside the target, then :func:`os.replace`) so an interrupted +fetch leaves nothing half-written. + +The write side's protocol lives in the SDK (``evalshift.stores``); this module carries the +read side. They share the key layout and URI grammar, not code. +""" + +from __future__ import annotations + +import os +import re +import tempfile +from collections.abc import Collection, Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import suppress +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Protocol, runtime_checkable + +from rich.console import Group, RenderableType +from rich.panel import Panel +from rich.text import Text + +from evalshift_cli.captures.reader import captures_root, toolsets_root + +#: Largest object a fetch will download. Captures are kilobytes; this is a safety stop. +MAX_OBJECT_BYTES = 32 * 1024 * 1024 + +_CAPTURE_NAME = re.compile(r"^[A-Za-z0-9_-]+\.json$") +_TOOLSET_NAME = re.compile(r"^[0-9a-f]{64}\.json$") +_DURATION = re.compile(r"^(\d+)([mhd])$") + + +def _is_safe_suite_segment(segment: str) -> bool: + """Whether ``segment`` is a suite directory name the SDK could have written. + + The SDK's ``_safe_segment`` turns separators into ``_``, collapses every ``..`` and strips + leading and trailing dots and spaces, so it never emits an empty name, a path separator, + ``..`` or an edge dot/space. Anything else came from somewhere other than the SDK and could + steer a write (``..``, ``.``, an empty segment, a Windows ``\\``), so it is refused. NUL is + refused too: no filesystem accepts it. + """ + return ( + bool(segment) + and not any(ch in segment for ch in ("/", "\\", "\x00")) + and ".." not in segment + and segment[0] not in ". " + and segment[-1] not in ". " + ) + + +@dataclass(frozen=True, slots=True) +class ObjectInfo: + """One listed object. + + Attributes: + key: Key relative to the store's prefix, e.g. ``captures//cap_.json``. + last_modified: The provider's timestamp, timezone-aware, or ``None`` if unknown. + size: Size in bytes from the listing, or ``None`` if the provider did not say. + """ + + key: str + last_modified: datetime | None + size: int | None + + +@runtime_checkable +class RemoteStore(Protocol): + """The read side of an object store: list keys under a prefix and fetch one object.""" + + uri: str + + def list(self, prefix: str) -> Iterator[ObjectInfo]: ... + + def get(self, key: str) -> bytes: ... + + +class RemoteStoreError(Exception): + """A remote store could not be opened, listed or read. + + Attributes: + summary: One sentence saying what failed. + hint: What to do about it, when there is something to do. + """ + + def __init__(self, summary: str, *, hint: str | None = None) -> None: + self.summary = summary + self.hint = hint + super().__init__(summary if hint is None else f"{summary} ({hint})") + + def format_rich(self) -> RenderableType: + """Render this error inside a Rich :class:`Panel`.""" + body: list[RenderableType] = [Text(self.summary, style="bold red")] + if self.hint: + body.append(Text(self.hint, style="dim")) + return Panel( + Group(*body), + title="[red]Remote capture store[/red]", + title_align="left", + border_style="red", + ) + + +class RemoteStoreUnavailable(RemoteStoreError): # noqa: N818 — the planned public name. + """The client library for the store's scheme is not installed.""" + + +@dataclass +class FetchSummary: + """What one fetch did, for the one-line console summary.""" + + store_uri: str + captures: int = 0 + toolsets: int = 0 + skipped_existing: int = 0 + skipped_promoted: int = 0 + skipped_old: int = 0 + skipped_large: int = 0 + skipped_malformed: int = 0 + + def describe(self) -> str: + """``fetched N capture(s) and M toolset(s) from (skipped ...)``.""" + text = ( + f"fetched {self.captures} capture(s) and {self.toolsets} toolset(s) " + f"from {self.store_uri}" + ) + skipped = [ + f"{n} {label}" + for n, label in ( + (self.skipped_existing, "already local"), + (self.skipped_promoted, "promoted"), + (self.skipped_old, "older than --since"), + (self.skipped_large, "over the size cap"), + (self.skipped_malformed, "malformed"), + ) + if n + ] + return f"{text} (skipped {', '.join(skipped)})" if skipped else text + + +def parse_since(text: str, *, now: datetime | None = None) -> datetime: + """Turn a ``--since`` value into an aware UTC instant. + + Accepts a duration (``30m``, ``24h``, ``7d``) relative to ``now`` (UTC now by default) or + an ISO date / datetime; a naive datetime is taken as UTC. + + Raises: + ValueError: when ``text`` is neither form. + """ + reference = now if now is not None else datetime.now(UTC) + cleaned = text.strip() + match = _DURATION.match(cleaned) + if match: + amount, unit = int(match.group(1)), match.group(2) + delta = { + "m": timedelta(minutes=amount), + "h": timedelta(hours=amount), + "d": timedelta(days=amount), + } + return reference - delta[unit] + try: + parsed = datetime.fromisoformat(cleaned) + except ValueError as exc: + raise ValueError( + f"--since expects a duration like 30m, 24h, 7d or an ISO date/datetime, got {text!r}" + ) from exc + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC) + + +def fetch_captures( + store: RemoteStore, + *, + base: Path | None = None, + suite: str | None = None, + since: datetime | None = None, + skip_ids: Collection[str] = frozenset(), + workers: int = 8, +) -> FetchSummary: + """Mirror missing toolsets and captures from ``store`` into ````. + + Toolsets are fetched first so a capture never lands locally before a sidecar that exists + remotely. A capture is skipped when its file already exists, when its id is in + ``skip_ids`` (the promoted ids -- so a capture ``capture clean`` removed is not pulled + back), when ``since`` is set and the object is older, or when it is over + :data:`MAX_OBJECT_BYTES`. Keys that do not match the SDK's layout are counted as + malformed and ignored. + + Raises: + RemoteStoreError: when listing or any download fails. Files already written stay, so a + re-run resumes; nothing half-written is left behind. + """ + summary = FetchSummary(store_uri=store.uri) + skip = set(skip_ids) + + toolset_jobs: list[tuple[str, Path]] = [] + for info in _list(store, "toolsets/"): + name = info.key.removeprefix("toolsets/") + if "/" in name or not _TOOLSET_NAME.match(name): + summary.skipped_malformed += 1 + continue + dest = toolsets_root(base) / name + if dest.exists(): + continue # content-addressed: present means correct; not worth a counter + if info.size is not None and info.size > MAX_OBJECT_BYTES: + summary.skipped_large += 1 + continue + toolset_jobs.append((info.key, dest)) + + capture_jobs: list[tuple[str, Path]] = [] + prefix = f"captures/{suite}/" if suite else "captures/" + for info in _list(store, prefix): + rel = info.key.removeprefix("captures/") + suite_seg, sep, name = rel.partition("/") + if ( + not sep + or "/" in name + or not _is_safe_suite_segment(suite_seg) + or not _CAPTURE_NAME.match(name) + ): + summary.skipped_malformed += 1 + continue + dest = captures_root(base) / suite_seg / name + if dest.exists(): + summary.skipped_existing += 1 + continue + if name[: -len(".json")] in skip: + summary.skipped_promoted += 1 + continue + if since is not None and info.last_modified is not None and info.last_modified < since: + summary.skipped_old += 1 + continue + if info.size is not None and info.size > MAX_OBJECT_BYTES: + summary.skipped_large += 1 + continue + capture_jobs.append((info.key, dest)) + + summary.toolsets, large = _download_all(store, toolset_jobs, workers) + summary.skipped_large += large + summary.captures, large = _download_all(store, capture_jobs, workers) + summary.skipped_large += large + return summary + + +def _list(store: RemoteStore, prefix: str) -> list[ObjectInfo]: + try: + return list(store.list(prefix)) + except RemoteStoreError: + raise + except Exception as exc: + raise RemoteStoreError( + f"could not list {prefix!r} in {store.uri}: {type(exc).__name__}: {exc}", + hint="check credentials and network, or use --offline to work with the captures " + "already on disk", + ) from exc + + +def _download_all( + store: RemoteStore, jobs: list[tuple[str, Path]], workers: int +) -> tuple[int, int]: + """Download every job; return ``(written, skipped_as_too_large)``.""" + if not jobs: + return 0, 0 + try: + with ThreadPoolExecutor(max_workers=max(1, min(workers, len(jobs)))) as pool: + results = list(pool.map(lambda job: _download(store, job[0], job[1]), jobs)) + except RemoteStoreError: + raise + except Exception as exc: + raise RemoteStoreError( + f"could not download from {store.uri}: {type(exc).__name__}: {exc}", + hint="check credentials and network, or use --offline to work with the captures " + "already on disk", + ) from exc + written = sum(1 for ok in results if ok) + return written, len(results) - written + + +def _download(store: RemoteStore, key: str, dest: Path) -> bool: + """Fetch ``key`` into ``dest`` atomically. ``False`` when the object is over the cap.""" + try: + data = store.get(key) + except Exception as exc: + raise RemoteStoreError( + f"could not download {key} from {store.uri}: {type(exc).__name__}: {exc}", + hint="check credentials and network, or use --offline to work with the captures " + "already on disk", + ) from exc + if len(data) > MAX_OBJECT_BYTES: + return False + dest.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(dir=dest.parent, prefix=".", suffix=".tmp") + try: + with os.fdopen(fd, "wb") as handle: + handle.write(data) + os.replace(tmp_name, dest) + finally: + with suppress(OSError): + os.remove(tmp_name) + return True + + +__all__ = [ + "MAX_OBJECT_BYTES", + "FetchSummary", + "ObjectInfo", + "RemoteStore", + "RemoteStoreError", + "RemoteStoreUnavailable", + "fetch_captures", + "parse_since", +] diff --git a/tests/unit/fake_remote_store.py b/tests/unit/fake_remote_store.py new file mode 100644 index 00000000..067d7cb1 --- /dev/null +++ b/tests/unit/fake_remote_store.py @@ -0,0 +1,110 @@ +"""An in-memory ``RemoteStore`` and a valid-capture builder for the remote-fetch tests.""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from datetime import UTC, datetime +from typing import Any + +from evalshift_cli.captures.remote import ObjectInfo + +TOOLSET_HEX = "ab" * 32 +TOOLSET_REF = f"sha256:{TOOLSET_HEX}" +TOOLSET_KEY = f"toolsets/{TOOLSET_HEX}.json" +TOOLSET_PAYLOAD = b'{"tools": []}' + + +def capture_payload( + capture_id: str, + suite: str = "alpha", + *, + model_input: Any = "where is order 12345?", + created_at: str = "2026-06-16T12:00:00+00:00", +) -> bytes: + """A minimal capture the CLI reader accepts: one model call, one final output.""" + envelope = { + "schema_version": "2.0.0", + "capture_id": capture_id, + "suite": suite, + "input_hash": f"hash-{capture_id}", + "code_version": "v1", + "created_at": created_at, + "trace": { + "run_id": capture_id, + "prompt_id": suite, + "example_id": capture_id, + "role": "source", + "events": [ + { + "type": "model_call", + "sequence_index": 0, + "timestamp": created_at, + "metadata": {}, + "model_id": "m", + "input": model_input, + "output": "out", + "toolset_ref": TOOLSET_REF, + "tools_offered": [], + }, + { + "type": "final_output", + "sequence_index": 1, + "timestamp": created_at, + "metadata": {}, + "text": "done", + }, + ], + }, + } + return json.dumps(envelope).encode("utf-8") + + +class FakeRemoteStore: + """Dict-backed store: ``objects`` maps relative key -> bytes; ``fail_keys`` raise on get.""" + + uri = "s3://fake/prefix" + + def __init__( + self, + objects: dict[str, bytes] | None = None, + *, + modified: dict[str, datetime | None] | None = None, + sizes: dict[str, int | None] | None = None, + fail_keys: set[str] | None = None, + ) -> None: + self.objects: dict[str, bytes] = dict(objects or {}) + # A key present with value None means "the provider reported nothing" (never filtered). + self.modified: dict[str, datetime | None] = dict(modified or {}) + self.sizes: dict[str, int | None] = dict(sizes or {}) + self.fail_keys = set(fail_keys or ()) + self.gets: list[str] = [] + + def list(self, prefix: str) -> Iterator[ObjectInfo]: + for key in sorted(self.objects): + if key.startswith(prefix): + yield ObjectInfo( + key=key, + last_modified=self.modified.get(key, datetime(2026, 6, 16, tzinfo=UTC)), + size=self.sizes.get(key, len(self.objects[key])), + ) + + def get(self, key: str) -> bytes: + self.gets.append(key) + if key in self.fail_keys: + raise OSError(f"simulated failure reading {key}") + return self.objects[key] + + +def populated_store(*capture_ids: str, suite: str = "alpha") -> FakeRemoteStore: + """A store holding the shared toolset sidecar plus one capture per id. + + Each capture gets a distinct model input: ``capture sync`` drops content duplicates, so two + captures with the same input would promote as one and hide a fetch bug behind a dedup. + """ + objects = {TOOLSET_KEY: TOOLSET_PAYLOAD} + for cid in capture_ids: + objects[f"captures/{suite}/{cid}.json"] = capture_payload( + cid, suite, model_input=f"where is order {cid}?" + ) + return FakeRemoteStore(objects) diff --git a/tests/unit/test_captures_remote.py b/tests/unit/test_captures_remote.py new file mode 100644 index 00000000..284df0e0 --- /dev/null +++ b/tests/unit/test_captures_remote.py @@ -0,0 +1,256 @@ +"""Incremental fetch from a RemoteStore into the local capture directory.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from evalshift_cli.captures import remote +from evalshift_cli.captures.remote import ( + MAX_OBJECT_BYTES, + FetchSummary, + RemoteStore, + RemoteStoreError, + _is_safe_suite_segment, + fetch_captures, + parse_since, +) +from tests.unit.fake_remote_store import ( + TOOLSET_HEX, + TOOLSET_KEY, + TOOLSET_PAYLOAD, + FakeRemoteStore, + capture_payload, + populated_store, +) + + +def _local_captures(base: Path) -> set[str]: + root = base / "captures" + return {str(p.relative_to(root)) for p in root.rglob("*.json")} if root.exists() else set() + + +def test_fake_satisfies_protocol() -> None: + assert isinstance(FakeRemoteStore(), RemoteStore) + + +def test_fetch_downloads_toolsets_then_captures(tmp_path: Path) -> None: + store = populated_store("cap_1", "cap_2") + summary = fetch_captures(store, base=tmp_path) + assert summary.captures == 2 + assert summary.toolsets == 1 + assert (tmp_path / "toolsets" / f"{TOOLSET_HEX}.json").read_bytes() == TOOLSET_PAYLOAD + assert _local_captures(tmp_path) == {"alpha/cap_1.json", "alpha/cap_2.json"} + assert store.gets[0] == TOOLSET_KEY # sidecars land before the captures that need them + + +def test_fetch_skips_objects_already_local(tmp_path: Path) -> None: + store = populated_store("cap_1", "cap_2") + fetch_captures(store, base=tmp_path) + store.gets.clear() + summary = fetch_captures(store, base=tmp_path) + assert summary.captures == 0 + assert summary.skipped_existing == 2 + assert store.gets == [] # nothing re-downloaded, toolset included + + +def test_fetch_skips_promoted_ids(tmp_path: Path) -> None: + store = populated_store("cap_1", "cap_2") + summary = fetch_captures(store, base=tmp_path, skip_ids={"cap_1"}) + assert summary.captures == 1 + assert summary.skipped_promoted == 1 + assert _local_captures(tmp_path) == {"alpha/cap_2.json"} + + +def test_fetch_filters_by_suite(tmp_path: Path) -> None: + store = populated_store("cap_a") + store.objects["captures/beta/cap_b.json"] = capture_payload("cap_b", "beta") + summary = fetch_captures(store, base=tmp_path, suite="beta") + assert summary.captures == 1 + assert _local_captures(tmp_path) == {"beta/cap_b.json"} + + +def test_fetch_since_filters_on_last_modified(tmp_path: Path) -> None: + now = datetime(2026, 10, 6, 12, 0, tzinfo=UTC) + store = populated_store("cap_old", "cap_new", "cap_unknown") + store.modified["captures/alpha/cap_old.json"] = now - timedelta(days=3) + store.modified["captures/alpha/cap_new.json"] = now - timedelta(hours=1) + store.modified["captures/alpha/cap_unknown.json"] = None + summary = fetch_captures(store, base=tmp_path, since=now - timedelta(days=1)) + assert summary.skipped_old == 1 + # Unknown last_modified is never filtered out. + assert _local_captures(tmp_path) == {"alpha/cap_new.json", "alpha/cap_unknown.json"} + + +def test_fetch_rejects_malformed_and_traversal_keys(tmp_path: Path) -> None: + store = FakeRemoteStore( + { + "captures/../../escape.json": b"{}", + "captures/alpha/evil.sh": b"#!", + "captures/alpha/nested/cap_x.json": b"{}", + "captures/alpha": b"{}", + "captures/.hidden/cap_y.json": b"{}", + "toolsets/not-a-hash.json": b"{}", + "toolsets/sub/" + "ab" * 32 + ".json": b"{}", + "README.md": b"ignored: not under captures/ or toolsets/", + } + ) + summary = fetch_captures(store, base=tmp_path) + assert summary.captures == 0 + assert summary.toolsets == 0 + assert summary.skipped_malformed == 7 + assert not (tmp_path / "captures").exists() + assert not (tmp_path / "toolsets").exists() + assert not (tmp_path.parent / "escape.json").exists() + assert store.gets == [] + + +@pytest.mark.parametrize( + "key", + [ + "captures/../cap_z.json", # would land in /cap_z.json, outside captures/ + "captures/./cap_z.json", + "captures//cap_z.json", # empty suite: would land directly in captures/ + "captures/a\\b/cap_z.json", # a path separator on Windows + "captures/a..b/cap_z.json", + "captures/alpha./cap_z.json", + "captures/ alpha/cap_z.json", + ], +) +def test_fetch_rejects_suite_segments_the_sdk_never_writes(tmp_path: Path, key: str) -> None: + # The capture basename is valid here, so only the suite-segment check stands in the way. + store = FakeRemoteStore({key: capture_payload("cap_z")}) + summary = fetch_captures(store, base=tmp_path) + assert summary.skipped_malformed == 1 + assert summary.captures == 0 + assert [p for p in tmp_path.rglob("*") if p.is_file()] == [] + assert not (tmp_path.parent / "cap_z.json").exists() + assert store.gets == [] + + +def test_fetch_accepts_every_suite_name_the_sdk_writes(tmp_path: Path) -> None: + # The SDK names suite directories with ``_safe_segment``, which keeps spaces, leading + # underscores and non-ASCII letters; all of them must mirror to the same local directory. + suites = ("Support Agent", "_internal", "ünïcode") + store = FakeRemoteStore( + { + f"captures/{suite}/cap_{i}.json": capture_payload( + f"cap_{i}", suite, model_input=f"input {i}" + ) + for i, suite in enumerate(suites) + } + ) + summary = fetch_captures(store, base=tmp_path) + assert summary.captures == 3 + assert summary.skipped_malformed == 0 + assert _local_captures(tmp_path) == {f"{suite}/cap_{i}.json" for i, suite in enumerate(suites)} + + +@pytest.mark.parametrize( + ("segment", "safe"), + [ + ("alpha", True), + ("Support Agent", True), + ("_internal", True), + ("ünïcode", True), + ("v1.2", True), + ("", False), + ("a/b", False), + ("a\\b", False), + ("a\x00b", False), + ("..", False), + ("a..b", False), + (".hidden", False), + ("alpha.", False), + (" alpha", False), + ("alpha ", False), + ], +) +def test_is_safe_suite_segment(segment: str, safe: bool) -> None: + assert _is_safe_suite_segment(segment) is safe + + +def test_fetch_skips_too_large_objects(tmp_path: Path) -> None: + store = populated_store("cap_big", "cap_ok") + store.sizes["captures/alpha/cap_big.json"] = MAX_OBJECT_BYTES + 1 + summary = fetch_captures(store, base=tmp_path) + assert summary.skipped_large == 1 + assert _local_captures(tmp_path) == {"alpha/cap_ok.json"} + + +def test_fetch_checks_size_after_download_when_listing_has_none(tmp_path: Path) -> None: + store = populated_store("cap_1") + store.sizes["captures/alpha/cap_1.json"] = None + store.objects["captures/alpha/cap_1.json"] = b"x" * (MAX_OBJECT_BYTES + 1) + summary = fetch_captures(store, base=tmp_path) + assert summary.skipped_large == 1 + assert _local_captures(tmp_path) == set() + + +def test_fetch_failure_leaves_no_temp_files(tmp_path: Path) -> None: + store = populated_store("cap_1", "cap_2") + store.fail_keys.add("captures/alpha/cap_2.json") + with pytest.raises(RemoteStoreError) as info: + fetch_captures(store, base=tmp_path, workers=1) + assert "cap_2.json" in info.value.summary + assert info.value.hint is not None and "--offline" in info.value.hint + leftovers = [p for p in tmp_path.rglob("*") if p.is_file() and p.suffix == ".tmp"] + assert leftovers == [] + # Already-fetched files stay: a re-run resumes. + assert (tmp_path / "toolsets" / f"{TOOLSET_HEX}.json").exists() + + +def test_fetch_write_failure_leaves_no_temp_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The get succeeds and the temp file is written; the rename into place then fails. + def failing_replace(src: str, dst: str) -> None: + raise OSError("simulated rename failure") + + monkeypatch.setattr(remote.os, "replace", failing_replace) + store = populated_store("cap_1") + with pytest.raises(RemoteStoreError) as info: + fetch_captures(store, base=tmp_path, workers=1) + assert "simulated rename failure" in info.value.summary + assert [p for p in tmp_path.rglob("*") if p.is_file()] == [] + + +def test_summary_describe() -> None: + full = FetchSummary( + store_uri="s3://fake/prefix", captures=2, toolsets=1, skipped_existing=3, skipped_promoted=1 + ) + assert full.describe() == ( + "fetched 2 capture(s) and 1 toolset(s) from s3://fake/prefix " + "(skipped 3 already local, 1 promoted)" + ) + assert FetchSummary(store_uri="gs://b").describe() == ( + "fetched 0 capture(s) and 0 toolset(s) from gs://b" + ) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("30m", timedelta(minutes=30)), + ("24h", timedelta(hours=24)), + ("7d", timedelta(days=7)), + (" 2d ", timedelta(days=2)), + ], +) +def test_parse_since_durations(text: str, expected: timedelta) -> None: + now = datetime(2026, 10, 6, 12, 0, tzinfo=UTC) + assert parse_since(text, now=now) == now - expected + + +def test_parse_since_iso_date_and_datetime() -> None: + assert parse_since("2026-10-01") == datetime(2026, 10, 1, tzinfo=UTC) + assert parse_since("2026-10-01T08:30:00+02:00") == datetime(2026, 10, 1, 6, 30, tzinfo=UTC) + assert parse_since("2026-10-01T08:30:00").tzinfo is UTC + + +@pytest.mark.parametrize("text", ["yesterday", "24", "h", "7w", ""]) +def test_parse_since_rejects_garbage(text: str) -> None: + with pytest.raises(ValueError, match="--since expects"): + parse_since(text) From 58725c2a4ec5df3da1bb5c96d6e8327f814de0b4 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 19:22:53 +0200 Subject: [PATCH 04/17] fix(captures): refuse colon suite segments and contain fetch destinations On Windows a suite segment like `D:` or `a:b` joins as a drive-relative path or an alternate data stream and escapes . The suite predicate now refuses `:`, and every destination must also normalize inside its root before a download is queued. Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/remote.py | 37 ++++++++++++++++++++++------ tests/unit/test_captures_remote.py | 19 ++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/evalshift_cli/captures/remote.py b/src/evalshift_cli/captures/remote.py index e813cfa9..90b45cc1 100644 --- a/src/evalshift_cli/captures/remote.py +++ b/src/evalshift_cli/captures/remote.py @@ -10,10 +10,11 @@ Keys read from a bucket are untrusted. Only basenames shaped like the SDK's are accepted (:data:`_CAPTURE_NAME`, :data:`_TOOLSET_NAME`), a suite segment must be one the SDK's ``_safe_segment`` could have produced (:func:`_is_safe_suite_segment` -- so ``Support Agent`` -or ``ünïcode`` pass, while ``..``, ``.hidden`` or a backslash do not), and anything else is -counted as malformed and never written, so no key can place a file outside ````. Local -writes are atomic (temp file beside the target, then :func:`os.replace`) so an interrupted -fetch leaves nothing half-written. +or ``ünïcode`` pass, while ``..``, ``.hidden``, a backslash or a ``:`` do not), and anything +else is counted as malformed and never written. As a second line of defence every destination +must also normalize inside its root (:func:`_absolute`), so no key can place a file outside +````. Local writes are atomic (temp file beside the target, then :func:`os.replace`) so +an interrupted fetch leaves nothing half-written. The write side's protocol lives in the SDK (``evalshift.stores``); this module carries the read side. They share the key layout and URI grammar, not code. @@ -54,10 +55,15 @@ def _is_safe_suite_segment(segment: str) -> bool: ``..`` or an edge dot/space. Anything else came from somewhere other than the SDK and could steer a write (``..``, ``.``, an empty segment, a Windows ``\\``), so it is refused. NUL is refused too: no filesystem accepts it. + + ``:`` is the one deliberate narrowing against ``_safe_segment``, which keeps it: on Windows + ``D:`` joins as a drive-relative path and ``a:b`` names a drive or an NTFS alternate data + stream, either of which lands outside ````. A suite named with a colon is mirrored + nowhere and counted as malformed. """ return ( bool(segment) - and not any(ch in segment for ch in ("/", "\\", "\x00")) + and not any(ch in segment for ch in ("/", "\\", ":", "\x00")) and ".." not in segment and segment[0] not in ". " and segment[-1] not in ". " @@ -206,6 +212,8 @@ def fetch_captures( """ summary = FetchSummary(store_uri=store.uri) skip = set(skip_ids) + toolsets_dir = _absolute(toolsets_root(base)) + captures_dir = _absolute(captures_root(base)) toolset_jobs: list[tuple[str, Path]] = [] for info in _list(store, "toolsets/"): @@ -213,7 +221,10 @@ def fetch_captures( if "/" in name or not _TOOLSET_NAME.match(name): summary.skipped_malformed += 1 continue - dest = toolsets_root(base) / name + dest = toolsets_dir / name + if not _absolute(dest.parent).is_relative_to(toolsets_dir): + summary.skipped_malformed += 1 + continue if dest.exists(): continue # content-addressed: present means correct; not worth a counter if info.size is not None and info.size > MAX_OBJECT_BYTES: @@ -234,7 +245,10 @@ def fetch_captures( ): summary.skipped_malformed += 1 continue - dest = captures_root(base) / suite_seg / name + dest = captures_dir / suite_seg / name + if not _absolute(dest.parent).is_relative_to(captures_dir): + summary.skipped_malformed += 1 + continue if dest.exists(): summary.skipped_existing += 1 continue @@ -256,6 +270,15 @@ def fetch_captures( return summary +def _absolute(path: Path) -> Path: + """``path`` made absolute and normalized lexically (``..`` collapsed, drive applied). + + Belt and braces behind the key checks: a destination whose parent normalizes outside its + root is refused. Lexical, so it works before the directory exists and costs no I/O. + """ + return Path(os.path.abspath(path)) + + def _list(store: RemoteStore, prefix: str) -> list[ObjectInfo]: try: return list(store.list(prefix)) diff --git a/tests/unit/test_captures_remote.py b/tests/unit/test_captures_remote.py index 284df0e0..975d5070 100644 --- a/tests/unit/test_captures_remote.py +++ b/tests/unit/test_captures_remote.py @@ -114,6 +114,8 @@ def test_fetch_rejects_malformed_and_traversal_keys(tmp_path: Path) -> None: "captures/./cap_z.json", "captures//cap_z.json", # empty suite: would land directly in captures/ "captures/a\\b/cap_z.json", # a path separator on Windows + "captures/D:/cap_z.json", # a drive-relative path on Windows + "captures/a:b/cap_z.json", # a drive or NTFS alternate data stream on Windows "captures/a..b/cap_z.json", "captures/alpha./cap_z.json", "captures/ alpha/cap_z.json", @@ -130,6 +132,21 @@ def test_fetch_rejects_suite_segments_the_sdk_never_writes(tmp_path: Path, key: assert store.gets == [] +def test_fetch_containment_check_rejects_escaping_destinations( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # With the suite predicate disabled, the containment check alone must stop a key whose + # destination resolves outside ``/captures``. + monkeypatch.setattr(remote, "_is_safe_suite_segment", lambda segment: True) + base = tmp_path / "base" + store = FakeRemoteStore({"captures/../cap_z.json": capture_payload("cap_z")}) + summary = fetch_captures(store, base=base) + assert summary.skipped_malformed == 1 + assert summary.captures == 0 + assert [p for p in tmp_path.rglob("*") if p.is_file()] == [] + assert store.gets == [] + + def test_fetch_accepts_every_suite_name_the_sdk_writes(tmp_path: Path) -> None: # The SDK names suite directories with ``_safe_segment``, which keeps spaces, leading # underscores and non-ASCII letters; all of them must mirror to the same local directory. @@ -160,6 +177,8 @@ def test_fetch_accepts_every_suite_name_the_sdk_writes(tmp_path: Path) -> None: ("a/b", False), ("a\\b", False), ("a\x00b", False), + ("D:", False), + ("a:b", False), ("..", False), ("a..b", False), (".hidden", False), From c57ce21a2861d628dd38b541c16acdd5fb7f8b6b Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 19:26:52 +0200 Subject: [PATCH 05/17] feat(captures): S3, GCS and Azure Blob read adapters behind optional extras; open_store Co-Authored-By: Claude Opus 5.5 --- mypy.ini | 15 ++ pyproject.toml | 12 + src/evalshift_cli/captures/remote.py | 50 ++++ src/evalshift_cli/captures/stores/__init__.py | 22 ++ src/evalshift_cli/captures/stores/azure.py | 60 +++++ src/evalshift_cli/captures/stores/gcs.py | 53 +++++ src/evalshift_cli/captures/stores/s3.py | 56 +++++ tests/unit/test_captures_stores.py | 219 ++++++++++++++++++ 8 files changed, 487 insertions(+) create mode 100644 src/evalshift_cli/captures/stores/__init__.py create mode 100644 src/evalshift_cli/captures/stores/azure.py create mode 100644 src/evalshift_cli/captures/stores/gcs.py create mode 100644 src/evalshift_cli/captures/stores/s3.py create mode 100644 tests/unit/test_captures_stores.py diff --git a/mypy.ini b/mypy.ini index b8775e34..a3368c9e 100644 --- a/mypy.ini +++ b/mypy.ini @@ -31,6 +31,21 @@ ignore_missing_imports = true [mypy-scipy.*] ignore_missing_imports = true +[mypy-boto3] +ignore_missing_imports = true + +[mypy-boto3.*] +ignore_missing_imports = true + +[mypy-botocore.*] +ignore_missing_imports = true + +[mypy-google.*] +ignore_missing_imports = true + +[mypy-azure.*] +ignore_missing_imports = true + [pydantic-mypy] init_forbid_extra = true init_typed = true diff --git a/pyproject.toml b/pyproject.toml index ed93aaa6..3c324497 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,11 @@ dependencies = [ ] [project.optional-dependencies] +# Remote capture stores (captures.store in evalshift.yaml). Each adapter imports its client +# lazily, so the CLI's default install pulls none of these. +s3 = ["boto3>=1.34"] +gcs = ["google-cloud-storage>=2.14"] +azure = ["azure-storage-blob>=12.19", "azure-identity>=1.15"] dev = [ "pytest>=8.0", "pytest-asyncio>=0.23", @@ -65,6 +70,13 @@ dev = [ "types-PyYAML>=6.0", "types-jsonschema>=4.22", "respx>=0.21", + # Remote-store adapters: the adapter tests drive stub clients, so no bucket is touched. + # These are here so mypy sees the adapters' real imports and the open_store dispatch + # test, which checks every module an extra provides, finds them installed. + "boto3>=1.34", + "google-cloud-storage>=2.14", + "azure-storage-blob>=12.19", + "azure-identity>=1.15", ] [project.urls] diff --git a/src/evalshift_cli/captures/remote.py b/src/evalshift_cli/captures/remote.py index 90b45cc1..aab3b51a 100644 --- a/src/evalshift_cli/captures/remote.py +++ b/src/evalshift_cli/captures/remote.py @@ -22,6 +22,7 @@ from __future__ import annotations +import importlib.util import os import re import tempfile @@ -38,6 +39,7 @@ from rich.text import Text from evalshift_cli.captures.reader import captures_root, toolsets_root +from evalshift_cli.captures.store_uri import StoreURI #: Largest object a fetch will download. Captures are kilobytes; this is a safety stop. MAX_OBJECT_BYTES = 32 * 1024 * 1024 @@ -46,6 +48,13 @@ _TOOLSET_NAME = re.compile(r"^[0-9a-f]{64}\.json$") _DURATION = re.compile(r"^(\d+)([mhd])$") +#: Every module a scheme's pip extra provides; ``open_store`` checks all of them. +_REQUIRED_MODULES: dict[str, tuple[str, ...]] = { + "s3": ("boto3",), + "gs": ("google.cloud.storage",), + "az": ("azure.storage.blob", "azure.identity"), +} + def _is_safe_suite_segment(segment: str) -> bool: """Whether ``segment`` is a suite directory name the SDK could have written. @@ -126,6 +135,46 @@ class RemoteStoreUnavailable(RemoteStoreError): # noqa: N818 — the planned pu """The client library for the store's scheme is not installed.""" +def _installed(module: str) -> bool: + """Whether ``module`` can be imported, without importing it.""" + try: + return importlib.util.find_spec(module) is not None + except ModuleNotFoundError: # a parent package is missing (no `google` at all) + return False + + +def open_store(parsed: StoreURI) -> RemoteStore: + """Return the adapter for ``parsed`` without building its client yet. + + Args: + parsed: A URI from :func:`evalshift_cli.captures.store_uri.parse_store_uri`. + + Returns: + The scheme's adapter; its client is built on the first ``list`` or ``get``. + + Raises: + RemoteStoreUnavailable: when a module the scheme's extra provides is not installed. + The summary names the first missing module; the hint names the pip extra. + """ + for module in _REQUIRED_MODULES[parsed.scheme]: + if not _installed(module): + raise RemoteStoreUnavailable( + f"{parsed.scheme}:// captures store needs the optional dependency {module!r}", + hint=f'install it with: pip install "evalshift[{parsed.extra}]"', + ) + if parsed.scheme == "s3": + from evalshift_cli.captures.stores.s3 import S3Store + + return S3Store(parsed.bucket, parsed.prefix) + if parsed.scheme == "gs": + from evalshift_cli.captures.stores.gcs import GCSStore + + return GCSStore(parsed.bucket, parsed.prefix) + from evalshift_cli.captures.stores.azure import AzureBlobStore + + return AzureBlobStore(parsed.bucket, parsed.container or "", parsed.prefix) + + @dataclass class FetchSummary: """What one fetch did, for the one-line console summary.""" @@ -345,5 +394,6 @@ def _download(store: RemoteStore, key: str, dest: Path) -> bool: "RemoteStoreError", "RemoteStoreUnavailable", "fetch_captures", + "open_store", "parse_since", ] diff --git a/src/evalshift_cli/captures/stores/__init__.py b/src/evalshift_cli/captures/stores/__init__.py new file mode 100644 index 00000000..ae15f48e --- /dev/null +++ b/src/evalshift_cli/captures/stores/__init__.py @@ -0,0 +1,22 @@ +"""Read-side adapters for the object stores ``captures.store`` can name. + +Each module imports its client library lazily, inside the method that needs it, so importing +this package never requires a cloud SDK; :func:`evalshift_cli.captures.remote.open_store` picks +the adapter from the URI scheme and reports a missing extra before any import happens. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def aware_utc(value: datetime | None) -> datetime | None: + """``value`` with a naive timestamp taken as UTC; aware values and ``None`` pass through. + + boto3, google-cloud-storage and azure-storage-blob all return aware datetimes today; this + keeps ``--since`` comparisons from raising if one ever hands back a naive one. + """ + return value.replace(tzinfo=UTC) if value is not None and value.tzinfo is None else value + + +__all__ = ["aware_utc"] diff --git a/src/evalshift_cli/captures/stores/azure.py b/src/evalshift_cli/captures/stores/azure.py new file mode 100644 index 00000000..956f7fa3 --- /dev/null +++ b/src/evalshift_cli/captures/stores/azure.py @@ -0,0 +1,60 @@ +"""Azure Blob Storage read adapter. Extra: ``evalshift[azure]``. + +The account URL is ``https://.blob.core.windows.net``; credentials come from +``DefaultAzureCredential``. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +from evalshift_cli.captures.remote import ObjectInfo +from evalshift_cli.captures.stores import aware_utc + + +class AzureBlobStore: + """List and read blobs under ``az:////``.""" + + def __init__( + self, account: str, container: str, prefix: str = "", *, client: Any | None = None + ) -> None: + self.account = account + self.container = container + self.prefix = prefix.strip("/") + self.account_url = f"https://{account}.blob.core.windows.net" + self.uri = f"az://{account}/{container}/{self.prefix}".rstrip("/") + self._client = client + + def _full(self, key: str) -> str: + return f"{self.prefix}/{key}" if self.prefix else key + + def _relative(self, full_key: str) -> str: + return full_key.removeprefix(f"{self.prefix}/") if self.prefix else full_key + + def _get_client(self) -> Any: + if self._client is None: + from azure.identity import DefaultAzureCredential # lazy: [azure] extra + from azure.storage.blob import BlobServiceClient + + self._client = BlobServiceClient(self.account_url, credential=DefaultAzureCredential()) + return self._client + + def list(self, prefix: str) -> Iterator[ObjectInfo]: + """Yield every blob whose name starts with ``prefix`` (relative to the store prefix).""" + container = self._get_client().get_container_client(self.container) + for blob in container.list_blobs(name_starts_with=self._full(prefix)): + yield ObjectInfo( + key=self._relative(blob.name), + last_modified=aware_utc(blob.last_modified), + size=blob.size, + ) + + def get(self, key: str) -> bytes: + """Return the bytes of ``key`` (relative to the store prefix).""" + blob = self._get_client().get_blob_client(self.container, self._full(key)) + data: bytes = blob.download_blob().readall() + return data + + +__all__ = ["AzureBlobStore"] diff --git a/src/evalshift_cli/captures/stores/gcs.py b/src/evalshift_cli/captures/stores/gcs.py new file mode 100644 index 00000000..9f530941 --- /dev/null +++ b/src/evalshift_cli/captures/stores/gcs.py @@ -0,0 +1,53 @@ +"""Google Cloud Storage read adapter. Extra: ``evalshift[gcs]``. + +Credentials come from Application Default Credentials. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +from evalshift_cli.captures.remote import ObjectInfo +from evalshift_cli.captures.stores import aware_utc + + +class GCSStore: + """List and read objects under ``gs:///``.""" + + def __init__(self, bucket: str, prefix: str = "", *, client: Any | None = None) -> None: + self.bucket = bucket + self.prefix = prefix.strip("/") + self.uri = f"gs://{bucket}/{self.prefix}".rstrip("/") + self._client = client + + def _full(self, key: str) -> str: + return f"{self.prefix}/{key}" if self.prefix else key + + def _relative(self, full_key: str) -> str: + return full_key.removeprefix(f"{self.prefix}/") if self.prefix else full_key + + def _get_client(self) -> Any: + if self._client is None: + from google.cloud import storage # lazy: the [gcs] extra is optional + + self._client = storage.Client() + return self._client + + def list(self, prefix: str) -> Iterator[ObjectInfo]: + """Yield every object whose key starts with ``prefix`` (relative to the store prefix).""" + for blob in self._get_client().list_blobs(self.bucket, prefix=self._full(prefix)): + yield ObjectInfo( + key=self._relative(blob.name), + last_modified=aware_utc(blob.updated), + size=blob.size, + ) + + def get(self, key: str) -> bytes: + """Return the bytes of ``key`` (relative to the store prefix).""" + blob = self._get_client().bucket(self.bucket).blob(self._full(key)) + data: bytes = blob.download_as_bytes() + return data + + +__all__ = ["GCSStore"] diff --git a/src/evalshift_cli/captures/stores/s3.py b/src/evalshift_cli/captures/stores/s3.py new file mode 100644 index 00000000..7469db62 --- /dev/null +++ b/src/evalshift_cli/captures/stores/s3.py @@ -0,0 +1,56 @@ +"""Amazon S3 (and S3-compatible) read adapter. Extra: ``evalshift[s3]``. + +boto3 honours ``AWS_ENDPOINT_URL``, which is how MinIO, Cloudflare R2, Backblaze B2 and Ceph +are reached with no endpoint knob here. Credentials come from boto3's default chain. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +from evalshift_cli.captures.remote import ObjectInfo +from evalshift_cli.captures.stores import aware_utc + + +class S3Store: + """List and read objects under ``s3:///``.""" + + def __init__(self, bucket: str, prefix: str = "", *, client: Any | None = None) -> None: + self.bucket = bucket + self.prefix = prefix.strip("/") + self.uri = f"s3://{bucket}/{self.prefix}".rstrip("/") + self._client = client + + def _full(self, key: str) -> str: + return f"{self.prefix}/{key}" if self.prefix else key + + def _relative(self, full_key: str) -> str: + return full_key.removeprefix(f"{self.prefix}/") if self.prefix else full_key + + def _get_client(self) -> Any: + if self._client is None: + import boto3 # lazy: the [s3] extra is optional + + self._client = boto3.client("s3") + return self._client + + def list(self, prefix: str) -> Iterator[ObjectInfo]: + """Yield every object whose key starts with ``prefix`` (relative to the store prefix).""" + paginator = self._get_client().get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self.bucket, Prefix=self._full(prefix)): + for obj in page.get("Contents", []): + yield ObjectInfo( + key=self._relative(obj["Key"]), + last_modified=aware_utc(obj.get("LastModified")), + size=obj.get("Size"), + ) + + def get(self, key: str) -> bytes: + """Return the bytes of ``key`` (relative to the store prefix).""" + body = self._get_client().get_object(Bucket=self.bucket, Key=self._full(key))["Body"] + data: bytes = body.read() + return data + + +__all__ = ["S3Store"] diff --git a/tests/unit/test_captures_stores.py b/tests/unit/test_captures_stores.py new file mode 100644 index 00000000..2d34e68b --- /dev/null +++ b/tests/unit/test_captures_stores.py @@ -0,0 +1,219 @@ +"""Each adapter against a stub client: pagination, prefix handling, timestamps, lazy import.""" + +from __future__ import annotations + +import io +from datetime import UTC, datetime +from typing import Any + +import pytest + +from evalshift_cli.captures import remote as remote_module +from evalshift_cli.captures.remote import RemoteStore, RemoteStoreUnavailable, open_store +from evalshift_cli.captures.store_uri import parse_store_uri +from evalshift_cli.captures.stores.azure import AzureBlobStore +from evalshift_cli.captures.stores.gcs import GCSStore +from evalshift_cli.captures.stores.s3 import S3Store + +T0 = datetime(2026, 10, 6, 12, 0, tzinfo=UTC) + +# --- S3 ------------------------------------------------------------------------------------ + + +class FakePaginator: + def __init__(self, pages: list[list[dict[str, Any]]], seen: list[dict[str, Any]]) -> None: + self._pages, self._seen = pages, seen + + def paginate(self, **kwargs: Any) -> list[dict[str, Any]]: + self._seen.append(kwargs) + return [{"Contents": page} for page in self._pages] + [{}] # a page with no Contents + + +class FakeS3Client: + def __init__(self, pages: list[list[dict[str, Any]]], blobs: dict[str, bytes]) -> None: + self.pages, self.blobs = pages, blobs + self.paginate_calls: list[dict[str, Any]] = [] + self.get_calls: list[dict[str, Any]] = [] + + def get_paginator(self, name: str) -> FakePaginator: + assert name == "list_objects_v2" + return FakePaginator(self.pages, self.paginate_calls) + + def get_object(self, **kwargs: Any) -> dict[str, Any]: + self.get_calls.append(kwargs) + return {"Body": io.BytesIO(self.blobs[kwargs["Key"]])} + + +def test_s3_list_paginates_and_strips_prefix() -> None: + client = FakeS3Client( + pages=[ + [{"Key": "p/captures/a/cap_1.json", "LastModified": T0, "Size": 10}], + [{"Key": "p/captures/a/cap_2.json", "LastModified": T0, "Size": 11}], + ], + blobs={}, + ) + store = S3Store("acme-evals", "p/", client=client) + infos = list(store.list("captures/")) + assert [i.key for i in infos] == ["captures/a/cap_1.json", "captures/a/cap_2.json"] + assert infos[0].last_modified == T0 and infos[1].size == 11 + assert client.paginate_calls == [{"Bucket": "acme-evals", "Prefix": "p/captures/"}] + assert store.uri == "s3://acme-evals/p" + + +def test_s3_get_reads_body_with_prefix() -> None: + client = FakeS3Client(pages=[], blobs={"p/toolsets/ab.json": b"[]"}) + assert S3Store("acme-evals", "p", client=client).get("toolsets/ab.json") == b"[]" + assert client.get_calls == [{"Bucket": "acme-evals", "Key": "p/toolsets/ab.json"}] + + +def test_s3_without_prefix_keys_are_relative_already() -> None: + client = FakeS3Client(pages=[[{"Key": "captures/a/cap_1.json"}]], blobs={}) + [info] = S3Store("acme-evals", client=client).list("captures/") + assert info.key == "captures/a/cap_1.json" + assert info.last_modified is None and info.size is None + + +def test_s3_satisfies_protocol() -> None: + assert isinstance(S3Store("b", client=FakeS3Client([], {})), RemoteStore) + + +def test_s3_naive_last_modified_is_taken_as_utc() -> None: + naive = datetime(2026, 10, 6, 12, 0) + client = FakeS3Client( + pages=[[{"Key": "captures/a/cap_1.json", "LastModified": naive}]], blobs={} + ) + [info] = S3Store("acme-evals", client=client).list("captures/") + assert info.last_modified == T0 and info.last_modified.tzinfo is not None + + +# --- GCS ----------------------------------------------------------------------------------- + + +class FakeGCSBlob: + def __init__(self, name: str, data: bytes = b"", updated: datetime | None = T0) -> None: + self.name, self._data, self.updated, self.size = name, data, updated, len(data) + + def download_as_bytes(self) -> bytes: + return self._data + + +class FakeGCSBucket: + def __init__(self, blobs: dict[str, FakeGCSBlob]) -> None: + self._blobs = blobs + + def blob(self, key: str) -> FakeGCSBlob: + return self._blobs[key] + + +class FakeGCSClient: + def __init__(self, blobs: dict[str, FakeGCSBlob]) -> None: + self._blobs = blobs + self.list_calls: list[tuple[str, str]] = [] + + def list_blobs(self, bucket: str, prefix: str) -> list[FakeGCSBlob]: + self.list_calls.append((bucket, prefix)) + return [b for k, b in sorted(self._blobs.items()) if k.startswith(prefix)] + + def bucket(self, name: str) -> FakeGCSBucket: + return FakeGCSBucket(self._blobs) + + +def test_gcs_list_and_get() -> None: + blobs = { + "p/captures/a/cap_1.json": FakeGCSBlob("p/captures/a/cap_1.json", b"{}"), + "p/toolsets/ab.json": FakeGCSBlob("p/toolsets/ab.json", b"[]"), + } + client = FakeGCSClient(blobs) + store = GCSStore("acme-evals", "p", client=client) + [info] = store.list("captures/") + assert info.key == "captures/a/cap_1.json" and info.size == 2 and info.last_modified == T0 + assert client.list_calls == [("acme-evals", "p/captures/")] + assert store.get("toolsets/ab.json") == b"[]" + assert store.uri == "gs://acme-evals/p" + + +# --- Azure --------------------------------------------------------------------------------- + + +class FakeBlobProperties: + def __init__(self, name: str, size: int, last_modified: datetime) -> None: + self.name, self.size, self.last_modified = name, size, last_modified + + +class FakeDownload: + def __init__(self, data: bytes) -> None: + self._data = data + + def readall(self) -> bytes: + return self._data + + +class FakeBlobClient: + def __init__(self, data: bytes) -> None: + self._data = data + + def download_blob(self) -> FakeDownload: + return FakeDownload(self._data) + + +class FakeContainerClient: + def __init__(self, props: list[FakeBlobProperties], seen: list[str]) -> None: + self._props, self._seen = props, seen + + def list_blobs(self, name_starts_with: str) -> list[FakeBlobProperties]: + self._seen.append(name_starts_with) + return [p for p in self._props if p.name.startswith(name_starts_with)] + + +class FakeBlobServiceClient: + def __init__(self, props: list[FakeBlobProperties], blobs: dict[str, bytes]) -> None: + self._props, self._blobs = props, blobs + self.list_prefixes: list[str] = [] + self.blob_requests: list[tuple[str, str]] = [] + + def get_container_client(self, container: str) -> FakeContainerClient: + return FakeContainerClient(self._props, self.list_prefixes) + + def get_blob_client(self, container: str, blob: str) -> FakeBlobClient: + self.blob_requests.append((container, blob)) + return FakeBlobClient(self._blobs[blob]) + + +def test_azure_list_and_get() -> None: + client = FakeBlobServiceClient( + props=[FakeBlobProperties("p/captures/a/cap_1.json", 7, T0)], + blobs={"p/captures/a/cap_1.json": b"{}"}, + ) + store = AzureBlobStore("acmeprod", "evals", "p", client=client) + [info] = store.list("captures/") + assert info.key == "captures/a/cap_1.json" and info.size == 7 and info.last_modified == T0 + assert client.list_prefixes == ["p/captures/"] + assert store.get("captures/a/cap_1.json") == b"{}" + assert client.blob_requests == [("evals", "p/captures/a/cap_1.json")] + assert store.uri == "az://acmeprod/evals/p" + assert store.account_url == "https://acmeprod.blob.core.windows.net" + + +# --- open_store ---------------------------------------------------------------------------- + + +def test_open_store_dispatches_without_building_a_client() -> None: + s3 = open_store(parse_store_uri("s3://b/p")) + assert isinstance(s3, S3Store) and s3._client is None + assert isinstance(open_store(parse_store_uri("gs://b/p")), GCSStore) + assert isinstance(open_store(parse_store_uri("az://a/c/p")), AzureBlobStore) + + +def test_open_store_missing_extra_names_pip_extra(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(remote_module, "_installed", lambda module: False) + with pytest.raises(RemoteStoreUnavailable) as info: + open_store(parse_store_uri("gs://b/p")) + assert info.value.hint == 'install it with: pip install "evalshift[gcs]"' + + +def test_open_store_azure_needs_identity_too(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(remote_module, "_installed", lambda module: module != "azure.identity") + with pytest.raises(RemoteStoreUnavailable) as info: + open_store(parse_store_uri("az://a/c/p")) + assert "'azure.identity'" in info.value.summary + assert info.value.hint == 'install it with: pip install "evalshift[azure]"' From 015c9a7234fc234437320f6a0247032768313394 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 19:32:49 +0200 Subject: [PATCH 06/17] feat(capture): fetch from captures.store; sync and list mirror the bucket first Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/capture.py | 184 +++++++++++++++- tests/unit/test_cli_capture_remote.py | 242 ++++++++++++++++++++++ 2 files changed, 416 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_cli_capture_remote.py diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index e9fe4cff..37091a3e 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -51,6 +51,15 @@ promoted_toolset_refs, toolsets_root, ) +from evalshift_cli.captures.remote import ( + FetchSummary, + RemoteStore, + RemoteStoreError, + fetch_captures, + open_store, + parse_since, +) +from evalshift_cli.captures.store_uri import parse_store_uri from evalshift_cli.cli.commands._suites import ( derive_suite_evaluators, inject_suites_block, @@ -59,6 +68,7 @@ suite_entry_payload, ) from evalshift_cli.cli.commands.doctor import CONFIG_FILENAME +from evalshift_cli.config.loader import ConfigError, load_config from evalshift_cli.config.models import SuiteEvaluatorsOverride from evalshift_cli.suite.loader import SuiteError, load_jsonl from evalshift_cli.traces.diff import diff_traces @@ -78,6 +88,33 @@ "--base", help="Capture base dir (default: $EVALSHIFT_DIR or .evalshift).", hidden=True ), ] +_ConfigOption = Annotated[ + Path, + typer.Option( + "--config", + "-c", + help=f"Path to evalshift.yaml (default: ./{CONFIG_FILENAME}).", + file_okay=True, + dir_okay=False, + ), +] +_OfflineOption = Annotated[ + bool, + typer.Option( + "--offline", + help="Skip fetching from captures.store; use only the captures already on disk.", + ), +] +_SinceOption = Annotated[ + str | None, + typer.Option( + "--since", + help=( + "Only fetch captures newer than this from captures.store: a duration " + "(30m, 24h, 7d) or an ISO date/datetime." + ), + ), +] def _warn_ci_pin(console: Console, config_path: Path) -> None: @@ -181,6 +218,90 @@ def _promoted_content_owners( return owners +def _configured_store(config_path: Path, *, console: Console, strict: bool) -> RemoteStore | None: + """The store ``captures.store`` names, or ``None`` when there is none to fetch from. + + A missing config file means no store. An unreadable one is a hard error when ``strict`` + (``capture fetch`` has nothing else to do) and a warning otherwise, so ``sync`` and ``list`` + keep working on the local mirror the way they did before ``captures.store`` existed. A + missing client extra is always a hard error: the user asked for a store it cannot reach. + """ + if not config_path.exists(): + return None + try: + cfg = load_config(config_path) + except ConfigError as exc: + if strict: + console.print(exc.format_rich()) + raise typer.Exit(code=1) from exc + console.print( + f"[yellow]⚠[/yellow] could not read {escape(str(config_path))} " + f"({escape(exc.summary)}); " + "skipping captures.store and using the captures already on disk.", + ) + return None + if cfg.captures.store is None: + return None + try: + return open_store(parse_store_uri(cfg.captures.store)) + except RemoteStoreError as exc: + console.print(exc.format_rich()) + raise typer.Exit(code=1) from exc + + +def _resolve_since(since: str | None) -> datetime | None: + if since is None: + return None + try: + return parse_since(since) + except ValueError as exc: + raise typer.BadParameter(str(exc), param_hint="--since") from exc + + +def _fetch_into_base( + store: RemoteStore, + *, + base: Path | None, + suite: str | None, + since: datetime | None, + console: Console, +) -> FetchSummary: + """Mirror ``store`` into ``base`` and print the one-line summary; exit 1 on failure.""" + skip_ids = _promoted_capture_ids_or_warn(base=base, console=console) + try: + summary = fetch_captures(store, base=base, suite=suite, since=since, skip_ids=skip_ids) + except RemoteStoreError as exc: + console.print(exc.format_rich()) + raise typer.Exit(code=1) from exc + # soft_wrap: one line per fetch, so a long store URI or skip list never splits a count + # from its label in a CI log someone greps. + console.print(f"[green]✓[/green] {summary.describe()}", soft_wrap=True) + return summary + + +def _maybe_fetch( + *, + config_path: Path, + offline: bool, + since: str | None, + suite: str | None, + base: Path | None, + console: Console, +) -> None: + """The automatic fetch step ``sync`` and ``list`` run before reading the local mirror.""" + since_dt = _resolve_since(since) + store = None if offline else _configured_store(config_path, console=console, strict=False) + if store is None: + if since_dt is not None: + console.print( + "[red]✗[/red] --since needs captures.store in evalshift.yaml and no --offline: " + "there is nothing to fetch from.", + ) + raise typer.Exit(code=1) + return + _fetch_into_base(store, base=base, suite=suite, since=since_dt, console=console) + + @capture_app.command(name="list") def capture_list( suite: Annotated[ @@ -191,10 +312,20 @@ def capture_list( bool, typer.Option("--json", help="Emit machine-readable JSON instead of a table."), ] = False, + config_path: _ConfigOption = Path(CONFIG_FILENAME), + offline: _OfflineOption = False, base: _BaseOption = None, ) -> None: """List captures recorded by the SDK.""" console = Console() + _maybe_fetch( + config_path=config_path, + offline=offline, + since=None, + suite=suite, + base=base, + console=console, + ) records = iter_captures(suite=suite, base=base, on_error=_warn_unreadable(console)) promoted = _promoted_capture_ids_or_warn(base=base, console=console) @@ -241,6 +372,38 @@ def capture_list( console.print(table) +@capture_app.command(name="fetch") +def capture_fetch( + suite: Annotated[ + str | None, + typer.Option("--suite", help="Only fetch captures for this suite."), + ] = None, + since: _SinceOption = None, + config_path: _ConfigOption = Path(CONFIG_FILENAME), + base: _BaseOption = None, +) -> None: + """Mirror new captures from captures.store (evalshift.yaml) into the local directory. + + ``capture sync`` and ``capture list`` run this step automatically; ``fetch`` exists for + inspecting with ``list``, ``diff`` or ``promote`` by hand before promoting anything. + """ + console = Console() + if not config_path.exists(): + console.print( + f"[red]✗[/red] {escape(str(config_path))} not found; capture fetch needs " + "captures.store in evalshift.yaml.", + ) + raise typer.Exit(code=1) + store = _configured_store(config_path, console=console, strict=True) + if store is None: + console.print( + "[red]✗[/red] no captures.store configured in evalshift.yaml " + "(for example: captures: {store: s3://bucket/prefix}).", + ) + raise typer.Exit(code=1) + _fetch_into_base(store, base=base, suite=suite, since=_resolve_since(since), console=console) + + def _check_rounds(rounds: str) -> None: """Reject a ``--rounds`` value the promoter has no meaning for.""" if rounds not in {"first", "all"}: @@ -751,16 +914,7 @@ def capture_sync( ), ), ] = False, - config_path: Annotated[ - Path, - typer.Option( - "--config", - "-c", - help=f"Path to evalshift.yaml (default: ./{CONFIG_FILENAME}).", - file_okay=True, - dir_okay=False, - ), - ] = Path(CONFIG_FILENAME), + config_path: _ConfigOption = Path(CONFIG_FILENAME), force: Annotated[ bool, typer.Option("--force", "-f", help="Overwrite promoted cases that already exist."), @@ -784,11 +938,21 @@ def capture_sync( ), ), ] = False, + since: _SinceOption = None, + offline: _OfflineOption = False, base: _BaseOption = None, ) -> None: """Promote every capture into a suite and wire the suites: block into evalshift.yaml.""" console = Console() _check_rounds(rounds) + _maybe_fetch( + config_path=config_path, + offline=offline, + since=since, + suite=suite, + base=base, + console=console, + ) records = iter_captures(suite=suite, base=base, on_error=_warn_unreadable(console)) if not records: where = f" for suite {suite!r}" if suite else "" diff --git a/tests/unit/test_cli_capture_remote.py b/tests/unit/test_cli_capture_remote.py new file mode 100644 index 00000000..201ccd07 --- /dev/null +++ b/tests/unit/test_cli_capture_remote.py @@ -0,0 +1,242 @@ +"""``capture fetch`` and the automatic fetch step in ``capture sync`` / ``capture list``.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +from typer.testing import CliRunner + +from evalshift_cli.captures.remote import RemoteStore, RemoteStoreUnavailable +from evalshift_cli.captures.store_uri import StoreURI +from evalshift_cli.cli.commands import capture as capture_module +from evalshift_cli.cli.commands.init import render_minimal_config +from evalshift_cli.cli.main import app +from evalshift_cli.suite.loader import load_jsonl +from tests.unit.fake_remote_store import FakeRemoteStore, populated_store + +runner = CliRunner() + + +def _invoke(args: list[str], base: Path) -> Any: + return runner.invoke(app, ["capture", *args, "--base", str(base)]) + + +def _write_config(path: Path, *, store: str | None = "s3://acme-evals/support-agent") -> Path: + text = render_minimal_config(profile="model-upgrade") + if store is not None: + text += f"\ncaptures:\n store: {store}\n" + path.write_text(text, encoding="utf-8") + return path + + +@pytest.fixture +def remote(monkeypatch: pytest.MonkeyPatch) -> FakeRemoteStore: + """Install a populated fake as the store every URI opens to.""" + store = populated_store("cap_1", "cap_2") + opened: list[StoreURI] = [] + + def _open(parsed: StoreURI) -> RemoteStore: + opened.append(parsed) + return store + + monkeypatch.setattr(capture_module, "open_store", _open) + store.opened = opened # type: ignore[attr-defined] + return store + + +# --- fetch --------------------------------------------------------------------------------- + + +def test_fetch_mirrors_store_into_base(tmp_path: Path, remote: FakeRemoteStore) -> None: + config = _write_config(tmp_path / "evalshift.yaml") + result = _invoke(["fetch", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + assert "fetched 2 capture(s) and 1 toolset(s) from s3://fake/prefix" in result.stdout + assert (tmp_path / "captures" / "alpha" / "cap_1.json").exists() + assert remote.opened[0].bucket == "acme-evals" # type: ignore[attr-defined] + + +def test_fetch_without_store_configured_errors(tmp_path: Path, remote: FakeRemoteStore) -> None: + config = _write_config(tmp_path / "evalshift.yaml", store=None) + result = _invoke(["fetch", "--config", str(config)], tmp_path) + assert result.exit_code == 1 + assert "no captures.store configured" in result.stdout + assert remote.gets == [] + + +def test_fetch_without_config_file_errors(tmp_path: Path, remote: FakeRemoteStore) -> None: + result = _invoke(["fetch", "--config", str(tmp_path / "missing.yaml")], tmp_path) + assert result.exit_code == 1 + assert "not found" in result.stdout + + +def test_fetch_invalid_config_prints_config_error(tmp_path: Path, remote: FakeRemoteStore) -> None: + config = _write_config(tmp_path / "evalshift.yaml", store="ftp://nope/x") + result = _invoke(["fetch", "--config", str(config)], tmp_path) + assert result.exit_code == 1 + assert "Invalid config" in result.stdout + + +def test_fetch_since_and_suite_are_passed_through( + tmp_path: Path, remote: FakeRemoteStore, monkeypatch: pytest.MonkeyPatch +) -> None: + seen: dict[str, Any] = {} + + def _fake_fetch(store: RemoteStore, **kwargs: Any) -> Any: + seen.update(kwargs) + return capture_module.FetchSummary(store_uri=store.uri) + + monkeypatch.setattr(capture_module, "fetch_captures", _fake_fetch) + config = _write_config(tmp_path / "evalshift.yaml") + result = _invoke( + ["fetch", "--suite", "alpha", "--since", "24h", "--config", str(config)], tmp_path + ) + assert result.exit_code == 0, result.stdout + assert seen["suite"] == "alpha" + assert seen["since"] is not None + assert seen["base"] == tmp_path + + +def test_fetch_rejects_bad_since(tmp_path: Path, remote: FakeRemoteStore) -> None: + config = _write_config(tmp_path / "evalshift.yaml") + result = _invoke(["fetch", "--since", "yesterday", "--config", str(config)], tmp_path) + assert result.exit_code == 2 + assert "--since expects" in result.output + + +# --- sync ---------------------------------------------------------------------------------- + + +def test_sync_fetches_first_then_promotes(tmp_path: Path, remote: FakeRemoteStore) -> None: + config = _write_config(tmp_path / "evalshift.yaml") + result = _invoke(["sync", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + assert "fetched 2 capture(s)" in result.stdout + assert "promoted 2 capture(s)" in result.stdout + golden = tmp_path / "suites" / "alpha" / "golden.jsonl" + assert len(load_jsonl(golden).ids()) == 2 + + +def test_sync_offline_skips_fetch(tmp_path: Path, remote: FakeRemoteStore) -> None: + config = _write_config(tmp_path / "evalshift.yaml") + result = _invoke(["sync", "--offline", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + assert "fetched" not in result.stdout + assert remote.gets == [] + + +def test_sync_without_store_is_unchanged(tmp_path: Path, remote: FakeRemoteStore) -> None: + config = _write_config(tmp_path / "evalshift.yaml", store=None) + result = _invoke(["sync", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + assert "fetched" not in result.stdout + assert remote.gets == [] + + +def test_sync_since_without_store_errors(tmp_path: Path, remote: FakeRemoteStore) -> None: + config = _write_config(tmp_path / "evalshift.yaml", store=None) + result = _invoke(["sync", "--since", "24h", "--config", str(config)], tmp_path) + assert result.exit_code == 1 + assert "--since needs captures.store" in result.stdout + assert not (tmp_path / "suites").exists() + + +def test_sync_fetch_failure_stops_before_promotion(tmp_path: Path, remote: FakeRemoteStore) -> None: + remote.fail_keys.add("captures/alpha/cap_2.json") + config = _write_config(tmp_path / "evalshift.yaml") + result = _invoke(["sync", "--config", str(config)], tmp_path) + assert result.exit_code == 1 + assert "Remote capture store" in result.stdout + assert "--offline" in result.stdout + assert not (tmp_path / "suites").exists() + + +def test_sync_does_not_refetch_promoted_then_cleaned_captures( + tmp_path: Path, remote: FakeRemoteStore +) -> None: + config = _write_config(tmp_path / "evalshift.yaml") + assert _invoke(["sync", "--config", str(config)], tmp_path).exit_code == 0 + cleaned = _invoke(["clean", "--yes"], tmp_path) + assert cleaned.exit_code == 0, cleaned.stdout + assert not (tmp_path / "captures" / "alpha" / "cap_1.json").exists() + remote.gets.clear() + result = _invoke(["sync", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + assert "fetched 0 capture(s)" in result.stdout + assert "2 promoted" in result.stdout + assert all(not key.startswith("captures/") for key in remote.gets) + + +def test_sync_with_unreadable_config_warns_and_continues_locally( + tmp_path: Path, remote: FakeRemoteStore +) -> None: + config = tmp_path / "evalshift.yaml" + config.write_text("prompts: []\n", encoding="utf-8") # invalid: prompts needs one entry + result = _invoke(["sync", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + assert "skipping captures.store" in result.stdout + assert remote.gets == [] + + +# --- list ---------------------------------------------------------------------------------- + + +def test_list_fetches_first_when_store_configured(tmp_path: Path, remote: FakeRemoteStore) -> None: + config = _write_config(tmp_path / "evalshift.yaml") + result = _invoke(["list", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + assert "fetched 2 capture(s)" in result.stdout + assert "cap_1" in result.stdout and "cap_2" in result.stdout + + +def test_list_offline_and_without_config_stay_local( + tmp_path: Path, remote: FakeRemoteStore +) -> None: + config = _write_config(tmp_path / "evalshift.yaml") + offline = _invoke(["list", "--offline", "--config", str(config)], tmp_path) + assert offline.exit_code == 0 and "no captures found" in offline.stdout + missing = _invoke(["list", "--config", str(tmp_path / "nope.yaml")], tmp_path) + assert missing.exit_code == 0 and "no captures found" in missing.stdout + assert remote.gets == [] + + +def test_list_json_output_is_not_polluted_by_fetch_line( + tmp_path: Path, remote: FakeRemoteStore +) -> None: + import json + + config = _write_config(tmp_path / "evalshift.yaml") + result = _invoke(["list", "--json", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + rows = json.loads(result.stdout[result.stdout.index("[") :]) + assert {r["capture_id"] for r in rows} == {"cap_1", "cap_2"} + + +# --- missing client extra ------------------------------------------------------------------- + + +@pytest.fixture +def missing_extra(monkeypatch: pytest.MonkeyPatch) -> None: + """Make every store URI fail to open as if its pip extra were not installed.""" + + def _open(parsed: StoreURI) -> RemoteStore: + raise RemoteStoreUnavailable( + "s3:// captures store needs the optional dependency 'boto3'", + hint='install it with: pip install "evalshift[s3]"', + ) + + monkeypatch.setattr(capture_module, "open_store", _open) + + +@pytest.mark.parametrize("command", ["fetch", "sync", "list"]) +def test_missing_extra_exits_1_naming_the_pip_extra( + tmp_path: Path, missing_extra: None, command: str +) -> None: + config = _write_config(tmp_path / "evalshift.yaml") + result = _invoke([command, "--config", str(config)], tmp_path) + assert result.exit_code == 1 + assert "Remote capture store" in result.stdout + assert 'pip install "evalshift[s3]"' in result.stdout + assert not (tmp_path / "suites").exists() From 9751e1860224e9413215a186f574015506628fde Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 19:37:31 +0200 Subject: [PATCH 07/17] fix(capture): escape the store URI, keep list --json stdout pure, stay silent on configs without captures Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/capture.py | 44 ++++++++++++++++------ tests/unit/test_cli_capture_remote.py | 46 +++++++++++++++++++++-- 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index 37091a3e..4b09bba6 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -218,13 +218,23 @@ def _promoted_content_owners( return owners +def _mentions_captures(config_path: Path) -> bool: + """Whether the raw config file text names ``captures`` at all (``False`` if unreadable).""" + try: + return b"captures" in config_path.read_bytes() + except OSError: + return False + + def _configured_store(config_path: Path, *, console: Console, strict: bool) -> RemoteStore | None: """The store ``captures.store`` names, or ``None`` when there is none to fetch from. - A missing config file means no store. An unreadable one is a hard error when ``strict`` - (``capture fetch`` has nothing else to do) and a warning otherwise, so ``sync`` and ``list`` - keep working on the local mirror the way they did before ``captures.store`` existed. A - missing client extra is always a hard error: the user asked for a store it cannot reach. + A missing config file means no store. An invalid one is a hard error when ``strict`` + (``capture fetch`` has nothing else to do). Otherwise ``sync`` and ``list`` keep working on + the local mirror the way they did before ``captures.store`` existed: an invalid or + unreadable file is skipped, with a warning only when its text mentions ``captures`` (a + config that never asked for a store must not change their output). A missing client extra + is always a hard error: the user asked for a store it cannot reach. """ if not config_path.exists(): return None @@ -234,11 +244,12 @@ def _configured_store(config_path: Path, *, console: Console, strict: bool) -> R if strict: console.print(exc.format_rich()) raise typer.Exit(code=1) from exc - console.print( - f"[yellow]⚠[/yellow] could not read {escape(str(config_path))} " - f"({escape(exc.summary)}); " - "skipping captures.store and using the captures already on disk.", - ) + _warn_store_skipped(config_path, exc.summary, console=console) + return None + except (OSError, UnicodeDecodeError) as exc: + if strict: + raise + _warn_store_skipped(config_path, type(exc).__name__, console=console) return None if cfg.captures.store is None: return None @@ -249,6 +260,16 @@ def _configured_store(config_path: Path, *, console: Console, strict: bool) -> R raise typer.Exit(code=1) from exc +def _warn_store_skipped(config_path: Path, reason: str, *, console: Console) -> None: + """Warn that captures.store is being skipped, if the file looks like it configures one.""" + if not _mentions_captures(config_path): + return + console.print( + f"[yellow]⚠[/yellow] could not read {escape(str(config_path))} ({escape(reason)}); " + "skipping captures.store and using the captures already on disk.", + ) + + def _resolve_since(since: str | None) -> datetime | None: if since is None: return None @@ -275,7 +296,7 @@ def _fetch_into_base( raise typer.Exit(code=1) from exc # soft_wrap: one line per fetch, so a long store URI or skip list never splits a count # from its label in a CI log someone greps. - console.print(f"[green]✓[/green] {summary.describe()}", soft_wrap=True) + console.print(f"[green]✓[/green] {escape(summary.describe())}", soft_wrap=True) return summary @@ -317,7 +338,8 @@ def capture_list( base: _BaseOption = None, ) -> None: """List captures recorded by the SDK.""" - console = Console() + # With --json, stdout carries only the JSON: the fetch line and every warning go to stderr. + console = Console(stderr=as_json) _maybe_fetch( config_path=config_path, offline=offline, diff --git a/tests/unit/test_cli_capture_remote.py b/tests/unit/test_cli_capture_remote.py index 201ccd07..bdef6c22 100644 --- a/tests/unit/test_cli_capture_remote.py +++ b/tests/unit/test_cli_capture_remote.py @@ -173,13 +173,52 @@ def test_sync_with_unreadable_config_warns_and_continues_locally( tmp_path: Path, remote: FakeRemoteStore ) -> None: config = tmp_path / "evalshift.yaml" - config.write_text("prompts: []\n", encoding="utf-8") # invalid: prompts needs one entry + # Invalid (prompts needs one entry), but it asks for a store, so skipping it is news. + config.write_text("prompts: []\ncaptures:\n store: s3://b/p\n", encoding="utf-8") result = _invoke(["sync", "--config", str(config)], tmp_path) assert result.exit_code == 0, result.stdout assert "skipping captures.store" in result.stdout assert remote.gets == [] +@pytest.mark.parametrize("command", ["sync", "list"]) +def test_invalid_config_without_captures_stays_silent( + tmp_path: Path, remote: FakeRemoteStore, command: str +) -> None: + config = tmp_path / "evalshift.yaml" + config.write_text("prompts: []\n", encoding="utf-8") # invalid, never mentions a store + result = _invoke([command, "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.output + assert "skipping captures.store" not in result.output + assert "could not read" not in result.output + assert remote.gets == [] + + +def test_list_with_undecodable_config_stays_local_without_traceback( + tmp_path: Path, remote: FakeRemoteStore +) -> None: + config = tmp_path / "evalshift.yaml" + config.write_bytes(b"prompts:\n - \xff\xfe\x80 not utf-8\n") + result = _invoke(["list", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.output + assert result.exception is None + assert "no captures found" in result.stdout + assert "could not read" not in result.output + assert remote.gets == [] + + +def test_bracketed_store_uri_prints_literally(tmp_path: Path, remote: FakeRemoteStore) -> None: + remote.uri = "s3://acme/team[/x]" + config = _write_config(tmp_path / "evalshift.yaml") + fetched = _invoke(["fetch", "--config", str(config)], tmp_path) + assert fetched.exit_code == 0, fetched.output + assert "from s3://acme/team[/x]" in fetched.stdout + remote.uri = "s3://acme/[bold]team" + synced = _invoke(["sync", "--config", str(config)], tmp_path) + assert synced.exit_code == 0, synced.output + assert "from s3://acme/[bold]team" in synced.stdout + + # --- list ---------------------------------------------------------------------------------- @@ -209,9 +248,10 @@ def test_list_json_output_is_not_polluted_by_fetch_line( config = _write_config(tmp_path / "evalshift.yaml") result = _invoke(["list", "--json", "--config", str(config)], tmp_path) - assert result.exit_code == 0, result.stdout - rows = json.loads(result.stdout[result.stdout.index("[") :]) + assert result.exit_code == 0, result.output + rows = json.loads(result.stdout) # the whole of stdout is the JSON payload assert {r["capture_id"] for r in rows} == {"cap_1", "cap_2"} + assert "fetched 2 capture(s)" in result.stderr # --- missing client extra ------------------------------------------------------------------- From 52a6d0031ef232fb11950829909ab2f7ed8f691e Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 19:41:23 +0200 Subject: [PATCH 08/17] feat(doctor): report whether captures.store is installed and reachable Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/doctor.py | 51 +++++++++++++++- tests/unit/test_doctor.py | 74 +++++++++++++++++++++++- 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/src/evalshift_cli/cli/commands/doctor.py b/src/evalshift_cli/cli/commands/doctor.py index 81a11052..f53057ab 100644 --- a/src/evalshift_cli/cli/commands/doctor.py +++ b/src/evalshift_cli/cli/commands/doctor.py @@ -8,7 +8,8 @@ * **0** — every check passed, or any failures were merely informational (e.g. an unset API key, or no config in this directory yet). * **1** — at least one **hard** failure was reported (currently: an - ``evalshift.yaml`` exists in the cwd but doesn't validate). + ``evalshift.yaml`` exists in the cwd but doesn't validate, or its + ``captures.store`` needs a client extra that isn't installed). Soft failures (missing API keys, no config yet) are surfaced visually with a yellow ``✗`` so users see them, but they never fail the command — this @@ -36,9 +37,12 @@ import typer from rich.console import Console +from rich.markup import escape from rich.table import Table from evalshift_cli import __version__ +from evalshift_cli.captures.remote import RemoteStoreError, open_store +from evalshift_cli.captures.store_uri import parse_store_uri from evalshift_cli.captures.toolset import EMPTY_TOOLSET_FINGERPRINT, fingerprint_tools from evalshift_cli.cli.commands._suites import SUITE_FILENAME from evalshift_cli.config.loader import ConfigError, load_config @@ -115,6 +119,7 @@ def run_checks(cwd: Path, env: Mapping[str, str]) -> list[CheckResult]: results = [_python_check(), _sdk_check()] results.extend(_api_key_check(env, aliases) for aliases in PROVIDER_KEYS) results.append(_config_check(cwd)) + results.extend(_captures_store_check(cwd)) results.extend(_tool_consistency_checks(cwd)) results.extend(_judge_family_checks(cwd)) results.extend(_ci_pin_check(cwd)) @@ -244,6 +249,42 @@ def _config_check(cwd: Path) -> CheckResult: ) +def _captures_store_check(cwd: Path) -> list[CheckResult]: + """One row for ``captures.store`` when the config names one: extra installed, bucket listable. + + No config, an invalid config (the ``evalshift.yaml`` row already reports that) or no + ``captures.store`` produce no row. A missing client extra is a ``fail`` -- it is a local, + deterministic problem with a one-line fix. A listing that raises is a ``warn``: doctor runs + on laptops without cloud credentials, and that must not fail the command. + """ + cfg_path = cwd / CONFIG_FILENAME + if not cfg_path.exists(): + return [] + try: + cfg = load_config(cfg_path) + except ConfigError: + return [] + uri = cfg.captures.store + if uri is None: + return [] + try: + store = open_store(parse_store_uri(uri)) + except RemoteStoreError as exc: + detail = exc.summary if exc.hint is None else f"{exc.summary} — {exc.hint}" + return [CheckResult(name="captures.store", status="fail", detail=detail)] + try: + next(iter(store.list("captures/")), None) + except Exception as exc: + return [ + CheckResult( + name="captures.store", + status="warn", + detail=f"{uri} not reachable: {type(exc).__name__}: {exc}", + ) + ] + return [CheckResult(name="captures.store", status="ok", detail=f"{uri} reachable")] + + def _named_suite_paths(cwd: Path, cfg: EvalShiftConfig) -> list[tuple[str, Path]]: """Return ``(name, path)`` for every golden-suite file the toolset check should inspect. @@ -485,14 +526,18 @@ def source_conformance_check(records: Sequence[EvalRecord]) -> CheckResult | Non def render_results(results: list[CheckResult], console: Console) -> None: - """Render the check results as a Rich table.""" + """Render the check results as a Rich table. + + Names and details are plain text, escaped so a bracketed value -- an install hint such as + ``pip install "evalshift[gcs]"`` or a user-written store URI -- is shown, not parsed as markup. + """ table = Table(show_header=False, box=None, padding=(0, 1)) table.add_column("status", no_wrap=True) table.add_column("name", style="bold") table.add_column("detail", overflow="fold") for r in results: glyph, style = _GLYPHS[r.status] - table.add_row(f"[{style}]{glyph}[/{style}]", r.name, r.detail) + table.add_row(f"[{style}]{glyph}[/{style}]", escape(r.name), escape(r.detail)) console.print(table) diff --git a/tests/unit/test_doctor.py b/tests/unit/test_doctor.py index dc87fcc4..f415aff4 100644 --- a/tests/unit/test_doctor.py +++ b/tests/unit/test_doctor.py @@ -11,7 +11,7 @@ from __future__ import annotations import json -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from importlib.metadata import PackageNotFoundError from pathlib import Path from types import ModuleType @@ -19,7 +19,9 @@ import pytest from typer.testing import CliRunner +from evalshift_cli.captures.remote import ObjectInfo, RemoteStoreUnavailable from evalshift_cli.captures.toolset import fingerprint_tools +from evalshift_cli.cli.commands import doctor as doctor_module from evalshift_cli.cli.commands.doctor import ( CONFIG_FILENAME, JUDGE_FAMILY_CHECK, @@ -782,3 +784,73 @@ def test_silent_when_the_arms_are_not_configured(self, tmp_path: Path) -> None: def test_silent_when_no_config_exists(self, tmp_path: Path) -> None: assert _judge_rows(run_checks(cwd=tmp_path, env=_empty_env())) == [] + + +class TestCapturesStoreCheck: + def _config(self, tmp_path: Path, store: str | None) -> None: + text = "prompts:\n - id: p\n detection: manual\n content: hi\n" + if store is not None: + text += f"captures:\n store: {store}\n" + (tmp_path / CONFIG_FILENAME).write_text(text, encoding="utf-8") + + def test_no_row_without_store(self, tmp_path: Path) -> None: + self._config(tmp_path, None) + names = [r.name for r in run_checks(cwd=tmp_path, env=_empty_env())] + assert "captures.store" not in names + + def test_reachable_store_is_ok(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from tests.unit.fake_remote_store import populated_store + + self._config(tmp_path, "s3://acme-evals/p") + monkeypatch.setattr(doctor_module, "open_store", lambda parsed: populated_store("cap_1")) + row = _by_name(run_checks(cwd=tmp_path, env=_empty_env()), "captures.store") + assert row.status == "ok" + assert "s3://acme-evals/p" in row.detail + + def test_missing_extra_fails(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + self._config(tmp_path, "gs://b/p") + + def _raise(parsed: object) -> object: + raise RemoteStoreUnavailable( + "gs:// needs google-cloud-storage", hint='pip install "evalshift[gcs]"' + ) + + monkeypatch.setattr(doctor_module, "open_store", _raise) + row = _by_name(run_checks(cwd=tmp_path, env=_empty_env()), "captures.store") + assert row.status == "fail" + assert 'pip install "evalshift[gcs]"' in row.detail + + def test_unreachable_store_warns(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from tests.unit.fake_remote_store import FakeRemoteStore + + class Unreachable(FakeRemoteStore): + def list(self, prefix: str) -> Iterator[ObjectInfo]: + raise ConnectionError("no route to host") + + self._config(tmp_path, "s3://acme-evals/p") + monkeypatch.setattr(doctor_module, "open_store", lambda parsed: Unreachable()) + row = _by_name(run_checks(cwd=tmp_path, env=_empty_env()), "captures.store") + assert row.status == "warn" + assert "ConnectionError" in row.detail + + def test_invalid_config_adds_no_store_row(self, tmp_path: Path) -> None: + (tmp_path / CONFIG_FILENAME).write_text("prompts: []\n", encoding="utf-8") + names = [r.name for r in run_checks(cwd=tmp_path, env=_empty_env())] + assert "captures.store" not in names # the evalshift.yaml row already says it is invalid + + def test_missing_extra_hint_survives_rendering( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # ``[gcs]`` is Rich markup syntax; rendered unescaped it would vanish from the hint. + self._config(tmp_path, "gs://b/p") + + def _raise(parsed: object) -> object: + raise RemoteStoreUnavailable( + "gs:// needs google-cloud-storage", hint='pip install "evalshift[gcs]"' + ) + + monkeypatch.setattr(doctor_module, "open_store", _raise) + monkeypatch.chdir(tmp_path) + result = runner.invoke(app, ["doctor"], env={"COLUMNS": "200"}) + assert result.exit_code == 1 + assert 'pip install "evalshift[gcs]"' in result.stdout From a3504676ebad10c08aa50e9182ec181d7c698e33 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 19:48:41 +0200 Subject: [PATCH 09/17] docs: captures.store, capture fetch, bucket-mirroring sync and list Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 23 +++++++++++++ DOCS.md | 26 +++++++++++---- README.md | 3 +- docs/getting-started.md | 5 +++ docs/hosted.md | 11 +++++-- docs/sdk.md | 31 +++++++++++++---- llms-full.txt | 73 +++++++++++++++++++++++++++++++---------- 7 files changed, 139 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fae51632..8673b775 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `captures.store` in `evalshift.yaml`: name the object store the capture SDK + ships captures to — `s3:///`, `gs:///` or + `az:////` — and `capture sync` and `capture list` + mirror new captures and toolset sidecars from it into `.evalshift/` before + they run, with no flags. Only objects missing locally are downloaded + (captures are immutable, toolsets content-addressed), captures already + promoted are skipped so `capture clean` never causes a re-download, and + every file is written atomically. New `capture fetch` does only that step; + `--since 30m|24h|7d|` limits a fetch to recently written captures and + `--offline` skips it. Keys from a bucket are validated against the SDK's + layout and anything else is ignored as malformed — including a suite + directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with + `.` or a space — and no destination can land outside `.evalshift/`. Extras: + `evalshift[s3]`, `[gcs]`, `[azure]` (azure-storage-blob and azure-identity); + a missing one exits 1 naming the `pip install` to run. Credentials come from + each provider's default chain and are rejected in the URI: a value with `@` + or `?` fails to load, naming the accepted forms without echoing it. + `capture list --json` keeps stdout pure JSON (the fetch line and warnings + go to stderr). `doctor` reports whether the store is installed (`fail`, exit 1, + when the extra is missing) and reachable (`warn` when it cannot be listed). + Without `captures.store` nothing changes; a `sync` or `list` whose + `evalshift.yaml` cannot be read or does not validate skips the fetch, + warning only when the file mentions `captures`. - `evaluators.trace_invariants`: hand-written rules over tool-call traces — `forbidden`, `required`, `order`, `call_count` (`min_calls` and/or `max_calls` per tool; equal bounds = an exact count) and diff --git a/DOCS.md b/DOCS.md index 768d2de0..5519e33e 100644 --- a/DOCS.md +++ b/DOCS.md @@ -14,7 +14,7 @@ The suite is the crux, so the capture SDK is the recommended way to build one: i - **Package name:** `evalshift` · **CLI entry point:** `evalshift` · **version:** 1.2.2 - **Python:** >= 3.11 · **License:** Apache-2.0 · **Status:** stable -- **Local-first.** Runs, scores, stats, and reports all happen on your machine under `.evalshift/`. The only network calls are the model API calls you asked for — and, if you opt in, pushes to the hosted service. +- **Local-first.** Runs, scores, stats, and reports all happen on your machine under `.evalshift/`. The only network calls are the model API calls you asked for — and, if you opt in, pushes to the hosted service and reads from a capture bucket you own (`captures.store`). - **Four pieces:** CLI (this doc), SDK, GitHub Action, hosted server — each with its own machine-readable reference for AI tools. See [Ecosystem and AI-tool references](#ecosystem-and-ai-tool-references). --- @@ -159,12 +159,22 @@ The evalshift-sdk (`pip install evalshift-sdk` — already a dependency of the C ```bash evalshift capture list # table of recorded captures (--json for machine-readable) +evalshift capture fetch # (captures.store only) mirror new captures into .evalshift/ evalshift capture sync # promote ALL captures → suites + wire config evalshift capture promote cap_ab12 --as refund_case_1 # promote one evalshift capture diff cap_ab12 cap_cd34 # compare two tool traces evalshift capture clean # delete promoted capture files + sweep orphaned toolsets ``` +**Captures that live in a bucket.** Production hosts on Fargate, Lambda or Kubernetes lose their disk when they stop, so the SDK can ship captures to an object store you own instead (`EVALSHIFT_SINK=s3:///`, `gs://…` or `az:////…` in the agent's environment, SDK 0.5.0+). Name the same store once in `evalshift.yaml`: + +```yaml +captures: + store: s3://acme-evals/support-agent +``` + +and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written, and no key can place a file outside `.evalshift/`. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching extra — `evalshift[s3]`, `[gcs]` or `[azure]` (azure-storage-blob plus azure-identity) — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load without being echoed back. A missing extra exits 1 naming the `pip install "evalshift[]"` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file mentions `captures`. Full field reference: [`captures`](docs/configuration.md#captures). + **`capture sync`** is the workhorse. Per suite it: 1. Groups captures (by `conversation_id` for multi-turn, ordered by `turn_index`). @@ -207,7 +217,7 @@ init → doctor → run → evaluate → analy .json (if policy) ``` -- **`doctor`** validates local config and shows which provider keys are visible. Exit 1 only when an existing `evalshift.yaml` fails validation — the row gives the problem count and points at `evalshift validate`, which prints each problem; missing keys are soft warnings. Its second row, `evalshift-sdk`, reports the SDK version the `evalshift` import name resolves to in this environment (`warn` when the SDK is missing or shadowed by an older CLI's leftover files; never a failure). It also reports the toolset each configured suite carries (or the flat `golden.jsonl`) and flags a suite whose examples carry more than one distinct toolset — legal (each example dispatches its own), but also the shape a wiring mistake takes. When a workflow under `.github/workflows/` uses the GitHub Action it adds a `ci pin` row: `ok` (`pinned to `) when CI installs this CLI version, `warn` when the pin is older, absent, or newer than the local CLI (see [Pin drift](#pin-drift)). When the config wires an `llm_judge` evaluator and names both `defaults.source_model` and `target_model`, a `judge family` row warns for every `judge_model` that resolves to the same provider as an arm (self-preference bias; `ok` "from a third family" otherwise, no row when either arm is unset) — advisory, never a failure; `validate` prints the same line and the report repeats it above the verdict (see [`evaluators.llm_judge`](docs/configuration.md#evaluatorsllm_judge)). +- **`doctor`** validates local config and shows which provider keys are visible. Exit 1 only when an existing `evalshift.yaml` fails validation — the row gives the problem count and points at `evalshift validate`, which prints each problem — or when its `captures.store` needs a client extra that is not installed; missing keys are soft warnings. Its second row, `evalshift-sdk`, reports the SDK version the `evalshift` import name resolves to in this environment (`warn` when the SDK is missing or shadowed by an older CLI's leftover files; never a failure). It also reports the toolset each configured suite carries (or the flat `golden.jsonl`) and flags a suite whose examples carry more than one distinct toolset — legal (each example dispatches its own), but also the shape a wiring mistake takes. When a workflow under `.github/workflows/` uses the GitHub Action it adds a `ci pin` row: `ok` (`pinned to `) when CI installs this CLI version, `warn` when the pin is older, absent, or newer than the local CLI (see [Pin drift](#pin-drift)). When the config sets `captures.store`, a `captures.store` row reports it: `fail` (exit 1) naming the `pip install "evalshift[]"` when the client extra is missing, `warn` when the bucket cannot be listed (no cloud credentials on this machine, say), `ok` (` reachable`) otherwise; no row without a store. When the config wires an `llm_judge` evaluator and names both `defaults.source_model` and `target_model`, a `judge family` row warns for every `judge_model` that resolves to the same provider as an arm (self-preference bias; `ok` "from a third family" otherwise, no row when either arm is unset) — advisory, never a failure; `validate` prints the same line and the report repeats it above the verdict (see [`evaluators.llm_judge`](docs/configuration.md#evaluatorsllm_judge)). - **`run`** parses prompts, validates every example against every prompt, estimates cost, then dispatches `(prompt × example × {source, target})` calls through an async orchestrator under a concurrency semaphore. Responses are cached — per call, and per replayed round for tool-calling examples (see [Response cache](#response-cache)); progress is checkpointed every 50 completions. - **`evaluate`** scores each (source, target) pair with the configured evaluators, one `EvalRecord` per pair × evaluator. Scoring runs under the same `defaults.concurrency` semaphore as `run`, and the embedding/judge calls it makes go through the same response cache. - **`analyze`** runs paired statistics per `(prompt, evaluator, slice)`, applies Benjamini–Hochberg FDR correction, classifies severities, and — when a `migration_policy` is configured — computes a pass/fail verdict. @@ -319,6 +329,7 @@ suites: {} | `migration_policy` | block \| absent | Regression budgets, see [Migration policy](#migration-policy-and-ci-gating) | | `suites` | map | Named suites (`{name: {source: captured\|jsonl, path: ..., evaluators: ..., managed: true}}`); the block between the `>>> evalshift suites` markers is managed by `capture sync`. See [Per-suite evaluators](#per-suite-evaluators) | | `retention` | block | `max_runs_per_suite` (default 20, `0` disables), `run_ttl_days` (default off) | +| `captures` | block | `store`: `s3://`, `gs://` or `az://` URI of the object store the SDK ships captures to; `sync` and `list` mirror it locally first. Omit for local disk only. See [Capturing from production](#capturing-from-production) | `thresholds` was a top-level field until it was removed. It was free-form and gated nothing — `migration_policy` is the single source of truth for gating — so rather than being silently ignored it is now rejected by name, and a config that still carries it fails to load: @@ -792,7 +803,7 @@ evalshift logout ### What uploads and what stays local -The full field-by-field data contract lives in [docs/hosted.md — Privacy model](docs/hosted.md#privacy-model--exactly-what-uploads); this is the summary. The CLI has **no telemetry** — no analytics, no crash reporting. Its only network traffic is (1) your configured model providers, with your own keys, during `run`/`evaluate`/`report`, and (2) the hosted API on `login`, `whoami`, and `push`. +The full field-by-field data contract lives in [docs/hosted.md — Privacy model](docs/hosted.md#privacy-model--exactly-what-uploads); this is the summary. The CLI has **no telemetry** — no analytics, no crash reporting. Its only network traffic is (1) your configured model providers, with your own keys, during `run`/`evaluate`/`report`, and (2) the hosted API on `login`, `whoami`, and `push` — plus, with `captures.store` configured, reads from your own capture bucket. **A push uploads**, inside `run_bundle.json.gz`: the manifest (run id, `org/project` slug, model ids, suite name, git SHA/branch/PR number, the local suite file path string, content hashes, timestamp, CLI version); per-example rows — the example's template `inputs` and `expected` output **verbatim**, both models' **full output text**, the replay's tool-call traces (tool names, arguments and call ids, round markers, final text and refusal messages, capped at 256 KB per side; imported agent traces are not uploaded), per-evaluator scores, error strings and one-line explanations, for each `trace_invariants` evaluator the rules each side broke (rule id, type, tool, round, detail; also for entries checked on imported traces, whose timelines otherwise stay local), per-side cost and latency, tags; aggregate/analysis/decision/economics (numbers, not content); methodology notes; the insights narrative (prose that can quote the regressions it summarizes); the evaluator config with every prompt body replaced by a `content_hash` (prompt names, file paths and variable names do ship, and so does each `llm_judge` `criterion_prompt`); and a dataset snapshot holding only metadata plus an `examples_hash`. Request metadata beside the bundle: the bearer token as an auth header to the configured host only, and the compressed size. @@ -858,7 +869,7 @@ Common conventions: `-c/--config` defaults to `./evalshift.yaml`; run artefacts `-f/--force` · `-d/--directory ` · `--ci` · `--wire-agents/--no-wire-agents` (default on) · `--provider gemini|openai|anthropic|deepseek` · `--profile model-upgrade|cost-reduction|local-model|quantization|provider-switch` (default `model-upgrade`) Without `--ci`, warns after writing when an existing workflow under `.github/workflows/` pins an older or newer CLI than this one, or none at all (see [Pin drift](#pin-drift)); `init --ci` writes the pin itself and does not warn about the file it just wrote. -**`evalshift doctor`** — environment/config check. Exit 1 only on an invalid existing config. Row 2, `evalshift-sdk`, confirms `import evalshift` is the SDK (`warn` when missing or shadowed, never a failure). Reports the toolset each configured suite carries and flags a suite whose examples carry more than one distinct toolset. The suite-side checks cover every suite in the config's `suites:` block, falling back to `./golden.jsonl` when none are wired. Adds a `ci pin` row when a workflow uses the GitHub Action (`warn` on pin drift, never a failure) and a `judge family` row when an `llm_judge` judge shares a provider with a configured arm (`warn`, never a failure). +**`evalshift doctor`** — environment/config check. Exit 1 only on an invalid existing config or a `captures.store` whose client extra is missing. Row 2, `evalshift-sdk`, confirms `import evalshift` is the SDK (`warn` when missing or shadowed, never a failure). Reports the toolset each configured suite carries and flags a suite whose examples carry more than one distinct toolset. The suite-side checks cover every suite in the config's `suites:` block, falling back to `./golden.jsonl` when none are wired. Adds a `ci pin` row when a workflow uses the GitHub Action (`warn` on pin drift, never a failure) and a `judge family` row when an `llm_judge` judge shares a provider with a configured arm (`warn`, never a failure). Adds a `captures.store` row when the config names a store (`fail` on a missing extra, `warn` when the bucket cannot be listed). **`evalshift run`** — paired evaluation run (costs money — calls real models). `-f/--from ` · `-t/--to ` · `-c/--config` · `-s/--suite ` · `--suite-name ` · `--resume` · `-y/--yes` @@ -884,9 +895,10 @@ All `run` flags, plus `--gate` · `--policy-gate` · `--open` · `--push` · `-- ### Captures -**`evalshift capture list [suite]`** — `--json` +**`evalshift capture list [suite]`** — `--json` · `-c/--config` · `--offline` +**`evalshift capture fetch`** — `--suite` · `--since <30m|24h|7d|ISO>` · `-c/--config` (needs `captures.store`) **`evalshift capture promote `** — `--as ` · `--suite` · `--input-var` (default `input`) · `--tag` (repeatable) · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` (default `first`) · `--allow-errored` · `-f/--force` -**`evalshift capture sync`** — `--suite` · `--input-var` · `--tag` · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` · `--allow-errored` · `-c/--config` · `-f/--force` · `--write/--print` (default write) · `--keep-duplicates` +**`evalshift capture sync`** — `--suite` · `--input-var` · `--tag` · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` · `--allow-errored` · `-c/--config` · `-f/--force` · `--write/--print` (default write) · `--keep-duplicates` · `--since` · `--offline` **`evalshift capture clean [suite]`** — `--promoted` (default) · `--all` · `-y/--yes` **`evalshift capture diff `** @@ -1026,7 +1038,7 @@ No. EvalShift compares model behaviour, not framework code: prompts come from co ### Where does my data go? -Nowhere, by default. Model inputs/outputs go to the providers you configured (that's the point); everything else stays under `.evalshift/` and `~/.evalshift/`. The hosted service only sees what `push` explicitly uploads. One thing worth knowing: [run insights](#run-insights) sends the worst regressions' inputs and outputs to `defaults.insights_model` — the same exposure as an `llm_judge` criterion. `--no-insights` turns it off. +Nowhere, by default. Model inputs/outputs go to the providers you configured (that's the point); everything else stays under `.evalshift/` and `~/.evalshift/`. The hosted service only sees what `push` explicitly uploads. One thing worth knowing: [run insights](#run-insights) sends the worst regressions' inputs and outputs to `defaults.insights_model` — the same exposure as an `llm_judge` criterion. `--no-insights` turns it off. With `captures.store` configured, `capture sync`, `list` and `fetch` read from the bucket *you* named using *your* cloud credentials; nothing is sent to EvalShift, and `doctor` reports whether that store is reachable. --- diff --git a/README.md b/README.md index 6d8bbc08..36259ee2 100644 --- a/README.md +++ b/README.md @@ -310,7 +310,8 @@ assets, works offline) has: Your prompts and suite stay local for `doctor`, `run`, `evaluate`, `analyze`, and `report`. The only outbound calls in local mode are to the LLM providers you -configure — any provider LiteLLM supports, called with your own API keys. +configure — any provider LiteLLM supports, called with your own API keys — and, +if you set `captures.store`, reads from your own capture bucket. Anthropic, DeepSeek, Google and OpenAI ids additionally get a curated pricing and capability entry; everything else is passed through with the provider inferred from the id. diff --git a/docs/getting-started.md b/docs/getting-started.md index b9ef6ec3..092b3dbe 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -123,6 +123,11 @@ Then exercise the agent with capture turned on: EVALSHIFT_CAPTURE=1 python your_agent.py # writes .evalshift/captures/ ``` +Running on Fargate, Lambda or Kubernetes, where the disk does not outlive the task? Add +`EVALSHIFT_SINK=s3:///` (or `gs://…` / `az://…`) next to it and put the same +URI under `captures: {store: …}` in `evalshift.yaml`; step 6 then fetches from the bucket +first. See [`captures`](configuration.md#captures). + Captures are off unless `EVALSHIFT_CAPTURE=1` is set, so the decorator can stay in production code. If the agent calls OpenAI, Anthropic or Google GenAI directly, wrap the client once — `wrap_openai(OpenAI())`, `wrap_anthropic`, diff --git a/docs/hosted.md b/docs/hosted.md index 5069a7c6..e656b24d 100644 --- a/docs/hosted.md +++ b/docs/hosted.md @@ -272,7 +272,7 @@ hand them. The CLI contains **no telemetry**: no analytics, no crash reporting, no phone-home of any kind. It opens exactly two kinds of network connections, -both initiated by you: +both initiated by you — a third only if you configure a capture bucket (below): 1. **Your model providers** (whichever you configure — any provider LiteLLM supports), using your own API keys: `run` sends the rendered prompts and @@ -285,6 +285,11 @@ both initiated by you: commands — `doctor`, `run`, `evaluate`, `analyze`, `report`, `bundle` — send nothing to EvalShift-operated services. +With [`captures.store`](configuration.md#captures) configured, `capture sync`, +`capture list`, `capture fetch` and `doctor` also read from that object store — +a bucket you own, with your own cloud credentials. Nothing from it goes to +EvalShift. + ### What `push` sends, block by block `push` uploads `run_bundle.json.gz` plus two pieces of request metadata: the @@ -322,7 +327,9 @@ The bundle itself contains: imported agent traces (`traces.jsonl`; only the verdicts computed on them upload — `agent_trace` scores and `trace_invariants` rule results — never the timelines; see the `examples[]` row above), the SQLite response cache, - `.evalshift/captures/`, `state.json`, `report.json`, and `report.html`. + `.evalshift/captures/` (and, with `captures.store` configured, the bucket + *you* named that they are mirrored from — read with your credentials, never + uploaded to EvalShift), `state.json`, `report.json`, and `report.html`. The content hashes that replace this data (`dataset_hash`, `examples_hash`, `prompts[].content_hash`) are SHA-256 digests, so hosted diffs and baselines diff --git a/docs/sdk.md b/docs/sdk.md index 61c84673..60cd3bc1 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -14,20 +14,25 @@ dense LLM reference at . ``` your agent (evalshift-sdk) → .evalshift/captures//cap_.json - → evalshift capture sync + → or, EVALSHIFT_SINK=s3://… : /captures//cap_.json + → evalshift capture sync (fetches from captures.store first) → .evalshift/suites//golden.jsonl + evalshift.yaml ``` -- **Disk is the only interface.** The SDK never imports or calls the CLI, and - the CLI reads captures from disk without calling SDK code (`doctor` only - checks which package the `evalshift` import name resolves to). +- **The layout is the interface.** The SDK never imports or calls the CLI, and the CLI reads + captures from disk without calling SDK code (`doctor` only checks which package the + `evalshift` import name resolves to). The same layout — `captures//cap_.json` + plus `toolsets/.json` — is what the SDK writes to local disk *or* under a prefix in an + object store you own; the CLI mirrors the latter into `.evalshift/` and reads it unchanged. - **One environment or two.** The CLI imports as `evalshift_cli` and depends on the SDK, so `pip install evalshift` gives you both and `import evalshift` is always the SDK. A production agent that only records captures installs `evalshift-sdk` alone. - **Off by default.** Nothing is recorded unless `EVALSHIFT_CAPTURE=1` is set, so the instrumentation is safe to leave in production code permanently. -- **No network.** The SDK writes local files only. Python 3.10+, stdlib-only. +- **No network by default.** The SDK writes local files unless `EVALSHIFT_SINK` names a bucket + you own (`s3://`, `gs://`, `az://`, SDK 0.5.0+), and then it writes only there — never to + EvalShift. Python 3.10+, stdlib-only; the cloud clients are optional extras. ## 1. Instrument the agent @@ -103,12 +108,26 @@ newest captures. `EVALSHIFT_MAX_CAPTURES`, `EVALSHIFT_DEDUP`, `EVALSHIFT_CAPTURE_TTL` and `EVALSHIFT_SAMPLE_RATE` tune that; they are read by the SDK, in your agent's process. +```bash +# on a host whose disk does not outlive it (Fargate, Lambda, pods): +EVALSHIFT_CAPTURE=1 EVALSHIFT_SINK=s3://acme-evals/support-agent python your_agent.py +``` + +The SDK ships each capture (and its toolset sidecar) to the bucket on a background thread, +fail-open; see the SDK docs for the `SIGTERM` / Lambda flush note. Bucket retention is a +lifecycle rule; `EVALSHIFT_MAX_CAPTURES` applies to local disk only. + ## 3. Promote captures into suites -Run the CLI from the directory holding `.evalshift/` (or set `EVALSHIFT_DIR`): +Run the CLI from the directory holding `.evalshift/` (or set `EVALSHIFT_DIR`). +If the captures live in a bucket, name it once in `evalshift.yaml` +(`captures: {store: s3://acme-evals/support-agent}`) and install `evalshift[s3]` (or `[gcs]` / +`[azure]`); `list` and `sync` then fetch new captures first. `evalshift capture fetch` does +only that step. ```bash evalshift capture list # what was recorded (--json for machine output) +evalshift capture fetch # mirror new captures from captures.store evalshift capture sync # promote ALL captures → suites + wire evalshift.yaml evalshift capture promote cap_ab12 --as refund_case_1 evalshift capture diff cap_ab12 cap_cd34 diff --git a/llms-full.txt b/llms-full.txt index b0240ac4..9233022f 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -5,9 +5,10 @@ Package: evalshift (PyPI) | CLI entry point: evalshift | version: 1.2.2 Python: >=3.11 | license: Apache-2.0 | status: stable Install: pip install evalshift (or: uv pip install evalshift) -Product flow: evalshift-sdk captures real agent behavior in production -> this CLI replays those -captures against a candidate model (scores, paired statistics, HTML report) -> opt-in hosted -service keeps run history, diffs and PR gates. The SDK is the recommended source of golden +Product flow: evalshift-sdk captures real agent behavior in production (to local disk, or to a +user-owned S3/GCS/Azure bucket) -> this CLI replays those captures against a candidate model +(scores, paired statistics, HTML report) -> opt-in hosted service keeps run history, diffs and +PR gates. The SDK is the recommended source of golden suites; hand-written golden.jsonl is equally supported. Purpose: local-first CLI for safe LLM model migrations. Runs the same prompts on two models @@ -17,7 +18,8 @@ runs paired statistics over the deltas (Shapiro-Wilk screen, paired-t or Wilcoxo with 95% CI, Benjamini-Hochberg FDR at alpha=0.05), classifies severities, computes an optional migration-policy verdict, and renders a single-file HTML report. Everything happens on the user's machine under .evalshift/; the only network traffic is the model API calls plus -opt-in pushes to hosted EvalShift (api.evalshift.dev). +opt-in pushes to hosted EvalShift (api.evalshift.dev) and, with captures.store set, reads from +the user's own capture bucket. Ecosystem — four pieces, one reference each. Fetch the one matching the task: | Piece | What it is | Reference for AI tools | @@ -28,9 +30,10 @@ Ecosystem — four pieces, one reference each. Fetch the one matching the task: Data flow: SDK captures -> CLI runs and bundles -> server stores/diffs -> web app displays. Companion package: evalshift-sdk (separate PyPI package, in-process capture SDK) records -production agent runs as JSON captures under .evalshift/captures//cap_.json; -this CLI promotes them into golden suites (`evalshift capture sync`). Disk is the only -interface between SDK and CLI — they never import or call each other. +production agent runs as JSON captures under .evalshift/captures//cap_.json; or, +with EVALSHIFT_SINK, under the same layout in the user's bucket; +this CLI promotes them into golden suites (`evalshift capture sync`). That file layout is the +only interface between SDK and CLI — they never import or call each other. The CLI imports as `evalshift_cli` and depends on evalshift-sdk, so one environment holds both and `import evalshift` is always the SDK. `python -m evalshift_cli` is the module form of the `evalshift` binary. @@ -150,8 +153,9 @@ evalshift init [-f/--force] [-d/--directory DIR] [--ci] [--wire-agents/--no-wire quantization: .02/0/.97/.005/.02/0/.00/.20 | provider-switch: .03/0/.95/.01/.03/0/.20/.40 evalshift doctor Env/config check. Exit 1 ONLY when an existing evalshift.yaml fails validation (the row shows - the problem count + "run `evalshift validate` for details"; validate prints each problem); - missing API keys are soft warnings (exit 0). + the problem count + "run `evalshift validate` for details"; validate prints each problem) or + its captures.store needs a client extra that is not installed; missing API keys are soft + warnings (exit 0). Row 2 `evalshift-sdk`: ok (" (import name `evalshift`)"); warn (exit 0) when the SDK is not installed, fails to import, or the name resolves to something else (an older CLI's leftover files, a local evalshift/ directory). @@ -171,6 +175,9 @@ evalshift doctor when none overlaps; no row when either arm is unset or no llm_judge is configured. Provider "other" (id the registry cannot place) never matches. Same wording from `validate`; the report repeats it (see Judge family below). + captures.store: fail if the extra is missing (detail names pip install "evalshift[]"; + exit 1), warn if the bucket cannot be listed (exit 0), ok " reachable"; no row without + a store (or when evalshift.yaml is invalid -- its own row reports that). evalshift run [-f/--from MODEL] [-t/--to MODEL] [-c CONFIG] [-s/--suite FILE] [--suite-name NAME] [--resume] [-y/--yes] Paired run over (prompt x example x {source,target}). --from/--to override @@ -293,8 +300,25 @@ evalshift push [RUN_ID] [--bundle PATH] [--project] [--host] [--token] server actually sent are shown (writing the CLI's other defaults would pin values meant to move with the CLI). Stops appearing once evalshift.yaml has its own migration_policy. -Captures (read from .evalshift/captures/ under CWD, or $EVALSHIFT_DIR): -evalshift capture list [SUITE] [--json] +Captures (read from .evalshift/captures/ under CWD, or $EVALSHIFT_DIR; with captures.store set +in evalshift.yaml, list/sync first mirror the bucket into it): +evalshift capture list [SUITE] [--json] [-c CONFIG] [--offline] + With captures.store set, runs fetch first (limited to SUITE when given) unless --offline. + With --json, stdout is pure JSON: the fetch summary and every warning go to stderr. +evalshift capture fetch [--suite S] [--since 30m|24h|7d|ISO] [-c CONFIG] + Mirrors captures.store into : lists toolsets/ then captures/, downloads only objects + missing locally (captures immutable, toolsets content-addressed -> exists == current), skips + capture ids already promoted (clean never causes a re-download), skips captures older than + --since and objects over 32 MB, counts keys not shaped like the SDK layout as malformed + (never written; a key cannot escape ). A suite segment is accepted as the SDK writes + it (spaces, unicode, leading _ fine); one containing ':' (Windows drive / NTFS stream), '/', + '\', NUL or '..', or starting/ending with '.' or space, is malformed. Writes atomically + (tmp + os.replace). Prints "fetched N capture(s) and M toolset(s) from (skipped ...)". + Exits 1 on any list/get failure (files already written stay; a re-run resumes), with a hint + to check credentials/network or use --offline on sync/list. Missing extra -> exit 1 naming + pip install "evalshift[s3|gcs|azure]" (az:// needs azure-storage-blob AND azure-identity, + both in [azure]). No captures.store, or no evalshift.yaml -> exit 1; an invalid + evalshift.yaml -> its load error, exit 1. Has no --offline flag. evalshift capture promote CAPTURE_ID [--as CASE_ID] [--suite S] [--input-var NAME=input] [--tag T]... [--strict-args] [--names-only] [--tool-count] [--rounds first(default)|all] [--allow-errored] [-f/--force] @@ -303,7 +327,11 @@ evalshift capture promote CAPTURE_ID [--as CASE_ID] [--suite S] [--input-var NAM EXITS 1 if the capture's trace carries an error event; --allow-errored promotes anyway. evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--strict-args] [--names-only] [--tool-count] [--rounds first(default)|all] [--allow-errored] [-c CONFIG] [-f] - [--write/--print (default write)] [--keep-duplicates] + [--write/--print (default write)] [--keep-duplicates] [--since 30m|24h|7d|ISO] [--offline] + With captures.store set, sync runs fetch first (same summary line) unless --offline; a fetch + failure or missing extra exits 1 before anything is promoted. --since without a store (or + with --offline) exits 1. An unreadable or invalid evalshift.yaml skips the fetch (warning + only when the file mentions captures) and sync/list continue on the local mirror as before. Promotes EVERY capture: groups by conversation_id, orders by turn_index, one SuiteExample per turn; writes .evalshift/suites//golden.jsonl and rewrites the managed suites: block in evalshift.yaml (between ">>> evalshift suites" markers). Content-duplicate captures skipped by @@ -701,6 +729,13 @@ migration_policy: # optional; fractions not percents retention: max_runs_per_suite: int = 20 (>=0; 0 disables) run_ttl_days: int>=1|null = null +captures: + store: s3:/// | gs:/// | az://// + -- optional block; store validated at load (bad scheme, missing bucket/container, '@' or '?' + -> load error naming the forms; the '@'/'?' error never echoes the value); prefix optional, + trailing slash stripped; default none = local disk only. Extras: evalshift[s3] (boto3; + AWS_ENDPOINT_URL covers MinIO/R2/B2), [gcs] (google-cloud-storage), [azure] + (azure-storage-blob + azure-identity). Credentials: each provider's default chain only. Model ids: built-in registry provides aliases/metadata but NEVER gates — LiteLLM is the call-time authority; any LiteLLM-supported model works. Unknown ids pass through with provider @@ -1083,9 +1118,11 @@ user asks what `push` sends, whether EvalShift phones home, or whether suite/pro content leaves the machine. - The CLI has NO telemetry, analytics, or crash reporting. Only two kinds of outbound traffic, - both user-initiated: (1) the user's own model providers with the user's own keys during - run/evaluate/report (rendered prompts + histories to both models; outputs to embedding/judge - models; worst regressions' inputs+outputs to defaults.insights_model unless --no-insights); + both user-initiated (plus, with captures.store set, reads from the user's own bucket by + capture sync/list/fetch and doctor, with the user's cloud credentials): (1) the user's own + model providers with the user's own keys during run/evaluate/report (rendered prompts + + histories to both models; outputs to embedding/judge models; worst regressions' + inputs+outputs to defaults.insights_model unless --no-insights); (2) the hosted API (https://api.evalshift.dev or --host) on `login`/`whoami`/`push`/ `compare --push` ONLY. doctor/run/evaluate/analyze/report/bundle send nothing to EvalShift. - `push` sends: run_bundle.json.gz + request metadata (bearer token as Authorization header to @@ -1119,7 +1156,8 @@ content leaves the machine. suite conversation histories (`history`, embedded system messages included); tool definitions/schemas (toolsets); raw.jsonl; imported agent traces (traces.jsonl; only the verdicts computed on them upload -- agent_trace scores and trace_invariants rule results -- - never the timelines; see examples[] above); the SQLite response cache; .evalshift/captures/; + never the timelines; see examples[] above); the SQLite response cache; .evalshift/captures/ + (with captures.store, captures are read from the user's own bucket, never sent to EvalShift); state.json; report.json; report.html. - Hashes that replace content (dataset_hash, examples_hash, prompts[].content_hash) are SHA-256 over canonical JSON, so hosted diffs/baselines align without the content. @@ -1262,7 +1300,8 @@ content leaves the machine. - Report is a single self-contained HTML file: no external assets, works offline. - Run pruning never touches an in-progress run or the run just finished. - Exit codes: 0 success; 1 handled errors and tripped CI gates; doctor exits 1 only on invalid - existing config; init exits 2 on unknown --provider; Typer usage errors exit 2. + existing config or a captures.store whose client extra is missing; init exits 2 on unknown + --provider; Typer usage errors exit 2. ## Minimal examples From d93d44ec19ddcdb88078017a4fdec2a11aa87c89 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 20:03:13 +0200 Subject: [PATCH 10/17] fix(capture): warn about a skipped store only for a top-level captures key evalshift init writes "captures" into its comments, so the substring test warned on every invalid init-generated config. Match a column-0 captures: key instead. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- DOCS.md | 2 +- llms-full.txt | 3 ++- src/evalshift_cli/cli/commands/capture.py | 15 ++++++++--- tests/unit/test_cli_capture_remote.py | 33 +++++++++++++++++++++++ 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8673b775..29590a06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 when the extra is missing) and reachable (`warn` when it cannot be listed). Without `captures.store` nothing changes; a `sync` or `list` whose `evalshift.yaml` cannot be read or does not validate skips the fetch, - warning only when the file mentions `captures`. + warning only when the file has a top-level `captures:` key. - `evaluators.trace_invariants`: hand-written rules over tool-call traces — `forbidden`, `required`, `order`, `call_count` (`min_calls` and/or `max_calls` per tool; equal bounds = an exact count) and diff --git a/DOCS.md b/DOCS.md index 5519e33e..9c7a4597 100644 --- a/DOCS.md +++ b/DOCS.md @@ -173,7 +173,7 @@ captures: store: s3://acme-evals/support-agent ``` -and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written, and no key can place a file outside `.evalshift/`. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching extra — `evalshift[s3]`, `[gcs]` or `[azure]` (azure-storage-blob plus azure-identity) — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load without being echoed back. A missing extra exits 1 naming the `pip install "evalshift[]"` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file mentions `captures`. Full field reference: [`captures`](docs/configuration.md#captures). +and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written, and no key can place a file outside `.evalshift/`. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching extra — `evalshift[s3]`, `[gcs]` or `[azure]` (azure-storage-blob plus azure-identity) — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load without being echoed back. A missing extra exits 1 naming the `pip install "evalshift[]"` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). **`capture sync`** is the workhorse. Per suite it: diff --git a/llms-full.txt b/llms-full.txt index 9233022f..ec553400 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -331,7 +331,8 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri With captures.store set, sync runs fetch first (same summary line) unless --offline; a fetch failure or missing extra exits 1 before anything is promoted. --since without a store (or with --offline) exits 1. An unreadable or invalid evalshift.yaml skips the fetch (warning - only when the file mentions captures) and sync/list continue on the local mirror as before. + only when the file has a top-level captures: key) and sync/list continue on the local mirror + as before. Promotes EVERY capture: groups by conversation_id, orders by turn_index, one SuiteExample per turn; writes .evalshift/suites//golden.jsonl and rewrites the managed suites: block in evalshift.yaml (between ">>> evalshift suites" markers). Content-duplicate captures skipped by diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index 4b09bba6..316f943e 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -16,6 +16,7 @@ import json import os +import re from collections.abc import Callable, Iterable, Mapping from datetime import UTC, datetime from pathlib import Path @@ -218,10 +219,18 @@ def _promoted_content_owners( return owners +#: A top-level ``captures:`` key: at column 0, so neither a comment nor a nested key matches. +_TOP_LEVEL_CAPTURES_KEY = re.compile(rb"^captures\s*:", re.MULTILINE) + + def _mentions_captures(config_path: Path) -> bool: - """Whether the raw config file text names ``captures`` at all (``False`` if unreadable).""" + """Whether the raw config file has a top-level ``captures`` key (``False`` if unreadable). + + Structural rather than a substring test: ``evalshift init`` writes "captures" into its + comments, and a config that only mentions the word never asked for a store. + """ try: - return b"captures" in config_path.read_bytes() + return _TOP_LEVEL_CAPTURES_KEY.search(config_path.read_bytes()) is not None except OSError: return False @@ -232,7 +241,7 @@ def _configured_store(config_path: Path, *, console: Console, strict: bool) -> R A missing config file means no store. An invalid one is a hard error when ``strict`` (``capture fetch`` has nothing else to do). Otherwise ``sync`` and ``list`` keep working on the local mirror the way they did before ``captures.store`` existed: an invalid or - unreadable file is skipped, with a warning only when its text mentions ``captures`` (a + unreadable file is skipped, with a warning only when it has a top-level ``captures`` key (a config that never asked for a store must not change their output). A missing client extra is always a hard error: the user asked for a store it cannot reach. """ diff --git a/tests/unit/test_cli_capture_remote.py b/tests/unit/test_cli_capture_remote.py index bdef6c22..171ed1fc 100644 --- a/tests/unit/test_cli_capture_remote.py +++ b/tests/unit/test_cli_capture_remote.py @@ -194,6 +194,39 @@ def test_invalid_config_without_captures_stays_silent( assert remote.gets == [] +@pytest.mark.parametrize("command", ["sync", "list"]) +def test_invalid_init_config_with_captures_only_in_comments_stays_silent( + tmp_path: Path, remote: FakeRemoteStore, command: str +) -> None: + # `evalshift init` output mentions "captures" in comments; that is not a store request. + config = tmp_path / "evalshift.yaml" + config.write_text( + render_minimal_config(profile="model-upgrade") + "\ntypo_key: 1\n", encoding="utf-8" + ) + assert b"captures" in config.read_bytes() + result = _invoke([command, "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.output + assert "skipping captures.store" not in result.output + assert "could not read" not in result.output + assert remote.gets == [] + + +@pytest.mark.parametrize("command", ["sync", "list"]) +def test_invalid_init_config_with_top_level_captures_warns( + tmp_path: Path, remote: FakeRemoteStore, command: str +) -> None: + config = tmp_path / "evalshift.yaml" + config.write_text( + render_minimal_config(profile="model-upgrade") + + "\ntypo_key: 1\ncaptures:\n store: s3://b/p\n", + encoding="utf-8", + ) + result = _invoke([command, "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.output + assert "skipping captures.store" in result.output + assert remote.gets == [] + + def test_list_with_undecodable_config_stays_local_without_traceback( tmp_path: Path, remote: FakeRemoteStore ) -> None: From 51cc97d63a10f64459f8c75df074190e56c76147 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 20:04:31 +0200 Subject: [PATCH 11/17] fix(config): never echo a rejected captures.store value The unsupported-scheme, no-bucket and Azure no-container messages quoted the whole value, so a pasted Azure connection string landed in the config error and CI logs. Name only the parsed scheme. Message strings match the SDK. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- DOCS.md | 2 +- docs/configuration.md | 3 +- llms-full.txt | 2 +- src/evalshift_cli/captures/store_uri.py | 17 ++++++---- tests/unit/test_captures_store_uri.py | 41 +++++++++++++++++++++++++ tests/unit/test_config_captures.py | 5 +++ 7 files changed, 63 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29590a06..d596dfd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `evalshift[s3]`, `[gcs]`, `[azure]` (azure-storage-blob and azure-identity); a missing one exits 1 naming the `pip install` to run. Credentials come from each provider's default chain and are rejected in the URI: a value with `@` - or `?` fails to load, naming the accepted forms without echoing it. + or `?` fails to load, naming the accepted forms. No rejected value is echoed + back (it may be a pasted key or connection string); only its scheme is named. `capture list --json` keeps stdout pure JSON (the fetch line and warnings go to stderr). `doctor` reports whether the store is installed (`fail`, exit 1, when the extra is missing) and reachable (`warn` when it cannot be listed). diff --git a/DOCS.md b/DOCS.md index 9c7a4597..06348c37 100644 --- a/DOCS.md +++ b/DOCS.md @@ -173,7 +173,7 @@ captures: store: s3://acme-evals/support-agent ``` -and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written, and no key can place a file outside `.evalshift/`. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching extra — `evalshift[s3]`, `[gcs]` or `[azure]` (azure-storage-blob plus azure-identity) — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load without being echoed back. A missing extra exits 1 naming the `pip install "evalshift[]"` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). +and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written, and no key can place a file outside `.evalshift/`. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching extra — `evalshift[s3]`, `[gcs]` or `[azure]` (azure-storage-blob plus azure-identity) — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load; no rejected value is echoed back, only its scheme. A missing extra exits 1 naming the `pip install "evalshift[]"` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). **`capture sync`** is the workhorse. Per suite it: diff --git a/docs/configuration.md b/docs/configuration.md index 9a73ec72..c351f375 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1043,7 +1043,8 @@ Accepted URI forms — the same grammar the SDK uses: | `az:////` | Azure Blob Storage | `pip install "evalshift[azure]"` | Managed Identity / federated credential in CI, `az login` locally | `` is optional. Credentials never go in the URI: a value containing `@` or `?` fails -to load, naming the accepted forms. Under the prefix the layout is exactly the local one — +to load, naming the accepted forms. A rejected value is never echoed back (it may be a pasted +key or connection string); the error names only its scheme. Under the prefix the layout is exactly the local one — `captures//cap_.json` and `toolsets/.json` — so the bucket is a mirror of `.evalshift/`, not a different format. diff --git a/llms-full.txt b/llms-full.txt index ec553400..5755047a 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -733,7 +733,7 @@ retention: captures: store: s3:/// | gs:/// | az://// -- optional block; store validated at load (bad scheme, missing bucket/container, '@' or '?' - -> load error naming the forms; the '@'/'?' error never echoes the value); prefix optional, + -> load error naming the forms; no error echoes the value, only its scheme); prefix optional, trailing slash stripped; default none = local disk only. Extras: evalshift[s3] (boto3; AWS_ENDPOINT_URL covers MinIO/R2/B2), [gcs] (google-cloud-storage), [azure] (azure-storage-blob + azure-identity). Credentials: each provider's default chain only. diff --git a/src/evalshift_cli/captures/store_uri.py b/src/evalshift_cli/captures/store_uri.py index 8a714779..3de1b480 100644 --- a/src/evalshift_cli/captures/store_uri.py +++ b/src/evalshift_cli/captures/store_uri.py @@ -58,8 +58,8 @@ def parse_store_uri(uri: str) -> StoreURI: Raises: ValueError: for an unknown scheme, a missing bucket or container, or a URI carrying credentials (``@``) or query parameters (``?``). The message names the accepted - forms so a config error is self-explanatory; the ``@`` and ``?`` messages never - echo the URI, since that is exactly where a key or signature would sit. + forms so a config error is self-explanatory, and never echoes the value: a rejected + value may be a key, a signature or a pasted connection string. """ text = uri.strip() if "@" in text: @@ -71,19 +71,24 @@ def parse_store_uri(uri: str) -> StoreURI: raise ValueError( "query parameters are not accepted in a store URI; use the provider's credential chain" ) + # No message below echoes the value either: a rejected value may be a pasted connection + # string or key. Only the parsed scheme, which cannot hold one, is quoted back. parts = urlsplit(text) + if not parts.scheme: + raise ValueError(f"store URI has no scheme; accepted forms: {STORE_URI_FORMS}") if parts.scheme not in _EXTRA_FOR_SCHEME: - raise ValueError(f"unsupported store URI {text!r}; accepted forms: {STORE_URI_FORMS}") + raise ValueError( + f"unsupported store URI scheme {parts.scheme!r}; accepted forms: {STORE_URI_FORMS}" + ) if not parts.netloc: - raise ValueError(f"store URI {text!r} names no bucket; accepted forms: {STORE_URI_FORMS}") + raise ValueError(f"store URI names no bucket; accepted forms: {STORE_URI_FORMS}") scheme = cast(Scheme, parts.scheme) path = parts.path.strip("/") if scheme == "az": container, _, prefix = path.partition("/") if not container: raise ValueError( - f"Azure store URI {text!r} names no container; " - "expected az:////" + "Azure store URI names no container; expected az:////" ) return StoreURI(scheme, parts.netloc, container, prefix.strip("/")) return StoreURI(scheme, parts.netloc, None, path) diff --git a/tests/unit/test_captures_store_uri.py b/tests/unit/test_captures_store_uri.py index ce1319d9..05a1e47f 100644 --- a/tests/unit/test_captures_store_uri.py +++ b/tests/unit/test_captures_store_uri.py @@ -63,3 +63,44 @@ def test_credential_rejection_does_not_echo_secret(uri: str, secret: str) -> Non with pytest.raises(ValueError, match="credential chain") as excinfo: parse_store_uri(uri) assert secret not in str(excinfo.value) + + +@pytest.mark.parametrize( + ("uri", "fragment", "secret"), + [ + # An Azure connection string pasted as the store: no `@` or `?`, so it reaches the + # scheme check, and that message must not echo it. + ( + "DefaultEndpointsProtocol=https;AccountName=x;AccountKey=SUPERSECRET==", + "accepted forms", + "SUPERSECRET", + ), + ("https://acct.blob.core.windows.net/c/SUPERSECRET", "accepted forms", "SUPERSECRET"), + ("s3:///AKIASUPERSECRET/key", "names no bucket", "SUPERSECRET"), + ("az://SUPERSECRETACCOUNT/", "names no container", "SUPERSECRET"), + ], +) +def test_grammar_rejection_does_not_echo_value(uri: str, fragment: str, secret: str) -> None: + with pytest.raises(ValueError, match=fragment) as excinfo: + parse_store_uri(uri) + assert secret not in str(excinfo.value) + + +def test_grammar_rejection_messages() -> None: + with pytest.raises(ValueError) as unsupported: + parse_store_uri("ftp://bucket/prefix") + assert str(unsupported.value) == ( + "unsupported store URI scheme 'ftp'; accepted forms: " + "s3:///, gs:///, az:////" + ) + with pytest.raises(ValueError) as no_scheme: + parse_store_uri("bucket/prefix") + assert str(no_scheme.value).startswith("store URI has no scheme; accepted forms: s3://") + with pytest.raises(ValueError) as no_bucket: + parse_store_uri("s3:///prefix") + assert str(no_bucket.value).startswith("store URI names no bucket; accepted forms: s3://") + with pytest.raises(ValueError) as no_container: + parse_store_uri("az://acmeprod") + assert str(no_container.value) == ( + "Azure store URI names no container; expected az:////" + ) diff --git a/tests/unit/test_config_captures.py b/tests/unit/test_config_captures.py index 1cfa65dc..e9474a17 100644 --- a/tests/unit/test_config_captures.py +++ b/tests/unit/test_config_captures.py @@ -69,6 +69,11 @@ def test_loader_reports_store_error_at_its_location(tmp_path: Path) -> None: [ ("s3://AKIAKEY:SECRET@b/p", ["AKIAKEY", "SECRET"]), ("az://acct/c?sv=1&sig=SECRETSIG", ["SECRETSIG", "sig="]), + ( + "DefaultEndpointsProtocol=https;AccountName=x;AccountKey=SUPERSECRET==", + ["SUPERSECRET", "AccountKey"], + ), + ("s3:///AKIASUPERSECRET/key", ["SUPERSECRET"]), ], ) def test_loader_error_never_echoes_a_secret_store( From 06d337604fa28a173f49f58f1c561d5f61145588 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 20:06:30 +0200 Subject: [PATCH 12/17] fix(capture): per-cloud fetch hints, local write errors, strict bucket keys - A list or download failure's hint names the scheme's credential chain and points --offline at capture sync / capture list (capture fetch has none). - A local write failure (mkdir, temp file, write, rename) raises its own error naming the file, with a disk-space-and-permissions hint. - Capture and toolset basenames use fullmatch, so a key with a trailing newline is malformed instead of written. - Folder-marker keys ending in / are skipped without counting as malformed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 +++- DOCS.md | 2 +- llms-full.txt | 8 ++- src/evalshift_cli/captures/remote.py | 63 +++++++++++++++++----- tests/unit/test_captures_remote.py | 81 ++++++++++++++++++++++++++++ 5 files changed, 146 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d596dfd7..ab7f4c4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `--offline` skips it. Keys from a bucket are validated against the SDK's layout and anything else is ignored as malformed — including a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with - `.` or a space — and no destination can land outside `.evalshift/`. Extras: + `.` or a space, and a key with a trailing newline — and no destination can + land outside `.evalshift/`; folder-marker keys ending in `/` (the S3 + console's "Create folder", ADLS directories) are skipped without being + counted. A list or download failure exits 1 with a hint naming the + provider's credential chain (`aws sso login` / `AWS_PROFILE` / the CI OIDC + role, `gcloud auth application-default login`, `az login`); a local write + failure names the file and asks you to check disk space and permissions on + its directory instead. Extras: `evalshift[s3]`, `[gcs]`, `[azure]` (azure-storage-blob and azure-identity); a missing one exits 1 naming the `pip install` to run. Credentials come from each provider's default chain and are rejected in the URI: a value with `@` diff --git a/DOCS.md b/DOCS.md index 06348c37..08e38d4f 100644 --- a/DOCS.md +++ b/DOCS.md @@ -173,7 +173,7 @@ captures: store: s3://acme-evals/support-agent ``` -and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written, and no key can place a file outside `.evalshift/`. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching extra — `evalshift[s3]`, `[gcs]` or `[azure]` (azure-storage-blob plus azure-identity) — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load; no rejected value is echoed back, only its scheme. A missing extra exits 1 naming the `pip install "evalshift[]"` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). +and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written (a key with a trailing newline included), and no key can place a file outside `.evalshift/`; folder-marker keys ending in `/` (the S3 console's "Create folder", ADLS directories) are skipped without being counted. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching extra — `evalshift[s3]`, `[gcs]` or `[azure]` (azure-storage-blob plus azure-identity) — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load; no rejected value is echoed back, only its scheme. A missing extra exits 1 naming the `pip install "evalshift[]"` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch, with a hint naming the provider's credential chain (`aws sso login` / `AWS_PROFILE` / the CI OIDC role, `gcloud auth application-default login`, `az login`); a local write failure names the file and asks you to check free disk space and permissions instead. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). **`capture sync`** is the workhorse. Per suite it: diff --git a/llms-full.txt b/llms-full.txt index 5755047a..9f411744 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -310,12 +310,16 @@ evalshift capture fetch [--suite S] [--since 30m|24h|7d|ISO] [-c CONFIG] missing locally (captures immutable, toolsets content-addressed -> exists == current), skips capture ids already promoted (clean never causes a re-download), skips captures older than --since and objects over 32 MB, counts keys not shaped like the SDK layout as malformed - (never written; a key cannot escape ). A suite segment is accepted as the SDK writes + (never written; a key cannot escape ; a trailing newline makes a key malformed; + folder-marker keys ending in '/' are skipped without counting). A suite segment is accepted as the SDK writes it (spaces, unicode, leading _ fine); one containing ':' (Windows drive / NTFS stream), '/', '\', NUL or '..', or starting/ending with '.' or space, is malformed. Writes atomically (tmp + os.replace). Prints "fetched N capture(s) and M toolset(s) from (skipped ...)". Exits 1 on any list/get failure (files already written stay; a re-run resumes), with a hint - to check credentials/network or use --offline on sync/list. Missing extra -> exit 1 naming + naming the scheme's credential chain (s3: aws sso login / AWS_PROFILE / the CI OIDC role; + gs: gcloud auth application-default login / Workload Identity; az: az login / managed + identity) and --offline on sync/list. A local write failure exits 1 naming the file, with a + hint to check free disk space and permissions on its directory. Missing extra -> exit 1 naming pip install "evalshift[s3|gcs|azure]" (az:// needs azure-storage-blob AND azure-identity, both in [azure]). No captures.store, or no evalshift.yaml -> exit 1; an invalid evalshift.yaml -> its load error, exit 1. Has no --offline flag. diff --git a/src/evalshift_cli/captures/remote.py b/src/evalshift_cli/captures/remote.py index aab3b51a..ac8c0d91 100644 --- a/src/evalshift_cli/captures/remote.py +++ b/src/evalshift_cli/captures/remote.py @@ -44,9 +44,18 @@ #: Largest object a fetch will download. Captures are kilobytes; this is a safety stop. MAX_OBJECT_BYTES = 32 * 1024 * 1024 -_CAPTURE_NAME = re.compile(r"^[A-Za-z0-9_-]+\.json$") -_TOOLSET_NAME = re.compile(r"^[0-9a-f]{64}\.json$") -_DURATION = re.compile(r"^(\d+)([mhd])$") +# Always used with ``fullmatch``: ``$`` also matches before a trailing newline. +_CAPTURE_NAME = re.compile(r"[A-Za-z0-9_-]+\.json") +_TOOLSET_NAME = re.compile(r"[0-9a-f]{64}\.json") +_DURATION = re.compile(r"(\d+)([mhd])") + +#: How to fix credentials, per store scheme, for a list or download failure. +_CREDENTIALS_HINT: dict[str, str] = { + "s3": "check AWS credentials (aws sso login / AWS_PROFILE / the CI OIDC role) and network", + "gs": "check Google credentials (gcloud auth application-default login / Workload Identity) " + "and network", + "az": "check Azure credentials (az login / managed identity) and network", +} #: Every module a scheme's pip extra provides; ``open_store`` checks all of them. _REQUIRED_MODULES: dict[str, tuple[str, ...]] = { @@ -219,7 +228,7 @@ def parse_since(text: str, *, now: datetime | None = None) -> datetime: """ reference = now if now is not None else datetime.now(UTC) cleaned = text.strip() - match = _DURATION.match(cleaned) + match = _DURATION.fullmatch(cleaned) if match: amount, unit = int(match.group(1)), match.group(2) delta = { @@ -266,8 +275,10 @@ def fetch_captures( toolset_jobs: list[tuple[str, Path]] = [] for info in _list(store, "toolsets/"): + if info.key.endswith("/"): + continue # a folder marker (S3 console "Create folder", ADLS directory), not an object name = info.key.removeprefix("toolsets/") - if "/" in name or not _TOOLSET_NAME.match(name): + if "/" in name or not _TOOLSET_NAME.fullmatch(name): summary.skipped_malformed += 1 continue dest = toolsets_dir / name @@ -284,13 +295,15 @@ def fetch_captures( capture_jobs: list[tuple[str, Path]] = [] prefix = f"captures/{suite}/" if suite else "captures/" for info in _list(store, prefix): + if info.key.endswith("/"): + continue # a folder marker, as above rel = info.key.removeprefix("captures/") suite_seg, sep, name = rel.partition("/") if ( not sep or "/" in name or not _is_safe_suite_segment(suite_seg) - or not _CAPTURE_NAME.match(name) + or not _CAPTURE_NAME.fullmatch(name) ): summary.skipped_malformed += 1 continue @@ -328,6 +341,21 @@ def _absolute(path: Path) -> Path: return Path(os.path.abspath(path)) +def _fetch_failure_hint(store: RemoteStore) -> str: + """The hint for a failed list or download: the scheme's credential chain, then --offline. + + ``capture fetch`` has no ``--offline``, so the advice names the commands that do. + """ + scheme, separator, _ = store.uri.partition("://") + credentials = _CREDENTIALS_HINT.get( + scheme if separator else "", "check credentials and network" + ) + return ( + f"{credentials}, or run `capture sync` / `capture list` with --offline " + "to use the captures already on disk" + ) + + def _list(store: RemoteStore, prefix: str) -> list[ObjectInfo]: try: return list(store.list(prefix)) @@ -336,8 +364,7 @@ def _list(store: RemoteStore, prefix: str) -> list[ObjectInfo]: except Exception as exc: raise RemoteStoreError( f"could not list {prefix!r} in {store.uri}: {type(exc).__name__}: {exc}", - hint="check credentials and network, or use --offline to work with the captures " - "already on disk", + hint=_fetch_failure_hint(store), ) from exc @@ -355,8 +382,7 @@ def _download_all( except Exception as exc: raise RemoteStoreError( f"could not download from {store.uri}: {type(exc).__name__}: {exc}", - hint="check credentials and network, or use --offline to work with the captures " - "already on disk", + hint=_fetch_failure_hint(store), ) from exc written = sum(1 for ok in results if ok) return written, len(results) - written @@ -369,11 +395,23 @@ def _download(store: RemoteStore, key: str, dest: Path) -> bool: except Exception as exc: raise RemoteStoreError( f"could not download {key} from {store.uri}: {type(exc).__name__}: {exc}", - hint="check credentials and network, or use --offline to work with the captures " - "already on disk", + hint=_fetch_failure_hint(store), ) from exc if len(data) > MAX_OBJECT_BYTES: return False + try: + _write_atomically(dest, data) + except OSError as exc: + # A local failure: the store answered, so credentials and network are not the problem. + raise RemoteStoreError( + f"could not write {dest}: {type(exc).__name__}: {exc}", + hint=f"check free disk space and permissions on {dest.parent}", + ) from exc + return True + + +def _write_atomically(dest: Path, data: bytes) -> None: + """Write ``data`` to ``dest`` via a temp file beside it and :func:`os.replace`.""" dest.parent.mkdir(parents=True, exist_ok=True) fd, tmp_name = tempfile.mkstemp(dir=dest.parent, prefix=".", suffix=".tmp") try: @@ -383,7 +421,6 @@ def _download(store: RemoteStore, key: str, dest: Path) -> bool: finally: with suppress(OSError): os.remove(tmp_name) - return True __all__ = [ diff --git a/tests/unit/test_captures_remote.py b/tests/unit/test_captures_remote.py index 975d5070..83a72400 100644 --- a/tests/unit/test_captures_remote.py +++ b/tests/unit/test_captures_remote.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Iterator from datetime import UTC, datetime, timedelta from pathlib import Path @@ -11,6 +12,7 @@ from evalshift_cli.captures.remote import ( MAX_OBJECT_BYTES, FetchSummary, + ObjectInfo, RemoteStore, RemoteStoreError, _is_safe_suite_segment, @@ -236,6 +238,85 @@ def failing_replace(src: str, dst: str) -> None: assert [p for p in tmp_path.rglob("*") if p.is_file()] == [] +def test_fetch_counts_keys_with_a_trailing_newline_as_malformed(tmp_path: Path) -> None: + store = FakeRemoteStore( + { + "captures/alpha/cap_1.json\n": capture_payload("cap_1"), + f"toolsets/{TOOLSET_HEX}.json\n": TOOLSET_PAYLOAD, + } + ) + summary = fetch_captures(store, base=tmp_path) + assert summary.skipped_malformed == 2 + assert summary.captures == summary.toolsets == 0 + assert list(tmp_path.rglob("*")) == [] + assert store.gets == [] + + +def test_fetch_skips_folder_marker_keys_silently(tmp_path: Path) -> None: + # S3 console "Create folder" and ADLS directory entries list as zero-byte "/" keys. + store = populated_store("cap_1") + store.objects.update( + {"captures/": b"", "captures/alpha/": b"", "toolsets/": b"", "captures/beta/": b""} + ) + summary = fetch_captures(store, base=tmp_path) + assert summary.skipped_malformed == 0 + assert summary.captures == 1 and summary.toolsets == 1 + assert _local_captures(tmp_path) == {"alpha/cap_1.json"} + assert not any(key.endswith("/") for key in store.gets) + + +class _ListFailingStore(FakeRemoteStore): + def list(self, prefix: str) -> Iterator[ObjectInfo]: + raise ConnectionError("simulated listing failure") + + +@pytest.mark.parametrize( + ("uri", "credentials"), + [ + ("s3://b/p", "check AWS credentials (aws sso login / AWS_PROFILE / the CI OIDC role)"), + ( + "gs://b/p", + "check Google credentials (gcloud auth application-default login / Workload Identity)", + ), + ("az://a/c/p", "check Azure credentials (az login / managed identity)"), + ("mem://x", "check credentials"), + ], +) +@pytest.mark.parametrize("failure", ["list", "get"]) +def test_fetch_failure_hint_names_the_scheme_credentials( + tmp_path: Path, uri: str, credentials: str, failure: str +) -> None: + if failure == "list": + store: FakeRemoteStore = _ListFailingStore() + else: + store = populated_store("cap_1") + store.fail_keys.add("captures/alpha/cap_1.json") + store.uri = uri + with pytest.raises(RemoteStoreError) as info: + fetch_captures(store, base=tmp_path, workers=1) + assert info.value.hint == ( + f"{credentials} and network, or run `capture sync` / `capture list` with --offline " + "to use the captures already on disk" + ) + + +def test_fetch_local_write_failure_names_the_path_not_credentials( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def failing_mkstemp(*args: object, **kwargs: object) -> tuple[int, str]: + raise OSError(28, "No space left on device") + + monkeypatch.setattr(remote.tempfile, "mkstemp", failing_mkstemp) + store = populated_store("cap_1") + with pytest.raises(RemoteStoreError) as info: + fetch_captures(store, base=tmp_path, workers=1) + toolsets_dir = tmp_path / "toolsets" + assert str(toolsets_dir / f"{TOOLSET_HEX}.json") in info.value.summary + assert "No space left on device" in info.value.summary + assert info.value.hint == f"check free disk space and permissions on {toolsets_dir}" + assert "credentials" not in str(info.value) + + def test_summary_describe() -> None: full = FetchSummary( store_uri="s3://fake/prefix", captures=2, toolsets=1, skipped_existing=3, skipped_promoted=1 From bd064f2201c9daa681fb00b3e87406a535ddc94d Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 20:07:02 +0200 Subject: [PATCH 13/17] fix(capture): reject an out-of-range --since duration instead of a traceback parse_since("99999999d") raised OverflowError; it now raises the usual --since ValueError, so the CLI exits 2 with its message. Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/remote.py | 14 +++++--------- tests/unit/test_captures_remote.py | 6 ++++++ tests/unit/test_cli_capture_remote.py | 5 +++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/evalshift_cli/captures/remote.py b/src/evalshift_cli/captures/remote.py index ac8c0d91..4f3b7ba9 100644 --- a/src/evalshift_cli/captures/remote.py +++ b/src/evalshift_cli/captures/remote.py @@ -229,17 +229,13 @@ def parse_since(text: str, *, now: datetime | None = None) -> datetime: reference = now if now is not None else datetime.now(UTC) cleaned = text.strip() match = _DURATION.fullmatch(cleaned) - if match: - amount, unit = int(match.group(1)), match.group(2) - delta = { - "m": timedelta(minutes=amount), - "h": timedelta(hours=amount), - "d": timedelta(days=amount), - } - return reference - delta[unit] try: + if match: + amount, unit = int(match.group(1)), match.group(2) + delta = {"m": timedelta(minutes=1), "h": timedelta(hours=1), "d": timedelta(days=1)} + return reference - delta[unit] * amount parsed = datetime.fromisoformat(cleaned) - except ValueError as exc: + except (ValueError, OverflowError) as exc: # OverflowError: a duration reaching before year 1 raise ValueError( f"--since expects a duration like 30m, 24h, 7d or an ISO date/datetime, got {text!r}" ) from exc diff --git a/tests/unit/test_captures_remote.py b/tests/unit/test_captures_remote.py index 83a72400..4b2988cf 100644 --- a/tests/unit/test_captures_remote.py +++ b/tests/unit/test_captures_remote.py @@ -354,3 +354,9 @@ def test_parse_since_iso_date_and_datetime() -> None: def test_parse_since_rejects_garbage(text: str) -> None: with pytest.raises(ValueError, match="--since expects"): parse_since(text) + + +@pytest.mark.parametrize("text", ["99999999d", "9999999999999h", "99999999999999999999m"]) +def test_parse_since_rejects_out_of_range_durations(text: str) -> None: + with pytest.raises(ValueError, match="--since expects"): + parse_since(text) diff --git a/tests/unit/test_cli_capture_remote.py b/tests/unit/test_cli_capture_remote.py index 171ed1fc..feaebd35 100644 --- a/tests/unit/test_cli_capture_remote.py +++ b/tests/unit/test_cli_capture_remote.py @@ -99,9 +99,10 @@ def _fake_fetch(store: RemoteStore, **kwargs: Any) -> Any: assert seen["base"] == tmp_path -def test_fetch_rejects_bad_since(tmp_path: Path, remote: FakeRemoteStore) -> None: +@pytest.mark.parametrize("since", ["yesterday", "99999999d"]) +def test_fetch_rejects_bad_since(tmp_path: Path, remote: FakeRemoteStore, since: str) -> None: config = _write_config(tmp_path / "evalshift.yaml") - result = _invoke(["fetch", "--since", "yesterday", "--config", str(config)], tmp_path) + result = _invoke(["fetch", "--since", since, "--config", str(config)], tmp_path) assert result.exit_code == 2 assert "--since expects" in result.output From a5c53caef40cea2280bac09b7cd58c0ef79d58a3 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 20:07:35 +0200 Subject: [PATCH 14/17] fix(capture): exit 1 with one line when capture fetch cannot read the config An unreadable or non-UTF-8 evalshift.yaml re-raised OSError or UnicodeDecodeError from capture fetch as a traceback. Print the escaped path and the error type only, and exit 1. Co-Authored-By: Claude Opus 5.5 --- llms-full.txt | 3 ++- src/evalshift_cli/cli/commands/capture.py | 12 +++++++--- tests/unit/test_cli_capture_remote.py | 28 +++++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/llms-full.txt b/llms-full.txt index 9f411744..7490adb9 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -322,7 +322,8 @@ evalshift capture fetch [--suite S] [--since 30m|24h|7d|ISO] [-c CONFIG] hint to check free disk space and permissions on its directory. Missing extra -> exit 1 naming pip install "evalshift[s3|gcs|azure]" (az:// needs azure-storage-blob AND azure-identity, both in [azure]). No captures.store, or no evalshift.yaml -> exit 1; an invalid - evalshift.yaml -> its load error, exit 1. Has no --offline flag. + evalshift.yaml -> its load error, exit 1; an unreadable or non-UTF-8 one -> a one-line + "could not read " error, exit 1. Has no --offline flag. evalshift capture promote CAPTURE_ID [--as CASE_ID] [--suite S] [--input-var NAME=input] [--tag T]... [--strict-args] [--names-only] [--tool-count] [--rounds first(default)|all] [--allow-errored] [-f/--force] diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index 316f943e..7c6994bb 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -238,8 +238,8 @@ def _mentions_captures(config_path: Path) -> bool: def _configured_store(config_path: Path, *, console: Console, strict: bool) -> RemoteStore | None: """The store ``captures.store`` names, or ``None`` when there is none to fetch from. - A missing config file means no store. An invalid one is a hard error when ``strict`` - (``capture fetch`` has nothing else to do). Otherwise ``sync`` and ``list`` keep working on + A missing config file means no store. An invalid or unreadable one is a hard error (exit 1, + one line, no traceback) when ``strict`` (``capture fetch`` has nothing else to do). Otherwise ``sync`` and ``list`` keep working on the local mirror the way they did before ``captures.store`` existed: an invalid or unreadable file is skipped, with a warning only when it has a top-level ``captures`` key (a config that never asked for a store must not change their output). A missing client extra @@ -257,7 +257,13 @@ def _configured_store(config_path: Path, *, console: Console, strict: bool) -> R return None except (OSError, UnicodeDecodeError) as exc: if strict: - raise + # The exception's type only: its text can carry raw bytes of the file. + console.print( + f"[red]✗[/red] could not read {escape(str(config_path))} " + f"({type(exc).__name__}); capture fetch needs a readable evalshift.yaml.", + soft_wrap=True, + ) + raise typer.Exit(code=1) from exc _warn_store_skipped(config_path, type(exc).__name__, console=console) return None if cfg.captures.store is None: diff --git a/tests/unit/test_cli_capture_remote.py b/tests/unit/test_cli_capture_remote.py index feaebd35..00e2eb02 100644 --- a/tests/unit/test_cli_capture_remote.py +++ b/tests/unit/test_cli_capture_remote.py @@ -99,6 +99,34 @@ def _fake_fetch(store: RemoteStore, **kwargs: Any) -> Any: assert seen["base"] == tmp_path +def test_fetch_undecodable_config_exits_1_without_traceback( + tmp_path: Path, remote: FakeRemoteStore +) -> None: + config = tmp_path / "evalshift[x].yaml" + config.write_bytes(b"captures:\n store: s3://b/\xff\xfeSECRET\n") + result = _invoke(["fetch", "--config", str(config)], tmp_path) + assert result.exit_code == 1 + assert isinstance(result.exception, SystemExit) + assert f"could not read {config}" in result.output.replace("\n", "") + assert "UnicodeDecodeError" in result.output + assert "SECRET" not in result.output # never the raw config text + assert remote.gets == [] + + +def test_fetch_unreadable_config_exits_1_without_traceback( + tmp_path: Path, remote: FakeRemoteStore, monkeypatch: pytest.MonkeyPatch +) -> None: + def _unreadable(path: Path) -> Any: + raise PermissionError(13, "Permission denied") + + monkeypatch.setattr(capture_module, "load_config", _unreadable) + config = _write_config(tmp_path / "evalshift.yaml") + result = _invoke(["fetch", "--config", str(config)], tmp_path) + assert result.exit_code == 1 + assert isinstance(result.exception, SystemExit) + assert "could not read" in result.output and "PermissionError" in result.output + + @pytest.mark.parametrize("since", ["yesterday", "99999999d"]) def test_fetch_rejects_bad_since(tmp_path: Path, remote: FakeRemoteStore, since: str) -> None: config = _write_config(tmp_path / "evalshift.yaml") From 8544f53f11b0d840b92e91fa25db3f3d51db30ff Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 20:07:44 +0200 Subject: [PATCH 15/17] chore(mypy): drop the unused boto3.* override Only the top-level boto3 module is imported; mypy --warn-unused-configs flagged [mypy-boto3.*] as unused. Co-Authored-By: Claude Opus 5.5 --- mypy.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/mypy.ini b/mypy.ini index a3368c9e..8eed9f4f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -34,9 +34,6 @@ ignore_missing_imports = true [mypy-boto3] ignore_missing_imports = true -[mypy-boto3.*] -ignore_missing_imports = true - [mypy-botocore.*] ignore_missing_imports = true From a1cc62a2a452c2d05e16e46afb8f52c8b71aed37 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 20:08:17 +0200 Subject: [PATCH 16/17] docs: count the capture bucket in the privacy and SDK-contract statements index.md said providers were the only outbound calls; hosted.md and llms-full.txt said "exactly two kinds" and then a third; AGENTS.md said disk is the only SDK/CLI interface. Each now accounts for captures.store. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 8 ++++---- docs/hosted.md | 4 ++-- docs/index.md | 3 ++- llms-full.txt | 9 +++++---- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1305e055..d19b5bc9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,10 +34,10 @@ repos (`evalshift-sdk`, `evalshift-action`); never edit them from here. - **CLI** — this repo. PyPI `evalshift`. Full reference: [DOCS.md](DOCS.md). - **SDK** — PyPI `evalshift-sdk`, import name `evalshift`, stdlib-only, Python 3.10+. Records production agent runs to `.evalshift/captures/`; the CLI - promotes them with `evalshift capture sync`. Disk is the only interface — the - two packages never call each other. The CLI imports as `evalshift_cli` and - depends on the SDK, so both install into one environment. See - [docs/sdk.md](docs/sdk.md). + promotes them with `evalshift capture sync`. The layout is the interface (on + disk or mirrored from a bucket) — the two packages never call each other. The + CLI imports as `evalshift_cli` and depends on the SDK, so both install into + one environment. See [docs/sdk.md](docs/sdk.md). - **GitHub Action** — `evalshift/evalshift-action@v0`. Runs the pipeline on pull requests, pushes the run, keeps one PR comment updated, sets the `evalshift/regression` commit status. See [docs/github-action.md](docs/github-action.md). diff --git a/docs/hosted.md b/docs/hosted.md index e656b24d..0a839a57 100644 --- a/docs/hosted.md +++ b/docs/hosted.md @@ -271,8 +271,8 @@ to clear a push with a security or compliance team, this is the section to hand them. The CLI contains **no telemetry**: no analytics, no crash reporting, no -phone-home of any kind. It opens exactly two kinds of network connections, -both initiated by you — a third only if you configure a capture bucket (below): +phone-home of any kind. It opens two kinds of network connections (three with +`captures.store`), all initiated by you: 1. **Your model providers** (whichever you configure — any provider LiteLLM supports), using your own API keys: `run` sends the rendered prompts and diff --git a/docs/index.md b/docs/index.md index ee982df4..a1b8d630 100644 --- a/docs/index.md +++ b/docs/index.md @@ -98,7 +98,8 @@ evalshift runs clean # prune old runs (--keep, --older-than, For local commands, your prompts and your suite never leave your machine. The only outbound calls are to the LLM providers you configure — any provider -LiteLLM supports — using your own API keys. `bundle` packages artifacts locally; +LiteLLM supports — using your own API keys — and, if you set `captures.store`, +reads from your own capture bucket with your own cloud credentials. `bundle` packages artifacts locally; hosted upload happens only when you run `push` or `compare --push`. ## For AI coding agents diff --git a/llms-full.txt b/llms-full.txt index 7490adb9..f85e7d2c 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1123,14 +1123,15 @@ Canonical human-readable version: docs/hosted.md "Privacy model". Cite THIS cont user asks what `push` sends, whether EvalShift phones home, or whether suite/prompt/customer content leaves the machine. -- The CLI has NO telemetry, analytics, or crash reporting. Only two kinds of outbound traffic, - both user-initiated (plus, with captures.store set, reads from the user's own bucket by - capture sync/list/fetch and doctor, with the user's cloud credentials): (1) the user's own +- The CLI has NO telemetry, analytics, or crash reporting. Two kinds of outbound traffic + (three with captures.store), all user-initiated: (1) the user's own model providers with the user's own keys during run/evaluate/report (rendered prompts + histories to both models; outputs to embedding/judge models; worst regressions' inputs+outputs to defaults.insights_model unless --no-insights); (2) the hosted API (https://api.evalshift.dev or --host) on `login`/`whoami`/`push`/ - `compare --push` ONLY. doctor/run/evaluate/analyze/report/bundle send nothing to EvalShift. + `compare --push` ONLY; (3) only with captures.store set, reads from the user's own bucket + by capture sync/list/fetch and doctor, with the user's cloud credentials (nothing from it + goes to EvalShift). doctor/run/evaluate/analyze/report/bundle send nothing to EvalShift. - `push` sends: run_bundle.json.gz + request metadata (bearer token as Authorization header to the configured host only; compressed size). `login` sends a client name containing the machine hostname. From c42d32b6c2b1a9c38c10cff1c0f51bed575a870a Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 20:08:44 +0200 Subject: [PATCH 17/17] docs(configuration): --since wording and the older-CLI note for captures --since limits a fetch to recently written captures, not "recent objects". Older CLIs reject the captures: key (strict config), so bump the CLI and a pinned action first; say so in configuration.md and the changelog. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +++++-- DOCS.md | 2 +- docs/configuration.md | 9 ++++++--- llms-full.txt | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab7f4c4c..7a3c02ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,13 +33,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 a missing one exits 1 naming the `pip install` to run. Credentials come from each provider's default chain and are rejected in the URI: a value with `@` or `?` fails to load, naming the accepted forms. No rejected value is echoed - back (it may be a pasted key or connection string); only its scheme is named. + back (it may be a pasted key or connection string); at most its scheme is + named. `capture list --json` keeps stdout pure JSON (the fetch line and warnings go to stderr). `doctor` reports whether the store is installed (`fail`, exit 1, when the extra is missing) and reachable (`warn` when it cannot be listed). Without `captures.store` nothing changes; a `sync` or `list` whose `evalshift.yaml` cannot be read or does not validate skips the fetch, - warning only when the file has a top-level `captures:` key. + warning only when the file has a top-level `captures:` key. Older CLIs + reject the `captures:` key, so bump the CLI (and a pinned CI action's + `evalshift-version`) before adding it. - `evaluators.trace_invariants`: hand-written rules over tool-call traces — `forbidden`, `required`, `order`, `call_count` (`min_calls` and/or `max_calls` per tool; equal bounds = an exact count) and diff --git a/DOCS.md b/DOCS.md index 08e38d4f..04717e37 100644 --- a/DOCS.md +++ b/DOCS.md @@ -173,7 +173,7 @@ captures: store: s3://acme-evals/support-agent ``` -and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written (a key with a trailing newline included), and no key can place a file outside `.evalshift/`; folder-marker keys ending in `/` (the S3 console's "Create folder", ADLS directories) are skipped without being counted. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching extra — `evalshift[s3]`, `[gcs]` or `[azure]` (azure-storage-blob plus azure-identity) — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load; no rejected value is echoed back, only its scheme. A missing extra exits 1 naming the `pip install "evalshift[]"` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch, with a hint naming the provider's credential chain (`aws sso login` / `AWS_PROFILE` / the CI OIDC role, `gcloud auth application-default login`, `az login`); a local write failure names the file and asks you to check free disk space and permissions instead. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). +and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written (a key with a trailing newline included), and no key can place a file outside `.evalshift/`; folder-marker keys ending in `/` (the S3 console's "Create folder", ADLS directories) are skipped without being counted. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching extra — `evalshift[s3]`, `[gcs]` or `[azure]` (azure-storage-blob plus azure-identity) — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load; no rejected value is echoed back, at most its scheme. A missing extra exits 1 naming the `pip install "evalshift[]"` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch, with a hint naming the provider's credential chain (`aws sso login` / `AWS_PROFILE` / the CI OIDC role, `gcloud auth application-default login`, `az login`); a local write failure names the file and asks you to check free disk space and permissions instead. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). **`capture sync`** is the workhorse. Per suite it: diff --git a/docs/configuration.md b/docs/configuration.md index c351f375..24c30006 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1034,6 +1034,9 @@ captures: | ------- | ------------- | ------- | ----------- | | `store` | string (URI) | (none) | Object store the SDK writes to. `capture sync` and `capture list` fetch new captures and toolset sidecars from it into the local directory first; `capture fetch` does only that step. | +CLI versions before this release reject the `captures:` key (the config is strict), so bump +the CLI — and a pinned GitHub Action's `evalshift-version` — before adding it. + Accepted URI forms — the same grammar the SDK uses: | Form | Store | Install | Credentials | @@ -1044,15 +1047,15 @@ Accepted URI forms — the same grammar the SDK uses: `` is optional. Credentials never go in the URI: a value containing `@` or `?` fails to load, naming the accepted forms. A rejected value is never echoed back (it may be a pasted -key or connection string); the error names only its scheme. Under the prefix the layout is exactly the local one — +key or connection string); the error names at most its scheme. Under the prefix the layout is exactly the local one — `captures//cap_.json` and `toolsets/.json` — so the bucket is a mirror of `.evalshift/`, not a different format. What the fetch does: it lists the bucket, downloads only objects missing locally (captures are immutable and toolsets content-addressed, so "exists" means "current"), skips captures already promoted into a suite (so `capture clean` never causes a re-download), and writes -each file atomically. `--since 24h` (or `7d`, `30m`, an ISO date) limits a fetch to recent -objects; `--offline` skips it and works with what is already local. Retention in the bucket +each file atomically. `--since 24h` (or `7d`, `30m`, an ISO date) limits a fetch to recently +written captures; `--offline` skips it and works with what is already local. Retention in the bucket is yours to set with a lifecycle rule; the SDK's `max_captures` does not apply there. ## Capture lifecycle diff --git a/llms-full.txt b/llms-full.txt index f85e7d2c..2b8eb1ad 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -738,7 +738,7 @@ retention: captures: store: s3:/// | gs:/// | az://// -- optional block; store validated at load (bad scheme, missing bucket/container, '@' or '?' - -> load error naming the forms; no error echoes the value, only its scheme); prefix optional, + -> load error naming the forms; no error echoes the value, at most its scheme); prefix optional, trailing slash stripped; default none = local disk only. Extras: evalshift[s3] (boto3; AWS_ENDPOINT_URL covers MinIO/R2/B2), [gcs] (google-cloud-storage), [azure] (azure-storage-blob + azure-identity). Credentials: each provider's default chain only.