From c06c4f36351f57c116ec1ac78c322079c7b1493c Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Thu, 8 Oct 2026 23:32:51 +0200 Subject: [PATCH 01/10] feat(push): send score explanations and trace-rule violations Each uploaded score row now carries the record's one-line explanation and, for trace_invariants rows that measured, the rules each side broke (rule id, type, tool, round, detail, sample). rules_checked, owner and failure_categories stay local; they already ship in evaluator_config and decision. The traces the verdicts were computed from upload already, so this adds no new exposure. Re-vendors the server schema that accepts the fields. Co-Authored-By: Claude Fable 5.1 --- src/evalshift_cli/hosted/bundle.py | 38 ++++++++ .../hosted/bundle_manifest.schema.json | 95 ++++++++++++++++++ tests/unit/test_bundle_shape.py | 97 +++++++++++++++++++ 3 files changed, 230 insertions(+) diff --git a/src/evalshift_cli/hosted/bundle.py b/src/evalshift_cli/hosted/bundle.py index 5faa86ef..453564d3 100644 --- a/src/evalshift_cli/hosted/bundle.py +++ b/src/evalshift_cli/hosted/bundle.py @@ -29,6 +29,7 @@ from evalshift_cli.config.loader import ConfigError, load_config from evalshift_cli.config.models import EvalShiftConfig, EvaluatorsConfig from evalshift_cli.evaluators.base import EvalRecord +from evalshift_cli.evaluators.trace_invariants import KIND as KIND_INVARIANTS from evalshift_cli.hosted.trace_events import from_tool_trace from evalshift_cli.insights.stage import read_bundle_insight from evalshift_cli.reports.economics import ( @@ -463,6 +464,11 @@ def _build_examples( "target_score": item.target_score, "delta": item.delta, "error": item.error, + # Added 2026-10-08; hosted EvalShift accepts both since + # its phase-29 deploy. Everything else in ``metadata`` + # stays local — see ``_score_violations``. + "explanation": item.explanation or None, + "violations": _score_violations(item), } for item in pair_scores ], @@ -514,6 +520,38 @@ def _tool_match( return all(item.delta >= 0 for item in tool_records) +#: What a ``violations`` entry may carry on the wire — the server's +#: ``ScoreViolation`` model is ``extra="forbid"``, so anything else the +#: evaluator ever adds to ``Violation.to_dict()`` must be dropped here, not +#: discovered as a 422 after the bundle has uploaded. +_VIOLATION_KEYS = ("rule_id", "rule_type", "tool", "round_index", "detail", "sample") + + +def _score_violations(record: EvalRecord) -> dict[str, list[dict[str, Any]]] | None: + """Which rules each side broke, for a ``trace_invariants`` row that measured. + + Read off the ``source_violations`` / ``target_violations`` metadata the + evaluator wrote (``Violation.to_dict()``, plus a ``sample`` ordinal on a + repeated-sampling run). ``None`` on every other evaluator and on an errored + row — an errored row has no verdict, and an empty ``target`` list would read + as a clean one. ``rules_checked``, ``owner`` and ``failure_categories`` + stay local: the first two ship in ``evaluator_config``, the third in + ``decision``. + """ + if record.kind != KIND_INVARIANTS or record.error is not None: + return None + + def entries(key: str) -> list[dict[str, Any]]: + raw = record.metadata.get(key) or [] + return [ + {field: item[field] for field in _VIOLATION_KEYS if field in item} + for item in raw + if isinstance(item, dict) + ] + + return {"source": entries("source_violations"), "target": entries("target_violations")} + + def _slice_name(example: SuiteExample | None) -> str | None: if example is None or not example.tags: return None diff --git a/src/evalshift_cli/hosted/bundle_manifest.schema.json b/src/evalshift_cli/hosted/bundle_manifest.schema.json index de73e3e6..457e2fe2 100644 --- a/src/evalshift_cli/hosted/bundle_manifest.schema.json +++ b/src/evalshift_cli/hosted/bundle_manifest.schema.json @@ -859,6 +859,18 @@ "title": "Evaluator Name", "type": "string" }, + "explanation": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Explanation" + }, "kind": { "default": "", "title": "Kind", @@ -871,6 +883,17 @@ "target_score": { "title": "Target Score", "type": "number" + }, + "violations": { + "anyOf": [ + { + "$ref": "#/$defs/ScoreViolations" + }, + { + "type": "null" + } + ], + "default": null } }, "required": [ @@ -1551,6 +1574,78 @@ "title": "RoleEconomics", "type": "object" }, + "ScoreViolation": { + "additionalProperties": false, + "description": "One trace rule one side broke, attributed to one call or one missing tool.\n\nMirrors the CLI's ``evaluators.invariants.Violation.to_dict()`` field for\nfield. ``sample`` is the zero-based sample ordinal on a\n``samples_per_example > 1`` run and absent otherwise.", + "properties": { + "detail": { + "title": "Detail", + "type": "string" + }, + "round_index": { + "minimum": 0, + "title": "Round Index", + "type": "integer" + }, + "rule_id": { + "minLength": 1, + "title": "Rule Id", + "type": "string" + }, + "rule_type": { + "title": "Rule Type", + "type": "string" + }, + "sample": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sample" + }, + "tool": { + "title": "Tool", + "type": "string" + } + }, + "required": [ + "rule_id", + "rule_type", + "tool", + "round_index", + "detail" + ], + "title": "ScoreViolation", + "type": "object" + }, + "ScoreViolations": { + "additionalProperties": false, + "description": "Both sides' broken rules for one ``trace_invariants`` score row.\n\nCarried as two lists rather than one tagged list so a reader can tell \"the\nsource broke this rule too\" by rule id without a join. A rule both sides\nbroke still fails the target \u2014 the CLI's policy counts it \u2014 which is the\nreason the source list is worth sending at all.", + "properties": { + "source": { + "items": { + "$ref": "#/$defs/ScoreViolation" + }, + "title": "Source", + "type": "array" + }, + "target": { + "items": { + "$ref": "#/$defs/ScoreViolation" + }, + "title": "Target", + "type": "array" + } + }, + "title": "ScoreViolations", + "type": "object" + }, "SliceAggregate": { "additionalProperties": false, "properties": { diff --git a/tests/unit/test_bundle_shape.py b/tests/unit/test_bundle_shape.py index beac3b2c..8a0d5168 100644 --- a/tests/unit/test_bundle_shape.py +++ b/tests/unit/test_bundle_shape.py @@ -95,6 +95,8 @@ def test_example_rows_carry_split_ids_and_hoisted_metrics(built_bundle_path: Pat "target_score", "delta", "error", + "explanation", + "violations", } for key in ( "worst_delta_score", @@ -477,6 +479,101 @@ def _rows(scores: list[EvalRecord], example_ids: tuple[str, ...]) -> dict[str, d return {row["example_id"]: row for row in rows} +NO_DELETE: dict[str, Any] = { + "rule_id": "no-delete", + "rule_type": "forbidden", + "tool": "delete_file", + "round_index": 0, + "detail": "delete_file was called", +} + + +def _rule_record( + example_id: str, + *, + target_violations: list[dict[str, Any]], + source_violations: list[dict[str, Any]] | None = None, + error: str | None = None, +) -> EvalRecord: + """A ``trace_invariants`` row as ``TraceInvariantsEvaluator._record`` writes it.""" + broken = sorted({v["rule_id"] for v in target_violations}) + return EvalRecord( + run_id="r1", + prompt_id="replay", + example_id=example_id, + evaluator_name="contracts", + kind="trace_invariants", + source_score=1.0 if not source_violations else 0.5, + target_score=1.0 if not target_violations else 0.5, + delta=(1.0 if not target_violations else 0.5) - (1.0 if not source_violations else 0.5), + explanation=( + f"target broke {len(broken)} trace rule(s): {', '.join(broken)}" + if broken + else "target kept every trace rule" + ), + metadata={ + "rules_checked": ["no-delete"], + "source_violations": source_violations or [], + "target_violations": target_violations, + "owner": "payments", + **({"failure_categories": ["INVARIANT_VIOLATION"]} if target_violations else {}), + }, + error=error, + ) + + +def test_trace_rule_rows_carry_the_explanation_and_both_sides_violations() -> None: + row = _rows([_rule_record("ex1", target_violations=[NO_DELETE])], ("ex1",))["ex1"] + score = row["scores"][0] + assert score["explanation"] == "target broke 1 trace rule(s): no-delete" + assert score["violations"] == {"source": [], "target": [NO_DELETE]} + + +def test_the_source_side_violations_ride_along() -> None: + """A rule both sides broke still fails the target; the source list is what lets a reader see that.""" + row = _rows( + [_rule_record("ex1", target_violations=[NO_DELETE], source_violations=[NO_DELETE])], + ("ex1",), + )["ex1"] + assert row["scores"][0]["violations"]["source"] == [NO_DELETE] + + +def test_rules_checked_owner_and_failure_categories_stay_local() -> None: + """They are in the uploaded evaluator config and decision already — the row must not repeat them.""" + row = _rows([_rule_record("ex1", target_violations=[NO_DELETE])], ("ex1",))["ex1"] + score = row["scores"][0] + assert "metadata" not in score + assert set(score["violations"]) == {"source", "target"} + + +def test_other_evaluators_carry_null_violations_and_their_explanation() -> None: + record = _record("ex1", "semantic", 1.0, 0.8).model_copy(update={"explanation": "cosine 0.80"}) + score = _rows([record], ("ex1",))["ex1"]["scores"][0] + assert score["violations"] is None + assert score["explanation"] == "cosine 0.80" + + +def test_an_empty_explanation_is_sent_as_null() -> None: + score = _rows([_record("ex1", "semantic", 1.0, 0.8)], ("ex1",))["ex1"]["scores"][0] + assert score["explanation"] is None + + +def test_an_errored_trace_rule_row_carries_no_violations() -> None: + """An evaluator that broke measured nothing; an empty target list would read as a clean target.""" + record = _rule_record("ex1", target_violations=[], error="trace missing") + record = record.model_copy(update={"metadata": {}}) + assert _rows([record], ("ex1",))["ex1"]["scores"][0]["violations"] is None + + +def test_violation_entries_are_filtered_to_the_server_contract() -> None: + """A future key on ``Violation.to_dict()`` must not reach a schema that forbids unknown properties.""" + row = _rows( + [_rule_record("ex1", target_violations=[{**NO_DELETE, "sample": 1, "severity": "high"}])], + ("ex1",), + )["ex1"] + assert row["scores"][0]["violations"]["target"] == [{**NO_DELETE, "sample": 1}] + + def test_a_pair_nothing_measured_is_not_reported_as_passed() -> None: """``all()`` over an empty list is ``True`` — absence read as success. From 8c0646a07da7afdd80c224e410835cf3553765d8 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Thu, 8 Oct 2026 23:38:54 +0200 Subject: [PATCH 02/10] docs: say that push uploads score explanations and trace-rule violations Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 7 +++++++ DOCS.md | 2 +- docs/hosted.md | 2 +- llms-full.txt | 7 ++++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bd1a43a..2ca69e5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `evaluators.trace_invariants` block when it regenerates the entry. The marker comment still says hand edits are overwritten; this block is the one exception. +- `push` now uploads each score row's one-line `explanation` and, for + `trace_invariants` rows, the rules each side broke (`violations.source` / + `violations.target`: rule id, type, tool, round, detail, sample). Hosted + EvalShift shows them on the example detail and on share links instead of + only the violation count. `rules_checked`, `owner` and `failure_categories` + still stay local. Requires the hosted service's 2026-10-08 phase-29 deploy; + nothing else in the bundle changed. ### Changed diff --git a/DOCS.md b/DOCS.md index 619d0862..b7071a9b 100644 --- a/DOCS.md +++ b/DOCS.md @@ -794,7 +794,7 @@ evalshift logout The full field-by-field data contract lives in [docs/hosted.md — Privacy model](docs/hosted.md#privacy-model--exactly-what-uploads); this is the summary. The CLI has **no telemetry** — no analytics, no crash reporting. Its only network traffic is (1) your configured model providers, with your own keys, during `run`/`evaluate`/`report`, and (2) the hosted API on `login`, `whoami`, and `push`. -**A push uploads**, inside `run_bundle.json.gz`: the manifest (run id, `org/project` slug, model ids, suite name, git SHA/branch/PR number, the local suite file path string, content hashes, timestamp, CLI version); per-example rows — the example's template `inputs` and `expected` output **verbatim**, both models' **full output text**, the replay's tool-call traces (tool names, arguments and call ids, round markers, final text and refusal messages, capped at 256 KB per side; imported agent traces are not uploaded), per-evaluator scores and error strings, per-side cost and latency, tags; aggregate/analysis/decision/economics (numbers, not content); methodology notes; the insights narrative (prose that can quote the regressions it summarizes); the evaluator config with every prompt body replaced by a `content_hash` (prompt names, file paths and variable names do ship, and so does each `llm_judge` `criterion_prompt`); and a dataset snapshot holding only metadata plus an `examples_hash`. Request metadata beside the bundle: the bearer token as an auth header to the configured host only, and the compressed size. +**A push uploads**, inside `run_bundle.json.gz`: the manifest (run id, `org/project` slug, model ids, suite name, git SHA/branch/PR number, the local suite file path string, content hashes, timestamp, CLI version); per-example rows — the example's template `inputs` and `expected` output **verbatim**, both models' **full output text**, the replay's tool-call traces (tool names, arguments and call ids, round markers, final text and refusal messages, capped at 256 KB per side; imported agent traces are not uploaded), per-evaluator scores, error strings and one-line explanations, and for each `trace_invariants` evaluator the rules each side broke (rule id, type, tool, round, detail), per-side cost and latency, tags; aggregate/analysis/decision/economics (numbers, not content); methodology notes; the insights narrative (prose that can quote the regressions it summarizes); the evaluator config with every prompt body replaced by a `content_hash` (prompt names, file paths and variable names do ship, and so does each `llm_judge` `criterion_prompt`); and a dataset snapshot holding only metadata plus an `examples_hash`. Request metadata beside the bundle: the bearer token as an auth header to the configured host only, and the compressed size. **Never uploads**: provider API keys, the hosted token (never inside a bundle), prompt bodies and system prompts, suite conversation histories, tool definitions/schemas, `raw.jsonl`, imported agent traces (`traces.jsonl`), the response cache, `.evalshift/captures/`, `state.json`, `report.json`, `report.html`. diff --git a/docs/hosted.md b/docs/hosted.md index 7e1f85e8..201dda2a 100644 --- a/docs/hosted.md +++ b/docs/hosted.md @@ -295,7 +295,7 @@ The bundle itself contains: | Block | What is inside | | --- | --- | | `manifest` | Run id, `org/project` slug, source and target model ids, suite name, git commit SHA, branch name, PR number, the **local suite file path** as a string (it can reveal directory or user names), two content hashes, the run timestamp, and the CLI version. | -| `examples[]` — one row per prompt × example | The example's template variables (`inputs`) **verbatim**; its `expected` reference output **verbatim**; both models' **full output text**; tool-call traces from the replay: each side's tool calls (names, arguments, call ids) with round markers, any final text and refusal messages, capped at 256 KB per side. Imported agent traces (`traces import`) are not uploaded; per-evaluator scores and error strings; per-side cost and latency; tags and slice names. | +| `examples[]` — one row per prompt × example | The example's template variables (`inputs`) **verbatim**; its `expected` reference output **verbatim**; both models' **full output text**; tool-call traces from the replay: each side's tool calls (names, arguments, call ids) with round markers, any final text and refusal messages, capped at 256 KB per side. Imported agent traces (`traces import`) are not uploaded; per-evaluator scores, error strings and one-line explanations (`target won`, `cosine 0.84 vs 1.00`, `target broke 1 trace rule(s): no-delete`); for each `trace_invariants` evaluator, the rules each side broke — rule id, rule type, tool, round, a one-line detail that may quote an argument value, and the sample ordinal on a repeated-sampling run — so the hosted report can name the rule, not only count it (`rules_checked`, `owner` and `failure_categories` stay local); per-side cost and latency; tags and slice names. | | `aggregate`, `analysis`, `decision`, `economics` | Pass/fail counts, statistical comparisons, the migration verdict, and per-role token/cost/latency rollups. Numbers and verdict labels, not content. `decision.policy` is the resolved `migration_policy` this run's verdict was computed under — every top-level budget with its default applied, plus `slices` — or `null` when no `migration_policy` is configured. It therefore carries `max_invariant_violations` (default `0`) even when the config has no trace rules; hosted EvalShift accepts and stores it (since 2026-10-08) but never re-evaluates it — a run that carries its own policy is answered with the CLI's decision, which did check it, and only runs pushed without a policy fall back to the server's six-budget re-check. It is what lets the hosted gate check a pull request against the exact budgets the verdict used, instead of a separate policy configured elsewhere. | | `methodology_notes` | The model ids and the statistical-contract sentences shown in every report. | | `insights` | The machine-written run narrative, when one was generated. It is prose *about* your run and can paraphrase or quote the regressions it summarizes. | diff --git a/llms-full.txt b/llms-full.txt index 4481a93b..8f1adf58 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -226,7 +226,10 @@ evalshift login [--token es_...] [--host URL] [--no-browser] [--timeout SECS=900 evalshift logout | evalshift whoami [--host] [--token] evalshift bundle RUN_ID [-c] [-s/--suite] [--suite-name] [-o/--output PATH] [--project org/project] Builds run_bundle.json.gz locally, no upload. Payload: manifest, examples (inputs, both - outputs, per-evaluator scores, cost/latency deltas, traces[] — the replay's own tool-call + outputs, per-evaluator scores (each score row also carries explanation (str|null) and, on + trace_invariants rows that measured, violations {source: [...], target: [...]} of {rule_id, + rule_type, tool, round_index, detail, sample} -- the rest of the record's metadata stays + local), cost/latency deltas, traces[] — the replay's own tool-call trace, one stream per model side: ordered tool calls w/ names, arguments, call ids; round markers; any final text; refusal messages. NO model_call events, NO tool results; a stream over 256KB keeps its leading events and is flagged truncated. Imported traces (traces import) @@ -653,6 +656,8 @@ evaluators: # every evaluator config also takes blocking # failure_categories [INVARIANT_VIOLATION] when the target broke anything (the decision's # failure_categories count it as the budget does: distinct examples with a blocking # target violation -- never per row, never for advisory entries). + # push uploads explanation + source/target violations per row (not + # rules_checked/owner/failure_categories). suites: # managed block; capture sync rewrites between : # ">>> evalshift suites" markers source: captured|jsonl = captured From 792e8faa1e369e1e1ad35d2a6e4e8a2cb65d5a94 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Thu, 8 Oct 2026 23:39:48 +0200 Subject: [PATCH 03/10] docs(hosted): use only explanations the CLI emits as examples Co-Authored-By: Claude Fable 5.1 --- docs/hosted.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/hosted.md b/docs/hosted.md index 201dda2a..4229c439 100644 --- a/docs/hosted.md +++ b/docs/hosted.md @@ -295,7 +295,7 @@ The bundle itself contains: | Block | What is inside | | --- | --- | | `manifest` | Run id, `org/project` slug, source and target model ids, suite name, git commit SHA, branch name, PR number, the **local suite file path** as a string (it can reveal directory or user names), two content hashes, the run timestamp, and the CLI version. | -| `examples[]` — one row per prompt × example | The example's template variables (`inputs`) **verbatim**; its `expected` reference output **verbatim**; both models' **full output text**; tool-call traces from the replay: each side's tool calls (names, arguments, call ids) with round markers, any final text and refusal messages, capped at 256 KB per side. Imported agent traces (`traces import`) are not uploaded; per-evaluator scores, error strings and one-line explanations (`target won`, `cosine 0.84 vs 1.00`, `target broke 1 trace rule(s): no-delete`); for each `trace_invariants` evaluator, the rules each side broke — rule id, rule type, tool, round, a one-line detail that may quote an argument value, and the sample ordinal on a repeated-sampling run — so the hosted report can name the rule, not only count it (`rules_checked`, `owner` and `failure_categories` stay local); per-side cost and latency; tags and slice names. | +| `examples[]` — one row per prompt × example | The example's template variables (`inputs`) **verbatim**; its `expected` reference output **verbatim**; both models' **full output text**; tool-call traces from the replay: each side's tool calls (names, arguments, call ids) with round markers, any final text and refusal messages, capped at 256 KB per side. Imported agent traces (`traces import`) are not uploaded; per-evaluator scores, error strings and one-line explanations (`target won`, `tie`, `target broke 1 trace rule(s): no-delete`); for each `trace_invariants` evaluator, the rules each side broke — rule id, rule type, tool, round, a one-line detail that may quote an argument value, and the sample ordinal on a repeated-sampling run — so the hosted report can name the rule, not only count it (`rules_checked`, `owner` and `failure_categories` stay local); per-side cost and latency; tags and slice names. | | `aggregate`, `analysis`, `decision`, `economics` | Pass/fail counts, statistical comparisons, the migration verdict, and per-role token/cost/latency rollups. Numbers and verdict labels, not content. `decision.policy` is the resolved `migration_policy` this run's verdict was computed under — every top-level budget with its default applied, plus `slices` — or `null` when no `migration_policy` is configured. It therefore carries `max_invariant_violations` (default `0`) even when the config has no trace rules; hosted EvalShift accepts and stores it (since 2026-10-08) but never re-evaluates it — a run that carries its own policy is answered with the CLI's decision, which did check it, and only runs pushed without a policy fall back to the server's six-budget re-check. It is what lets the hosted gate check a pull request against the exact budgets the verdict used, instead of a separate policy configured elsewhere. | | `methodology_notes` | The model ids and the statistical-contract sentences shown in every report. | | `insights` | The machine-written run narrative, when one was generated. It is prose *about* your run and can paraphrase or quote the regressions it summarizes. | From bf728dd2f79d724a9c82a73c0da4e915989b5305 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Thu, 8 Oct 2026 23:46:13 +0200 Subject: [PATCH 04/10] docs: disclose that trace-rule verdicts on imported traces upload Co-Authored-By: Claude Fable 5.1 --- DOCS.md | 4 ++-- docs/hosted.md | 5 +++-- llms-full.txt | 13 ++++++++----- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/DOCS.md b/DOCS.md index b7071a9b..f4436b64 100644 --- a/DOCS.md +++ b/DOCS.md @@ -794,9 +794,9 @@ evalshift logout The full field-by-field data contract lives in [docs/hosted.md — Privacy model](docs/hosted.md#privacy-model--exactly-what-uploads); this is the summary. The CLI has **no telemetry** — no analytics, no crash reporting. Its only network traffic is (1) your configured model providers, with your own keys, during `run`/`evaluate`/`report`, and (2) the hosted API on `login`, `whoami`, and `push`. -**A push uploads**, inside `run_bundle.json.gz`: the manifest (run id, `org/project` slug, model ids, suite name, git SHA/branch/PR number, the local suite file path string, content hashes, timestamp, CLI version); per-example rows — the example's template `inputs` and `expected` output **verbatim**, both models' **full output text**, the replay's tool-call traces (tool names, arguments and call ids, round markers, final text and refusal messages, capped at 256 KB per side; imported agent traces are not uploaded), per-evaluator scores, error strings and one-line explanations, and for each `trace_invariants` evaluator the rules each side broke (rule id, type, tool, round, detail), per-side cost and latency, tags; aggregate/analysis/decision/economics (numbers, not content); methodology notes; the insights narrative (prose that can quote the regressions it summarizes); the evaluator config with every prompt body replaced by a `content_hash` (prompt names, file paths and variable names do ship, and so does each `llm_judge` `criterion_prompt`); and a dataset snapshot holding only metadata plus an `examples_hash`. Request metadata beside the bundle: the bearer token as an auth header to the configured host only, and the compressed size. +**A push uploads**, inside `run_bundle.json.gz`: the manifest (run id, `org/project` slug, model ids, suite name, git SHA/branch/PR number, the local suite file path string, content hashes, timestamp, CLI version); per-example rows — the example's template `inputs` and `expected` output **verbatim**, both models' **full output text**, the replay's tool-call traces (tool names, arguments and call ids, round markers, final text and refusal messages, capped at 256 KB per side; imported agent traces are not uploaded), per-evaluator scores, error strings and one-line explanations, and for each `trace_invariants` evaluator the rules each side broke (rule id, type, tool, round, detail) (also for entries checked on imported traces, whose timelines otherwise stay local), per-side cost and latency, tags; aggregate/analysis/decision/economics (numbers, not content); methodology notes; the insights narrative (prose that can quote the regressions it summarizes); the evaluator config with every prompt body replaced by a `content_hash` (prompt names, file paths and variable names do ship, and so does each `llm_judge` `criterion_prompt`); and a dataset snapshot holding only metadata plus an `examples_hash`. Request metadata beside the bundle: the bearer token as an auth header to the configured host only, and the compressed size. -**Never uploads**: provider API keys, the hosted token (never inside a bundle), prompt bodies and system prompts, suite conversation histories, tool definitions/schemas, `raw.jsonl`, imported agent traces (`traces.jsonl`), the response cache, `.evalshift/captures/`, `state.json`, `report.json`, `report.html`. +**Never uploads**: provider API keys, the hosted token (never inside a bundle), prompt bodies and system prompts, suite conversation histories, tool definitions/schemas, `raw.jsonl`, imported agent traces (`traces.jsonl`; only the trace-rule verdicts computed on them upload — see above), the response cache, `.evalshift/captures/`, `state.json`, `report.json`, `report.html`. **Still your responsibility**: `inputs`, `expected`, outputs and traces upload verbatim, so whatever customer data or secrets your suite or your models put in them uploads too. Redact at capture time (SDK redaction boundary) and inspect the exact bytes first: `evalshift bundle `, then `gunzip -c .evalshift/runs//run_bundle.json.gz | jq .` — `push --bundle` uploads exactly the file you inspected. diff --git a/docs/hosted.md b/docs/hosted.md index 4229c439..8c77450c 100644 --- a/docs/hosted.md +++ b/docs/hosted.md @@ -295,7 +295,7 @@ The bundle itself contains: | Block | What is inside | | --- | --- | | `manifest` | Run id, `org/project` slug, source and target model ids, suite name, git commit SHA, branch name, PR number, the **local suite file path** as a string (it can reveal directory or user names), two content hashes, the run timestamp, and the CLI version. | -| `examples[]` — one row per prompt × example | The example's template variables (`inputs`) **verbatim**; its `expected` reference output **verbatim**; both models' **full output text**; tool-call traces from the replay: each side's tool calls (names, arguments, call ids) with round markers, any final text and refusal messages, capped at 256 KB per side. Imported agent traces (`traces import`) are not uploaded; per-evaluator scores, error strings and one-line explanations (`target won`, `tie`, `target broke 1 trace rule(s): no-delete`); for each `trace_invariants` evaluator, the rules each side broke — rule id, rule type, tool, round, a one-line detail that may quote an argument value, and the sample ordinal on a repeated-sampling run — so the hosted report can name the rule, not only count it (`rules_checked`, `owner` and `failure_categories` stay local); per-side cost and latency; tags and slice names. | +| `examples[]` — one row per prompt × example | The example's template variables (`inputs`) **verbatim**; its `expected` reference output **verbatim**; both models' **full output text**; tool-call traces from the replay: each side's tool calls (names, arguments, call ids) with round markers, any final text and refusal messages, capped at 256 KB per side. Imported agent traces (`traces import`) are not uploaded; per-evaluator scores, error strings and one-line explanations (`target won`, `tie`, `target broke 1 trace rule(s): no-delete`); for each `trace_invariants` evaluator, the rules each side broke — rule id, rule type, tool, round, a one-line detail that may quote an argument value, and the sample ordinal on a repeated-sampling run — so the hosted report can name the rule, not only count it (`rules_checked`, `owner` and `failure_categories` stay local) — including rules checked on imported traces (`traces: imported`), so an entry can name a tool, and for an `arguments` rule quote the offending value, from a timeline that otherwise stays local; per-side cost and latency; tags and slice names. | | `aggregate`, `analysis`, `decision`, `economics` | Pass/fail counts, statistical comparisons, the migration verdict, and per-role token/cost/latency rollups. Numbers and verdict labels, not content. `decision.policy` is the resolved `migration_policy` this run's verdict was computed under — every top-level budget with its default applied, plus `slices` — or `null` when no `migration_policy` is configured. It therefore carries `max_invariant_violations` (default `0`) even when the config has no trace rules; hosted EvalShift accepts and stores it (since 2026-10-08) but never re-evaluates it — a run that carries its own policy is answered with the CLI's decision, which did check it, and only runs pushed without a policy fall back to the server's six-budget re-check. It is what lets the hosted gate check a pull request against the exact budgets the verdict used, instead of a separate policy configured elsewhere. | | `methodology_notes` | The model ids and the statistical-contract sentences shown in every report. | | `insights` | The machine-written run narrative, when one was generated. It is prose *about* your run and can paraphrase or quote the regressions it summarizes. | @@ -316,7 +316,8 @@ The bundle itself contains: in the bundle; only the calls a model actually made at run time appear, in the traces. - **Local artefacts**: `raw.jsonl` (the raw provider requests and responses), - imported agent traces (`traces.jsonl`), the SQLite response cache, + imported agent traces (`traces.jsonl`; only the trace-rule verdicts computed + on them upload — see the `examples[]` row above), the SQLite response cache, `.evalshift/captures/`, `state.json`, `report.json`, and `report.html`. The content hashes that replace this data (`dataset_hash`, `examples_hash`, diff --git a/llms-full.txt b/llms-full.txt index 8f1adf58..831803c9 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -233,7 +233,8 @@ evalshift bundle RUN_ID [-c] [-s/--suite] [--suite-name] [-o/--output PATH] [--p trace, one stream per model side: ordered tool calls w/ names, arguments, call ids; round markers; any final text; refusal messages. NO model_call events, NO tool results; a stream over 256KB keeps its leading events and is flagged truncated. Imported traces (traces import) - stay LOCAL, never bundled. Same trace shown on the hosted run-detail page), aggregate, + stay LOCAL, never bundled (trace-rule verdicts on them do). Same trace shown on the hosted + run-detail page), aggregate, analysis, decision (decision.policy = resolved migration_policy w/ CLI defaults applied, or null when none is configured -- see "Migration policy verdict algorithm" below), # examples[].passed is False when the pair scored NO rows: all() over an empty list is True, @@ -1096,8 +1097,9 @@ content leaves the machine. - examples[] (one per prompt x example): `inputs` VERBATIM; `expected` VERBATIM; both models' FULL output text; traces = the replay's own tool calls (names, arguments, call ids, round markers; final text; refusal messages; no tool results; stream capped 256KB/side, - flagged truncated); imported agent traces are NOT uploaded; per-evaluator scores + error - strings; per-side cost/latency; tags/slice names; turn_index. + flagged truncated); imported agent traces are NOT uploaded (trace-rule verdicts on them + do); per-evaluator scores + error strings; per-side cost/latency; tags/slice names; + turn_index. - aggregate/analysis/decision/economics: numbers and verdict labels, not content. - methodology_notes: model ids + statistical-contract sentences. - insights: the machine-written narrative — prose that can quote the regressions it @@ -1113,8 +1115,9 @@ content leaves the machine. - NEVER uploads: provider API keys; the hosted token (never inside a bundle); prompt bodies / system prompts (manual content -> content_hash; python_string bodies never enter the config); suite conversation histories (`history`, embedded system messages included); tool - definitions/schemas (toolsets); raw.jsonl; imported agent traces (traces.jsonl); the SQLite - response cache; .evalshift/captures/; state.json; report.json; report.html. + definitions/schemas (toolsets); raw.jsonl; imported agent traces (traces.jsonl; only the + trace-rule verdicts computed on them upload -- see examples[] above); the SQLite response + cache; .evalshift/captures/; state.json; report.json; report.html. - Hashes that replace content (dataset_hash, examples_hash, prompts[].content_hash) are SHA-256 over canonical JSON, so hosted diffs/baselines align without the content. - Residual risk: inputs/expected/outputs/traces upload verbatim — customer data or secrets a From 3996e12a15721604d2203986a6fa2d65075039e3 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Thu, 8 Oct 2026 23:47:11 +0200 Subject: [PATCH 05/10] docs(llms): name explanations and violations in the privacy section Co-Authored-By: Claude Fable 5.1 --- llms-full.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llms-full.txt b/llms-full.txt index 831803c9..1b434d1e 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1098,8 +1098,9 @@ content leaves the machine. FULL output text; traces = the replay's own tool calls (names, arguments, call ids, round markers; final text; refusal messages; no tool results; stream capped 256KB/side, flagged truncated); imported agent traces are NOT uploaded (trace-rule verdicts on them - do); per-evaluator scores + error strings; per-side cost/latency; tags/slice names; - turn_index. + do); per-evaluator scores + error strings + one-line explanation; on trace_invariants + rows, violations {source, target} = the rules each side broke (detail can quote an + argument value); per-side cost/latency; tags/slice names; turn_index. - aggregate/analysis/decision/economics: numbers and verdict labels, not content. - methodology_notes: model ids + statistical-contract sentences. - insights: the machine-written narrative — prose that can quote the regressions it From 9a8eb7f5be9fda61b92adf123cfbfbaf6cb451f4 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 00:13:41 +0200 Subject: [PATCH 06/10] chore(hosted): re-vendor the server bundle schema Co-Authored-By: Claude Fable 5.1 --- src/evalshift_cli/hosted/bundle_manifest.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evalshift_cli/hosted/bundle_manifest.schema.json b/src/evalshift_cli/hosted/bundle_manifest.schema.json index 457e2fe2..40ccc26a 100644 --- a/src/evalshift_cli/hosted/bundle_manifest.schema.json +++ b/src/evalshift_cli/hosted/bundle_manifest.schema.json @@ -1576,7 +1576,7 @@ }, "ScoreViolation": { "additionalProperties": false, - "description": "One trace rule one side broke, attributed to one call or one missing tool.\n\nMirrors the CLI's ``evaluators.invariants.Violation.to_dict()`` field for\nfield. ``sample`` is the zero-based sample ordinal on a\n``samples_per_example > 1`` run and absent otherwise.", + "description": "One trace rule one side broke, attributed to one call or one missing tool.\n\nMirrors the CLI's ``evaluators.invariants.Violation.to_dict()`` plus the\n``sample`` ordinal its repeated-sampling merge adds (zero-based; absent on\na single-sample run).", "properties": { "detail": { "title": "Detail", From 10418aca137ce032601aa0a42344a0be8e4ba14d Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 00:16:02 +0200 Subject: [PATCH 07/10] docs: state exactly which imported-trace verdicts upload Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 3 ++- DOCS.md | 2 +- docs/hosted.md | 12 ++++++++---- docs/traces.md | 11 +++++++++-- llms-full.txt | 10 ++++++---- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ca69e5e..7c7942b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,7 +53,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `violations.target`: rule id, type, tool, round, detail, sample). Hosted EvalShift shows them on the example detail and on share links instead of only the violation count. `rules_checked`, `owner` and `failure_categories` - still stay local. Requires the hosted service's 2026-10-08 phase-29 deploy; + are not repeated on the score row (they already ship in `evaluator_config` / + `decision`). Requires the hosted service's 2026-10-08 phase-29 deploy; nothing else in the bundle changed. ### Changed diff --git a/DOCS.md b/DOCS.md index f4436b64..719c2e39 100644 --- a/DOCS.md +++ b/DOCS.md @@ -796,7 +796,7 @@ The full field-by-field data contract lives in [docs/hosted.md — Privacy model **A push uploads**, inside `run_bundle.json.gz`: the manifest (run id, `org/project` slug, model ids, suite name, git SHA/branch/PR number, the local suite file path string, content hashes, timestamp, CLI version); per-example rows — the example's template `inputs` and `expected` output **verbatim**, both models' **full output text**, the replay's tool-call traces (tool names, arguments and call ids, round markers, final text and refusal messages, capped at 256 KB per side; imported agent traces are not uploaded), per-evaluator scores, error strings and one-line explanations, and for each `trace_invariants` evaluator the rules each side broke (rule id, type, tool, round, detail) (also for entries checked on imported traces, whose timelines otherwise stay local), per-side cost and latency, tags; aggregate/analysis/decision/economics (numbers, not content); methodology notes; the insights narrative (prose that can quote the regressions it summarizes); the evaluator config with every prompt body replaced by a `content_hash` (prompt names, file paths and variable names do ship, and so does each `llm_judge` `criterion_prompt`); and a dataset snapshot holding only metadata plus an `examples_hash`. Request metadata beside the bundle: the bearer token as an auth header to the configured host only, and the compressed size. -**Never uploads**: provider API keys, the hosted token (never inside a bundle), prompt bodies and system prompts, suite conversation histories, tool definitions/schemas, `raw.jsonl`, imported agent traces (`traces.jsonl`; only the trace-rule verdicts computed on them upload — see above), the response cache, `.evalshift/captures/`, `state.json`, `report.json`, `report.html`. +**Never uploads**: provider API keys, the hosted token (never inside a bundle), prompt bodies and system prompts, suite conversation histories, tool definitions/schemas, `raw.jsonl`, imported agent traces (`traces.jsonl`; only the verdicts computed on them upload — `agent_trace` scores and `trace_invariants` rule results — never the timelines; see above), the response cache, `.evalshift/captures/`, `state.json`, `report.json`, `report.html`. **Still your responsibility**: `inputs`, `expected`, outputs and traces upload verbatim, so whatever customer data or secrets your suite or your models put in them uploads too. Redact at capture time (SDK redaction boundary) and inspect the exact bytes first: `evalshift bundle `, then `gunzip -c .evalshift/runs//run_bundle.json.gz | jq .` — `push --bundle` uploads exactly the file you inspected. diff --git a/docs/hosted.md b/docs/hosted.md index 8c77450c..5069a7c6 100644 --- a/docs/hosted.md +++ b/docs/hosted.md @@ -108,7 +108,10 @@ The output is `.evalshift/runs//run_bundle.json.gz`. It carries: any final text, refusal messages and round markers. No `model_call` events or tool results are included. A stream over 256 KB keeps its leading events and is flagged `truncated`. Imported agent traces (`traces import`) stay - local and are not uploaded, + local and are not uploaded; the verdicts `evaluate` computes on them do — + scores, explanations, and for `trace_invariants` the broken rules (tool + names and, for `arguments` rules, the quoted offending value; see + [What `push` sends](#what-push-sends-block-by-block)), - the aggregate, `analysis`, and the policy `decision`, - `economics` — a run-level per-role rollup of calls, tokens, cost and latency, - `methodology_notes` and the evaluator config and dataset snapshot, @@ -295,7 +298,7 @@ The bundle itself contains: | Block | What is inside | | --- | --- | | `manifest` | Run id, `org/project` slug, source and target model ids, suite name, git commit SHA, branch name, PR number, the **local suite file path** as a string (it can reveal directory or user names), two content hashes, the run timestamp, and the CLI version. | -| `examples[]` — one row per prompt × example | The example's template variables (`inputs`) **verbatim**; its `expected` reference output **verbatim**; both models' **full output text**; tool-call traces from the replay: each side's tool calls (names, arguments, call ids) with round markers, any final text and refusal messages, capped at 256 KB per side. Imported agent traces (`traces import`) are not uploaded; per-evaluator scores, error strings and one-line explanations (`target won`, `tie`, `target broke 1 trace rule(s): no-delete`); for each `trace_invariants` evaluator, the rules each side broke — rule id, rule type, tool, round, a one-line detail that may quote an argument value, and the sample ordinal on a repeated-sampling run — so the hosted report can name the rule, not only count it (`rules_checked`, `owner` and `failure_categories` stay local) — including rules checked on imported traces (`traces: imported`), so an entry can name a tool, and for an `arguments` rule quote the offending value, from a timeline that otherwise stays local; per-side cost and latency; tags and slice names. | +| `examples[]` — one row per prompt × example | The example's template variables (`inputs`) **verbatim**; its `expected` reference output **verbatim**; both models' **full output text**; tool-call traces from the replay: each side's tool calls (names, arguments, call ids) with round markers, any final text and refusal messages, capped at 256 KB per side. Imported agent traces (`traces import`) are not uploaded; per-evaluator scores, error strings and one-line explanations (`target won`, `tie`, `target broke 1 trace rule(s): no-delete`); for each `trace_invariants` evaluator, the rules each side broke — rule id, rule type, tool, round, a one-line detail that may quote an argument value, and the sample ordinal on a repeated-sampling run — so the hosted report can name the rule, not only count it (`rules_checked`, `owner` and `failure_categories` are not repeated on the score row; they already ship in `evaluator_config` / `decision`) — including rules checked on imported traces (`traces: imported`), so an entry can name a tool, and for an `arguments` rule quote the offending value, from a timeline that otherwise stays local; per-side cost and latency; tags and slice names. | | `aggregate`, `analysis`, `decision`, `economics` | Pass/fail counts, statistical comparisons, the migration verdict, and per-role token/cost/latency rollups. Numbers and verdict labels, not content. `decision.policy` is the resolved `migration_policy` this run's verdict was computed under — every top-level budget with its default applied, plus `slices` — or `null` when no `migration_policy` is configured. It therefore carries `max_invariant_violations` (default `0`) even when the config has no trace rules; hosted EvalShift accepts and stores it (since 2026-10-08) but never re-evaluates it — a run that carries its own policy is answered with the CLI's decision, which did check it, and only runs pushed without a policy fall back to the server's six-budget re-check. It is what lets the hosted gate check a pull request against the exact budgets the verdict used, instead of a separate policy configured elsewhere. | | `methodology_notes` | The model ids and the statistical-contract sentences shown in every report. | | `insights` | The machine-written run narrative, when one was generated. It is prose *about* your run and can paraphrase or quote the regressions it summarizes. | @@ -316,8 +319,9 @@ The bundle itself contains: in the bundle; only the calls a model actually made at run time appear, in the traces. - **Local artefacts**: `raw.jsonl` (the raw provider requests and responses), - imported agent traces (`traces.jsonl`; only the trace-rule verdicts computed - on them upload — see the `examples[]` row above), the SQLite response cache, + imported agent traces (`traces.jsonl`; only the verdicts computed on them + upload — `agent_trace` scores and `trace_invariants` rule results — never + the timelines; see the `examples[]` row above), the SQLite response cache, `.evalshift/captures/`, `state.json`, `report.json`, and `report.html`. The content hashes that replace this data (`dataset_hash`, `examples_hash`, diff --git a/docs/traces.md b/docs/traces.md index 127459ca..b160a4aa 100644 --- a/docs/traces.md +++ b/docs/traces.md @@ -5,7 +5,10 @@ your own runtime — EvalShift never runs your agent — and are imported into a completed local run, after which `evaluate`, `analyze`, and `report` work as usual. Imported traces stay local: `evaluate` scores them and the local report shows them, but `push` uploads only the replay's own tool-call trace (see -[Hosted](#hosted)). +[Hosted](#hosted)). The timelines are not uploaded; the verdicts `evaluate` +computes on them are — scores, explanations, and for `trace_invariants` the +broken rules (tool names and, for `arguments` rules, the quoted offending +value). ## Import @@ -155,7 +158,11 @@ evalshift replay case --model target --trace Imported traces are not uploaded. They stay in `.evalshift/runs//traces.jsonl`, where `evaluate` (the `agent_trace` evaluator and `trace_invariants` entries with `traces: imported`), the local -report and `diff case` / `inspect case` / `replay case` read them. +report and `diff case` / `inspect case` / `replay case` read them. The +verdicts `evaluate` computes on them do upload — scores, explanations, and for +`trace_invariants` the broken rules (tool names and, for `arguments` rules, the +quoted offending value); see the privacy model's +[What `push` sends](hosted.md#what-push-sends-block-by-block). `evalshift bundle` / `evalshift push` carry only the replay's own tool-call trace — one stream per model side with the tool calls (names, arguments, call diff --git a/llms-full.txt b/llms-full.txt index 1b434d1e..c9ed53c3 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -657,8 +657,9 @@ evaluators: # every evaluator config also takes blocking # failure_categories [INVARIANT_VIOLATION] when the target broke anything (the decision's # failure_categories count it as the budget does: distinct examples with a blocking # target violation -- never per row, never for advisory entries). - # push uploads explanation + source/target violations per row (not - # rules_checked/owner/failure_categories). + # push uploads explanation + source/target violations per row; + # rules_checked/owner/failure_categories are not repeated on the score row (they + # already ship in evaluator_config / decision). suites: # managed block; capture sync rewrites between : # ">>> evalshift suites" markers source: captured|jsonl = captured @@ -1117,8 +1118,9 @@ content leaves the machine. system prompts (manual content -> content_hash; python_string bodies never enter the config); suite conversation histories (`history`, embedded system messages included); tool definitions/schemas (toolsets); raw.jsonl; imported agent traces (traces.jsonl; only the - trace-rule verdicts computed on them upload -- see examples[] above); the SQLite response - cache; .evalshift/captures/; state.json; report.json; report.html. + verdicts computed on them upload -- agent_trace scores and trace_invariants rule results -- + never the timelines; see examples[] above); the SQLite response cache; .evalshift/captures/; + state.json; report.json; report.html. - Hashes that replace content (dataset_hash, examples_hash, prompts[].content_hash) are SHA-256 over canonical JSON, so hosted diffs/baselines align without the content. - Residual risk: inputs/expected/outputs/traces upload verbatim — customer data or secrets a From 37067bf43d4653b08d4dd76bfd8ed6207460ba4f Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 00:17:47 +0200 Subject: [PATCH 08/10] docs: polish the upload wording Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 13 +++++++------ DOCS.md | 2 +- llms-full.txt | 24 ++++++++++++------------ src/evalshift_cli/hosted/bundle.py | 5 +++-- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c7942b7..fae51632 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,12 +50,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 exception. - `push` now uploads each score row's one-line `explanation` and, for `trace_invariants` rows, the rules each side broke (`violations.source` / - `violations.target`: rule id, type, tool, round, detail, sample). Hosted - EvalShift shows them on the example detail and on share links instead of - only the violation count. `rules_checked`, `owner` and `failure_categories` - are not repeated on the score row (they already ship in `evaluator_config` / - `decision`). Requires the hosted service's 2026-10-08 phase-29 deploy; - nothing else in the bundle changed. + `violations.target`: rule id, type, tool, round, detail, and sample on + repeated-sampling runs). Hosted EvalShift shows them on the example detail + and on share links instead of only the violation count. `rules_checked`, + `owner` and `failure_categories` are not repeated on the score row (they + already ship in `evaluator_config` / `decision`). Requires a hosted service + that accepts the two fields (hosted EvalShift was updated before this + release); nothing else in the bundle changed. ### Changed diff --git a/DOCS.md b/DOCS.md index 719c2e39..768d2de0 100644 --- a/DOCS.md +++ b/DOCS.md @@ -794,7 +794,7 @@ evalshift logout The full field-by-field data contract lives in [docs/hosted.md — Privacy model](docs/hosted.md#privacy-model--exactly-what-uploads); this is the summary. The CLI has **no telemetry** — no analytics, no crash reporting. Its only network traffic is (1) your configured model providers, with your own keys, during `run`/`evaluate`/`report`, and (2) the hosted API on `login`, `whoami`, and `push`. -**A push uploads**, inside `run_bundle.json.gz`: the manifest (run id, `org/project` slug, model ids, suite name, git SHA/branch/PR number, the local suite file path string, content hashes, timestamp, CLI version); per-example rows — the example's template `inputs` and `expected` output **verbatim**, both models' **full output text**, the replay's tool-call traces (tool names, arguments and call ids, round markers, final text and refusal messages, capped at 256 KB per side; imported agent traces are not uploaded), per-evaluator scores, error strings and one-line explanations, and for each `trace_invariants` evaluator the rules each side broke (rule id, type, tool, round, detail) (also for entries checked on imported traces, whose timelines otherwise stay local), per-side cost and latency, tags; aggregate/analysis/decision/economics (numbers, not content); methodology notes; the insights narrative (prose that can quote the regressions it summarizes); the evaluator config with every prompt body replaced by a `content_hash` (prompt names, file paths and variable names do ship, and so does each `llm_judge` `criterion_prompt`); and a dataset snapshot holding only metadata plus an `examples_hash`. Request metadata beside the bundle: the bearer token as an auth header to the configured host only, and the compressed size. +**A push uploads**, inside `run_bundle.json.gz`: the manifest (run id, `org/project` slug, model ids, suite name, git SHA/branch/PR number, the local suite file path string, content hashes, timestamp, CLI version); per-example rows — the example's template `inputs` and `expected` output **verbatim**, both models' **full output text**, the replay's tool-call traces (tool names, arguments and call ids, round markers, final text and refusal messages, capped at 256 KB per side; imported agent traces are not uploaded), per-evaluator scores, error strings and one-line explanations, for each `trace_invariants` evaluator the rules each side broke (rule id, type, tool, round, detail; also for entries checked on imported traces, whose timelines otherwise stay local), per-side cost and latency, tags; aggregate/analysis/decision/economics (numbers, not content); methodology notes; the insights narrative (prose that can quote the regressions it summarizes); the evaluator config with every prompt body replaced by a `content_hash` (prompt names, file paths and variable names do ship, and so does each `llm_judge` `criterion_prompt`); and a dataset snapshot holding only metadata plus an `examples_hash`. Request metadata beside the bundle: the bearer token as an auth header to the configured host only, and the compressed size. **Never uploads**: provider API keys, the hosted token (never inside a bundle), prompt bodies and system prompts, suite conversation histories, tool definitions/schemas, `raw.jsonl`, imported agent traces (`traces.jsonl`; only the verdicts computed on them upload — `agent_trace` scores and `trace_invariants` rule results — never the timelines; see above), the response cache, `.evalshift/captures/`, `state.json`, `report.json`, `report.html`. diff --git a/llms-full.txt b/llms-full.txt index c9ed53c3..b0240ac4 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -228,15 +228,15 @@ evalshift bundle RUN_ID [-c] [-s/--suite] [--suite-name] [-o/--output PATH] [--p Builds run_bundle.json.gz locally, no upload. Payload: manifest, examples (inputs, both outputs, per-evaluator scores (each score row also carries explanation (str|null) and, on trace_invariants rows that measured, violations {source: [...], target: [...]} of {rule_id, - rule_type, tool, round_index, detail, sample} -- the rest of the record's metadata stays - local), cost/latency deltas, traces[] — the replay's own tool-call - trace, one stream per model side: ordered tool calls w/ names, arguments, call ids; round - markers; any final text; refusal messages. NO model_call events, NO tool results; a stream - over 256KB keeps its leading events and is flagged truncated. Imported traces (traces import) - stay LOCAL, never bundled (trace-rule verdicts on them do). Same trace shown on the hosted - run-detail page), aggregate, - analysis, decision (decision.policy = resolved migration_policy w/ CLI defaults applied, or - null when none is configured -- see "Migration policy verdict algorithm" below), + rule_type, tool, round_index, detail, sample (on repeated-sampling runs)} -- the rest of + the record's metadata stays local), cost/latency deltas, traces[] — the replay's own + tool-call trace, one stream per model side: ordered tool calls w/ names, arguments, call + ids; round markers; any final text; refusal messages. NO model_call events, NO tool + results; a stream over 256KB keeps its leading events and is flagged truncated. Imported + traces (traces import) stay LOCAL, never bundled (trace-rule verdicts on them do). Same + trace shown on the hosted run-detail page), aggregate, analysis, decision + (decision.policy = resolved migration_policy w/ CLI defaults applied, or null when none is + configured -- see "Migration policy verdict algorithm" below), # examples[].passed is False when the pair scored NO rows: all() over an empty list is True, # and "nothing measured" reading as "passed" is the same silence-as-success bug as the # fabricated skip scores. Example.passed is a required non-nullable bool server-side, so @@ -657,9 +657,9 @@ evaluators: # every evaluator config also takes blocking # failure_categories [INVARIANT_VIOLATION] when the target broke anything (the decision's # failure_categories count it as the budget does: distinct examples with a blocking # target violation -- never per row, never for advisory entries). - # push uploads explanation + source/target violations per row; - # rules_checked/owner/failure_categories are not repeated on the score row (they - # already ship in evaluator_config / decision). + # push uploads explanation per row, and source_violations/target_violations as + # violations.source/.target; rules_checked/owner/failure_categories are not repeated + # on the score row (they already ship in evaluator_config / decision). suites: # managed block; capture sync rewrites between : # ">>> evalshift suites" markers source: captured|jsonl = captured diff --git a/src/evalshift_cli/hosted/bundle.py b/src/evalshift_cli/hosted/bundle.py index 453564d3..7fbe6329 100644 --- a/src/evalshift_cli/hosted/bundle.py +++ b/src/evalshift_cli/hosted/bundle.py @@ -465,8 +465,9 @@ def _build_examples( "delta": item.delta, "error": item.error, # Added 2026-10-08; hosted EvalShift accepts both since - # its phase-29 deploy. Everything else in ``metadata`` - # stays local — see ``_score_violations``. + # its 2026-10 deploy that added the columns. Everything + # else in ``metadata`` stays local — see + # ``_score_violations``. "explanation": item.explanation or None, "violations": _score_violations(item), } From 950c2f2d0fde3a7cb33bf831c0bd1b481c2c912e Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 00:19:16 +0200 Subject: [PATCH 09/10] refactor(hosted): rename the violation wire-field allowlist Co-Authored-By: Claude Fable 5.1 --- src/evalshift_cli/hosted/bundle.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/evalshift_cli/hosted/bundle.py b/src/evalshift_cli/hosted/bundle.py index 7fbe6329..83ca4bbf 100644 --- a/src/evalshift_cli/hosted/bundle.py +++ b/src/evalshift_cli/hosted/bundle.py @@ -521,11 +521,12 @@ def _tool_match( return all(item.delta >= 0 for item in tool_records) -#: What a ``violations`` entry may carry on the wire — the server's -#: ``ScoreViolation`` model is ``extra="forbid"``, so anything else the -#: evaluator ever adds to ``Violation.to_dict()`` must be dropped here, not -#: discovered as a 422 after the bundle has uploaded. -_VIOLATION_KEYS = ("rule_id", "rule_type", "tool", "round_index", "detail", "sample") +#: The fields a ``violations`` entry may carry on the wire — an allowlist, not +#: the metadata keys the lists are read from. The server's ``ScoreViolation`` +#: model is ``extra="forbid"``, so anything else the evaluator ever adds to +#: ``Violation.to_dict()`` must be dropped here, not discovered as a 422 after +#: the bundle has uploaded. +_VIOLATION_WIRE_FIELDS = ("rule_id", "rule_type", "tool", "round_index", "detail", "sample") def _score_violations(record: EvalRecord) -> dict[str, list[dict[str, Any]]] | None: @@ -545,7 +546,7 @@ def _score_violations(record: EvalRecord) -> dict[str, list[dict[str, Any]]] | N def entries(key: str) -> list[dict[str, Any]]: raw = record.metadata.get(key) or [] return [ - {field: item[field] for field in _VIOLATION_KEYS if field in item} + {field: item[field] for field in _VIOLATION_WIRE_FIELDS if field in item} for item in raw if isinstance(item, dict) ] From f913b9bcfb0d8f5917216cd03405a5e99d047270 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 9 Oct 2026 00:19:45 +0200 Subject: [PATCH 10/10] test(hosted): validate a populated violations row against the vendored schema Co-Authored-By: Claude Fable 5.1 --- tests/unit/test_bundle_shape.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/test_bundle_shape.py b/tests/unit/test_bundle_shape.py index 8a0d5168..98bb4464 100644 --- a/tests/unit/test_bundle_shape.py +++ b/tests/unit/test_bundle_shape.py @@ -574,6 +574,24 @@ def test_violation_entries_are_filtered_to_the_server_contract() -> None: assert row["scores"][0]["violations"]["target"] == [{**NO_DELETE, "sample": 1}] +def test_a_populated_violations_row_validates_against_the_vendored_schema() -> None: + """A drift guard, not a red-first test: it passes on arrival and must keep passing. + + The fixture bundle's rows carry no violations, so the end-to-end schema + test never sees a populated ``ScoreViolation``. This one does — with the + optional ``sample`` ordinal set — so a CLI-side field the server does not + accept, or a re-vendored schema that tightens the shape, fails here + instead of as a 422 after upload. + """ + row = _rows([_rule_record("ex1", target_violations=[{**NO_DELETE, "sample": 1}])], ("ex1",))[ + "ex1" + ]["scores"][0] + assert row["violations"]["target"], "the guard is vacuous without a populated entry" + Draft202012Validator( + {"$ref": "#/$defs/ExampleScore", "$defs": _vendored_schema()["bundle"]["$defs"]} + ).validate(row) + + def test_a_pair_nothing_measured_is_not_reported_as_passed() -> None: """``all()`` over an empty list is ``True`` — absence read as success.