diff --git a/examples/optimization/eval_optimize_loop/.gitignore b/examples/optimization/eval_optimize_loop/.gitignore new file mode 100644 index 000000000..b24bd972d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/.gitignore @@ -0,0 +1,4 @@ +/artifacts/ +/optimization_report.json +/optimization_report.md +__pycache__/ diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 000000000..2a255547c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,188 @@ +# Design + +`run_pipeline.py` only parses arguments and assembles an optional live callback. +`pipeline/orchestrator.py` is the sole composition root. Raw SDK +`EvaluateResult` objects are handed directly to `evaluation.normalize_result`; +no other module unfolds them. + +The dependency direction is flat and one-way: + +```text +CLI -> orchestrator -> preflight / evaluation_runtime / candidate_runtime / reporting +configuration -> live_adapter / models / schema +models -> schema +evaluation_runtime -> backend contract / normalizer / cost ledger / artifact sink +candidate_runtime -> generator contract / split policy / prompt workspace / artifact sink +backends -> offline_evaluation / trace_fixture / live_adapter / contracts / models +live_adapter -> none +optimizer_worker -> live_adapter / schema +preflight -> live_adapter +trace_fixture -> models / schema +offline_evaluation -> SDK evaluation types +pure policies -> models +reporting -> models / artifacts +``` + +`schema.py` owns strict parsing and model primitives; `configuration.py` owns +validated input and policy configuration; `models.py` owns stage outputs and +report facts. `preflight.py` owns validated inputs and run identity. +`evaluation_runtime.py` owns ordered backend calls, completed-call accounting, +and snapshot persistence. `candidate_runtime.py` owns inner-split persistence, +the OS-temporary optimizer workspace, and sanitized optimizer output import. +`trace_fixture.py` owns trace schema, hash and case-matrix validation. +`offline_evaluation.py` owns offline rule parsing, evidence extraction, +deterministic evaluators, and the run-local replacement registry. `backends.py` +only adapts fake, trace, and live inputs to SDK calls. These are concrete flat +modules, not additional protocol or service layers. + +Only `EvaluationBackend` and `CandidateGenerator` are protocols because each has +real fake, trace, and live substitutions. Attribution, comparison, normalization, +and gate logic are ordinary pure functions. + +The example does not extend or patch the SDK. Standard fake, trace, and live +evaluation calls `AgentEvaluator.evaluate_eval_set` with validated in-memory +`EvalSet` and `EvalConfig` objects. Only deterministic offline LLM metrics need a +run-local `EvaluatorRegistry`, which `AgentEvaluator` cannot receive per call; for +that narrow path `backends.py` composes the exported `LocalEvalService`, +`RemoteEvalService`, and `InMemoryEvalSetsManager` APIs and rebuilds the exported +`EvaluateResult` aggregate at the example boundary. The registry never mutates +global state. The SDK remains the owner of remote metric +compatibility: black-box evaluation rejects trajectory and knowledge-recall +metrics because callback results do not expose their required intermediate data. + +Offline substitutions preserve each metric's contract. `llm_final_response` uses +deterministic exact-reference scoring. Rubric metrics require an explicit +machine-readable offline rule in each rubric's `type`; natural-language rubric +text is never guessed or treated as a pass. Response rules can compare with the +reference, compare or search literal rubric content, or require a non-empty +response. Knowledge rules inspect configured knowledge-tool responses and are +trace-only because black-box callbacks do not expose intermediate data. Each +rubric is scored independently. Normalization preserves its ID, score, pass +status, and reason; attribution consumes only failed rubric outcomes. Unsupported +rules, missing operands, or unavailable evidence fail the evaluation instead of +fabricating a score. + +Candidate generators return independent cost sources instead of a lossy +aggregate. The SDK defines `OptimizeResult.total_llm_cost` as total optimizer +cost, including evaluator calls, so the live adapter records it once without an +invented unknown judge source. Live evaluation remains unknown unless both +per-agent-call and per-metric-call maxima are configured; then each evaluation +source is explicitly marked as an upper bound. Unknown accounting remains +`null` through the ledger, and an enabled cost gate rejects it with +`COST_UNAVAILABLE`. + +The report schema is `v2`. It intentionally replaces `rubricIds` with structured +`rubrics` and candidate aggregate accounting with source-level accounting; no +compatibility shim reconstructs the discarded `v1` information. + +## Lifecycle + +```text +validated = preflight(config, train, validation, prompts, trace, callback_source) +with prompt_set_lock(validated.prompt_paths): + baseline = evaluate(train, validation) + failures = attribute(structured_evidence, explicit_maps, reason_semantics, metric_fallback) + candidate = optimizer_worker(inner_train(failures), inner_selection) + regression = compare(evaluate(candidate, train, validation), baseline) + preliminary = hard_overfit_guard(regression) -> configured_gate(regression, cost, duration) + persist_pre_apply_audit_or_raise() + apply_verified(candidate) only when preliminary == ACCEPT and explicitly_authorized + terminal = configured_gate(regression, cost, current_duration) + restore_verified_baseline() if terminal rejects an applied candidate + persist_terminal_audit_once_or_raise() +on cancel: shielded_reaper(request_stop -> bounded_wait -> terminate -> bounded_wait -> kill) -> re-raise cancel +``` + +1. Preflight strictly parses config and datasets, rejects duplicate JSON keys, + validates split isolation, hashes every input, and validates the complete + trace phase/split/case matrix without side effects. +2. A cross-process lock is acquired for the complete prompt path set, then the + artifact directory is created exclusively; contention or a reused run ID + fails before prompt mutation. +3. Baseline train and held-out validation are evaluated sequentially. +4. A seeded inner train/selection split is persisted. Only inner-train failure + attribution is passed to the candidate generator. +5. The candidate is generated with source updates disabled, then evaluated on + full train and held-out validation inside a verified temporary prompt context. +6. Pure comparison and gate functions produce the only business decision. The + non-configurable overfit guard treats either train score or train pass-rate + improvement combined with either validation score or pass-rate regression as + a rejection. +7. A report is persisted before optional apply. Apply occurs only on ACCEPT when + explicitly enabled, and every write is read back and hash verified. +8. The terminal gate and report duration are sampled at the final decision + boundary. If that decision changes to REJECT after an apply, the baseline is + restored before publication. The terminal report, manifest, JSON snapshot, + and Markdown snapshot are then published once. Audit I/O is outside the + business duration gate; the 180-second end-to-end requirement is measured by + the acceptance runner. The report excludes its own files from its embedded + artifact list; the manifest hashes the final JSON and Markdown files. + +The immutable `artifacts//` directory and its manifest are the authority. +Root-level report files are atomic latest-run snapshots for interactive use. They +are not published as a transaction, so a consumer that needs a coherent JSON and +Markdown pair reads the run ID from the root JSON and then verifies both files in +the corresponding immutable run directory. Unique temporary names prevent +concurrent latest-run publishers from corrupting or deleting each other's writes. + +Any evaluator, optimizer, normalization, comparison, gate, render, apply, write, +cancel, or system-exit failure restores and verifies the baseline prompt. A +restoration failure raises `PromptRestoreError`; it is never downgraded to +REJECT. Reports store no chain-of-thought and recursively redact credentials. +Source `inputs.hashes` bind the submitted contracts used for replay. Separate +`inputs.auditHashes` and inner-split `auditHashes` bind the redacted dataset +copies stored on disk, so credential removal cannot invalidate or misrepresent +either fact. +Live optimizer config, prompt sandbox and raw optimizer output stay in an OS +temporary directory; only sanitized known artifact types enter the audit tree. +The import boundary snapshots only regular, single-link files. It rejects +symlinks, hard links and Windows reparse points, verifies file identity before +and after bounded reads, and enforces file-count, per-file-byte and total-byte +budgets before publishing any optimizer artifact. This boundary protects audit +storage; it is not +a sandbox for a programmatic generator, which is trusted in-process code. + +Default live evaluation and optimization resolve one importable callback through +`LiveAdapterSpec`; preflight binds its source path, file SHA-256, and callable code +fingerprint to the run, and the worker verifies all three after re-importing it. A +different callback object, source, cached code version, or source version is +rejected before optimization. Live optimization always uses +`optimizer_worker.py`. Cancellation starts an independent reaper that reuses one +observed wait task, writes the SDK stop signal, waits for the configured bound, +then terminates and finally kills a worker that still does not exit. Repeated +parent cancellation cannot interrupt this bounded cleanup. Programmatically injected +backends and generators remain trusted in-process components and make the report +non-reproducible. A successful SDK result must report the exact workspace +baseline and registered best-prompt keys before it can enter regression. Failed +or canceled SDK results retain their structured rounds, duration, error and cost +facts in the terminal report. Terminal report +persistence failures raise `AuditPersistenceError`; they are never discarded +while returning an apparently handled pipeline error. Error chains are +credential-redacted with bounded item count and total text size, retaining the +primary and restoration causes. Audit writes redact credentials without +truncation and fail before publication when the configured file-byte ceiling is +exceeded. + +A replay claim is emitted only for a clean, pinned Git commit when every +effective config, dataset, prompt, trace and live callback source is inside that +repository and tracked. Absolute external or untracked inputs remain executable +but are explicitly marked non-reproducible. + +## Industrial Acceptance + +The executable acceptance matrix covers every orchestration stage, cancellation, +partial backend failures, failed candidate accounting, malicious artifact sizes, +reference-free attribution, fake/trace replay, and live adapter shutdown. Local +policy matrices assert every declared attribution category and gate decision; +they are contract tests, not an independent accuracy benchmark. Issue-level +accuracy thresholds must be measured by a separately owned hidden corpus. Fake +and trace runs must finish within 180 seconds and produce hash-valid immutable +manifests. The dependency test is an allowlist for all flat pipeline modules and +rejects any new pipeline module, reverse import, or cycle until the architecture +contract is updated deliberately. + +The implementation stays in one flat package with explicit ownership and an +import allowlist. Line count is tracked as a review signal, not used as a reason +to merge unrelated responsibilities or introduce facade layers. The largest +modules remain validated result models, normalization, backend adapters, report +persistence, and the composition root. diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 000000000..781b136a2 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,75 @@ +# Evaluation + Optimization Loop + +This example runs a baseline on train and held-out validation data, attributes +failures, generates a candidate from an inner training split, reruns full +regression, and makes a deterministic gate decision. Every completed run writes +an immutable, authoritative audit directory under `artifacts//`. The +root-level generated `optimization_report.json` and `optimization_report.md` +files are ignored atomic latest-run convenience snapshots, not a transactional +report pair. Consumers that need a coherent pair must read the JSON run ID and +then use the two immutable files and manifest under +`artifacts//`. + +The default fake mode is deterministic, offline, and does not read API keys: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --run-id local-fake +``` + +Trace mode replays the hash-pinned fixture through the SDK evaluator: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --run-id local-trace +``` + +In fake and trace modes, LLM rubric metrics use explicit deterministic rules in +each rubric's `type`. Response rubrics support `OFFLINE_RESPONSE_EXACT_REFERENCE`, +`OFFLINE_RESPONSE_EQUALS`, `OFFLINE_RESPONSE_CONTAINS`, and +`OFFLINE_RESPONSE_NON_EMPTY`. Trace knowledge rubrics support +`OFFLINE_KNOWLEDGE_CONTAINS` and `OFFLINE_KNOWLEDGE_NON_EMPTY`. Literal operands +come from `content.text`. Natural-language-only rubrics and knowledge recall in +fake black-box mode fail fast because they cannot be scored deterministically. +Reports retain each rubric's ID, score, pass status, and reason. Failure +attribution uses only rubric outcomes that failed their metric threshold. + +Live mode requires one importable async `query -> str` callback. Preflight binds +its resolved source path, file SHA-256, and callable code fingerprint to the run; +the optimizer worker reloads the callback and verifies all three before use. The +optimizer is run in a supervised subprocess and always called with +`update_source=False`; only this pipeline's gate may apply a candidate: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --mode live --call-agent my_package.agent:call_agent --run-id live-review +``` + +Source prompts remain unchanged by default. Add `--apply-candidate` only when a +gate ACCEPT should be written back. REJECT is a completed audit run and exits 0; +ERROR exits non-zero. Run IDs are immutable and cannot be reused. A reproducible +report command preserves all effective inputs but appends `-replay` to the run ID +so it can execute without overwriting the authoritative original run. A report +is marked reproducible only when the worktree is clean and every effective +config, dataset, prompt, trace, and callback source is tracked by the pinned Git +commit. + +Cost is reported by source. Any unreported source makes the total cost unknown; +when a cost budget is enabled, the gate fails closed with `COST_UNAVAILABLE`. +Live evaluation can participate in a cost gate by configuring both +`liveAgentCallMaxCostUsd` and `liveMetricCallMaxCostUsd`; these values produce a +conservative upper bound rather than a fabricated bill. + +Optimizer artifacts are accepted only through a sanitized allowlist and bounded +by configurable file-count, per-file-byte, and total-byte limits. Live optimizer +cancellation first requests cooperative shutdown, then terminates the isolated +worker after `optimizerShutdownTimeoutSeconds`. Audit values are redacted but +never silently truncated; a report exceeding `maxAuditFileBytes` fails +explicitly. Concurrent runs targeting the same prompt files are rejected by a +cross-process lock. Failed optimizer runs retain reported rounds, costs, duration, +and error facts. The terminal decision and duration are finalized before one +terminal audit publication, so an applied prompt cannot be paired with a stale +ACCEPT report. A successful optimizer result is rejected before regression when +its reported baseline or best-prompt key set differs from the pipeline workspace. + +See [SOLUTION.md](SOLUTION.md) for the concise Chinese issue proposal and +[DESIGN.md](DESIGN.md) for detailed stage contracts and failure semantics. A +versioned example report is available under [sample_output](sample_output/). diff --git a/examples/optimization/eval_optimize_loop/SOLUTION.md b/examples/optimization/eval_optimize_loop/SOLUTION.md new file mode 100644 index 000000000..697ccdcd1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/SOLUTION.md @@ -0,0 +1,7 @@ +# 方案设计说明 + +本方案把评测、归因、优化、回归和审计组织为单向流水线。预检严格解析并哈希配置、数据集、prompt 与 trace，验证训练/验证隔离和完整 trace 矩阵；只有全部有效输入受同一干净 Git 提交管理时才声明可复现。基线阶段用 AgentEvaluator 分别运行训练集与验证集，保留逐用例、逐运行、逐指标分数、阈值、原因和关键轨迹。失败归因按结构证据、显式 rubric/指标映射、原因语义和指标兜底的顺序决策，每个失败输出主类别、次类别、证据与置信度。 + +候选生成只接收训练集内部切分的失败事实，不读取外部验证集；live 评测与优化绑定同一回调身份，优化运行在可终止 worker 中，取消先协作停止，超时再终止进程。候选完成后重跑完整训练集和保留验证集，逐用例标记新增通过、新增失败、提升、下降或不变。接受门禁依次检查验证总分与通过率、新增 hard fail、关键用例、指标回退、成本和耗时；训练分数或通过率提升而验证下降是不可配置的过拟合拒绝条件，未知成本在启用预算时失败关闭，live 评测可使用显式单次调用上界。 + +prompt 由跨进程锁保护，默认不回写；仅门禁接受且显式授权时原子应用、回读和验哈希，终态拒绝、异常或取消必须恢复基线。审计一次发布终态，保存各终态完整候选轮次、成本来源、耗时、种子、门禁理由和产物哈希，只脱敏而不静默截断，超出文件预算或终态落盘失败会显式报错。fake/trace 使用确定性规则且不读真实密钥，最终生成 JSON、Markdown 和清单，支持复现、比较与追责。 diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 000000000..202118203 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,85 @@ +{ + "evaluate": { + "metrics": [ + { + "metricName": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "finalResponse": { + "text": { + "match": "exact", + "caseInsensitive": false + } + } + } + } + ], + "numRuns": 1 + }, + "optimize": { + "evalCaseParallelism": 1, + "stop": { + "requiredMetrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflectionLm": { + "modelName": "${TRPC_AGENT_MODEL_NAME}", + "baseUrl": "${TRPC_AGENT_BASE_URL}", + "apiKey": "${TRPC_AGENT_API_KEY}", + "generationConfig": { + "maxTokens": 1024, + "temperature": 0.2 + } + }, + "reflectionMinibatchSize": 2, + "skipPerfectScore": false, + "maxMetricCalls": 12 + } + }, + "pipeline": { + "mode": "fake", + "seed": 42, + "innerSelectionRatio": 0.34, + "applyCandidate": false, + "artifactRoot": "artifacts", + "traceFixture": "traces/trace_cases.json", + "maxAuditFileBytes": 26214400, + "liveAgentCallMaxCostUsd": null, + "liveMetricCallMaxCostUsd": null, + "optimizerShutdownTimeoutSeconds": 10.0, + "maxImportFiles": 256, + "maxImportFileBytes": 5242880, + "maxImportTotalBytes": 26214400, + "promptPaths": { + "system": "prompts/system.md" + }, + "criticalCaseIds": [ + "validation_stable" + ], + "hardCaseIds": [ + "validation_regress" + ], + "metricWeights": { + "final_response_avg_score": 1.0 + }, + "attribution": { + "metricCategories": { + "final_response_avg_score": "FINAL_RESPONSE_MISMATCH" + } + }, + "gate": { + "minValidationScoreDelta": 0.05, + "minValidationPassRateDelta": 0.0, + "maxNewHardFailures": 0, + "criticalCaseMinDelta": 0.0, + "metricMaxRegression": { + "final_response_avg_score": 0.0 + }, + "maxCostUsd": null, + "maxDurationSeconds": 180.0, + "epsilon": 1e-09 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline/__init__.py b/examples/optimization/eval_optimize_loop/pipeline/__init__.py new file mode 100644 index 000000000..80dac5c6b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/__init__.py @@ -0,0 +1 @@ +"""Auditable evaluation and prompt-optimization pipeline example.""" diff --git a/examples/optimization/eval_optimize_loop/pipeline/artifacts.py b/examples/optimization/eval_optimize_loop/pipeline/artifacts.py new file mode 100644 index 000000000..ce868fd80 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/artifacts.py @@ -0,0 +1,346 @@ +"""Immutable audit directory, recursive sanitization and content manifest.""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import threading +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .models import ArtifactRecord +from .schema import parse_strict_json, sanitize, validate_safe_component + +_REPORT_FILES = {"optimization_report.json", "optimization_report.md"} +_TEXT_ARTIFACT_SUFFIXES = {".log", ".md", ".txt"} +_ATOMIC_REPLACE_LOCK = threading.Lock() +_ATOMIC_REPLACE_ATTEMPTS = 8 + + +class AuditPersistenceError(RuntimeError): + """Raised when a terminal audit report cannot be durably persisted.""" + + +def load_strict_json(path: str | Path) -> dict[str, Any]: + return parse_strict_json(Path(path).read_text(encoding="utf-8")) + + +@dataclass(frozen=True) +class _ArtifactSnapshot: + relative_path: str + content: str + + +_REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) +_NO_FOLLOW = getattr(os, "O_NOFOLLOW", 0) + + +def _stat_is_link_or_reparse(info: os.stat_result) -> bool: + return stat.S_ISLNK(info.st_mode) or bool(getattr(info, "st_file_attributes", 0) & _REPARSE_POINT) + + +def _path_is_junction(path: Path) -> bool: + is_junction = getattr(path, "is_junction", None) + return bool(is_junction()) if callable(is_junction) else False + + +def _require_safe_root(path: Path) -> os.stat_result: + try: + info = os.lstat(path) + except FileNotFoundError: + raise FileNotFoundError(path) from None + if _stat_is_link_or_reparse(info) or _path_is_junction(path): + raise ValueError("optimizer artifact symlinks, hard links and reparse points are not allowed") + if not stat.S_ISDIR(info.st_mode): + raise ValueError("optimizer artifact source must be a directory") + return info + + +def _same_file_identity(before: os.stat_result, after: os.stat_result) -> bool: + return ( + before.st_dev, + before.st_ino, + before.st_mode, + before.st_size, + before.st_mtime_ns, + ) == ( + after.st_dev, + after.st_ino, + after.st_mode, + after.st_size, + after.st_mtime_ns, + ) + + +class AuditSink: + """Own safe paths and atomic persistence beneath one immutable run directory.""" + + def __init__( + self, + artifact_root: str | Path, + run_id: str, + *, + publication_root: str | Path | None = None, + max_file_bytes: int = 25 * 1024 * 1024, + max_import_files: int = 256, + max_import_file_bytes: int = 5 * 1024 * 1024, + max_import_total_bytes: int = 25 * 1024 * 1024, + ) -> None: + validate_safe_component(run_id, name="run ID") + self.root = Path(artifact_root).resolve() + self.publication_root = Path(publication_root).resolve() if publication_root is not None else self.root.parent + self.run_id = run_id + self.run_dir = self.root / run_id + if min(max_file_bytes, max_import_files, max_import_file_bytes, max_import_total_bytes) < 1: + raise ValueError("audit and import limits must be positive") + if max_import_total_bytes < max_import_file_bytes: + raise ValueError("total import byte limit must be at least the per-file limit") + self.max_file_bytes = max_file_bytes + self.max_import_files = max_import_files + self.max_import_file_bytes = max_import_file_bytes + self.max_import_total_bytes = max_import_total_bytes + + def create(self) -> Path: + self.root.mkdir(parents=True, exist_ok=True) + self.run_dir.mkdir(exist_ok=False) + return self.run_dir + + def phase_dir(self, name: str) -> Path: + validate_safe_component(name, name="stage name") + path = self.run_dir / name + path.mkdir(exist_ok=False) + return path + + def _resolve(self, relative_path: str) -> Path: + raw = Path(relative_path) + if raw.is_absolute() or not raw.parts: + raise ValueError("artifact path must be relative") + for part in raw.parts: + validate_safe_component(part, name="artifact path component") + resolved = (self.run_dir / raw).resolve() + if self.run_dir not in resolved.parents: + raise ValueError("artifact path escapes the run directory") + return resolved + + def _atomic_write(self, path: Path, content: str) -> None: + byte_size = len(content.encode("utf-8")) + if byte_size > self.max_file_bytes: + raise ValueError(f"audit file exceeds byte limit: {path.name} ({byte_size} > {self.max_file_bytes})") + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + temporary.write_text(content, encoding="utf-8", newline="\n") + with _ATOMIC_REPLACE_LOCK: + for attempt in range(_ATOMIC_REPLACE_ATTEMPTS): + try: + os.replace(temporary, path) + break + except PermissionError: + if attempt + 1 == _ATOMIC_REPLACE_ATTEMPTS: + raise + time.sleep(0.005 * (2**attempt)) + finally: + temporary.unlink(missing_ok=True) + + def write_json(self, relative_path: str, payload: Any) -> Path: + path = self._resolve(relative_path) + clean = sanitize(payload, max_text_chars=None) + content = json.dumps(clean, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + "\n" + self._atomic_write(path, content) + return path + + def write_text(self, relative_path: str, content: str) -> Path: + path = self._resolve(relative_path) + clean = sanitize(content, max_text_chars=None) + if not isinstance(clean, str): + raise TypeError("text audit content must remain text after sanitization") + self._atomic_write(path, clean) + return path + + def write_jsonl(self, relative_path: str, payloads: list[dict[str, Any]]) -> Path: + path = self._resolve(relative_path) + lines = [ + json.dumps( + sanitize(payload, max_text_chars=None), + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) for payload in payloads + ] + self._atomic_write(path, "\n".join(lines) + ("\n" if lines else "")) + return path + + def _read_verified_file(self, path: Path, before: os.stat_result) -> bytes: + if before.st_nlink != 1: + raise ValueError(f"optimizer artifact hard links are not allowed: {path.name}") + flags = os.O_RDONLY | _NO_FOLLOW + try: + descriptor = os.open(path, flags) + except OSError as error: + raise ValueError(f"optimizer artifact could not be opened safely: {path.name}") from error + try: + opened = os.fstat(descriptor) + if _stat_is_link_or_reparse(opened) or not stat.S_ISREG(opened.st_mode): + raise ValueError(f"optimizer artifact is not a regular file: {path.name}") + if opened.st_nlink != 1 or not _same_file_identity(before, opened): + raise ValueError(f"optimizer artifact changed during validation: {path.name}") + chunks: list[bytes] = [] + remaining = self.max_import_file_bytes + 1 + while remaining: + chunk = os.read(descriptor, min(1024 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + after = os.fstat(descriptor) + if not _same_file_identity(opened, after): + raise ValueError(f"optimizer artifact changed while reading: {path.name}") + content = b"".join(chunks) + if len(content) > self.max_import_file_bytes: + raise ValueError(f"optimizer artifact exceeds per-file byte limit: " + f"{path.name} ({len(content)} > {self.max_import_file_bytes})") + return content + finally: + os.close(descriptor) + + @staticmethod + def _render_json(payload: Any) -> str: + return json.dumps( + sanitize(payload, max_text_chars=None), + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + "\n" + + @staticmethod + def _render_jsonl(payload: Any) -> str: + return json.dumps( + sanitize(payload, max_text_chars=None), + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + + def _snapshot_file(self, path: Path, relative_path: str, before: os.stat_result) -> _ArtifactSnapshot: + suffix = path.suffix.casefold() + if suffix not in {".json", ".jsonl", *_TEXT_ARTIFACT_SUFFIXES}: + raise ValueError(f"unsupported optimizer artifact type: {relative_path}") + if before.st_size > self.max_import_file_bytes: + raise ValueError(f"optimizer artifact exceeds per-file byte limit: " + f"{relative_path} ({before.st_size} > {self.max_import_file_bytes})") + raw = self._read_verified_file(path, before) + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as error: + raise ValueError(f"optimizer text artifact is not UTF-8: {relative_path}") from error + if suffix == ".json": + content = self._render_json(parse_strict_json(text)) + elif suffix == ".jsonl": + payloads = [parse_strict_json(line) for line in text.splitlines() if line.strip()] + content = "\n".join(self._render_jsonl(payload) for payload in payloads) + content += "\n" if payloads else "" + else: + clean = sanitize(text, max_text_chars=None) + if not isinstance(clean, str): + raise TypeError("text audit content must remain text after sanitization") + content = clean + if len(content.encode("utf-8")) > self.max_file_bytes: + raise ValueError(f"audit file exceeds byte limit: {relative_path}") + return _ArtifactSnapshot(relative_path=relative_path, content=content) + + def _snapshot_tree(self, source: str | Path) -> tuple[_ArtifactSnapshot, ...]: + source_root = Path(source) + if _path_is_junction(source_root): + raise ValueError("optimizer artifact symlinks, hard links and reparse points are not allowed") + try: + _require_safe_root(source_root) + except FileNotFoundError: + return () + pending: list[tuple[Path, str]] = [(source_root, "")] + snapshots: list[_ArtifactSnapshot] = [] + total_bytes = 0 + while pending: + directory, prefix = pending.pop() + try: + entries = sorted(os.scandir(directory), key=lambda entry: entry.name) + except OSError as error: + raise ValueError(f"optimizer artifact directory cannot be read: {directory}") from error + for entry in entries: + relative = f"{prefix}/{entry.name}" if prefix else entry.name + for component in Path(relative).parts: + validate_safe_component(component, name="optimizer artifact component") + try: + info = os.lstat(entry.path) + except OSError as error: + raise ValueError(f"optimizer artifact cannot be inspected: {relative}") from error + if _stat_is_link_or_reparse(info) or _path_is_junction(Path(entry.path)): + raise ValueError(f"optimizer artifact symlinks, hard links and reparse points are not allowed: " + f"{relative}") + if stat.S_ISDIR(info.st_mode): + pending.append((Path(entry.path), relative)) + continue + if not stat.S_ISREG(info.st_mode): + raise ValueError(f"optimizer artifact must be a regular file: {relative}") + if len(snapshots) >= self.max_import_files: + raise ValueError(f"optimizer artifact count exceeds limit: " + f"{len(snapshots) + 1} > {self.max_import_files}") + total_bytes += info.st_size + if total_bytes > self.max_import_total_bytes: + raise ValueError(f"optimizer artifacts exceed total byte limit: " + f"{total_bytes} > {self.max_import_total_bytes}") + snapshots.append(self._snapshot_file(Path(entry.path), relative, info)) + return tuple(snapshots) + + def import_tree(self, source: str | Path, destination: str) -> None: + """Validate and sanitize the complete optimizer tree before publishing it.""" + + snapshots = self._snapshot_tree(source) + for snapshot in snapshots: + target = (Path(destination) / snapshot.relative_path).as_posix() + self._atomic_write(self._resolve(target), snapshot.content) + + def records( + self, + *, + include_manifest: bool = False, + include_report_files: bool = True, + ) -> tuple[ArtifactRecord, ...]: + records: list[ArtifactRecord] = [] + for path in sorted(item for item in self.run_dir.rglob("*") if item.is_file()): + relative = path.relative_to(self.run_dir).as_posix() + if not include_manifest and relative == "manifest.json": + continue + if not include_report_files and relative in _REPORT_FILES: + continue + content = path.read_bytes() + records.append( + ArtifactRecord( + path=relative, + sha256=hashlib.sha256(content).hexdigest(), + byte_size=len(content), + )) + return tuple(records) + + def write_manifest(self) -> tuple[ArtifactRecord, ...]: + records = self.records() + self.write_json( + "manifest.json", + { + "schemaVersion": "v1", + "files": [record.model_dump(by_alias=True) for record in records] + }, + ) + return records + + def publish_latest_snapshot(self, source_name: str, destination_name: str) -> Path: + source = self._resolve(source_name) + validate_safe_component(destination_name, name="latest report name") + destination = self.publication_root / destination_name + self._atomic_write(destination, source.read_text(encoding="utf-8")) + return destination diff --git a/examples/optimization/eval_optimize_loop/pipeline/attribution.py b/examples/optimization/eval_optimize_loop/pipeline/attribution.py new file mode 100644 index 000000000..c4de09800 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/attribution.py @@ -0,0 +1,229 @@ +"""Deterministic, evidence-based failure attribution without I/O or model calls.""" + +from __future__ import annotations + +from collections import Counter +from typing import Any + +from trpc_agent_sdk.evaluation import Invocation, get_all_tool_calls + +from .configuration import AttributionConfig +from .models import ( + AttributionPair, + AttributionSnapshot, + AttributionStatistics, + EvaluationSnapshot, + FailureAttribution, + FailureCategory, + SnapshotPair, +) +from .schema import sanitized_text + +_KNOWN_METRICS = { + "tool_trajectory_avg_score": FailureCategory.TOOL_CALL_ERROR, + "llm_rubric_knowledge_recall": FailureCategory.KNOWLEDGE_RECALL_INSUFFICIENT, + "llm_rubric_response": FailureCategory.LLM_RUBRIC_NOT_MET, + "llm_final_response": FailureCategory.LLM_RUBRIC_NOT_MET, + "response_evaluation_score": FailureCategory.LLM_RUBRIC_NOT_MET, + "final_response_avg_score": FailureCategory.FINAL_RESPONSE_MISMATCH, + "response_match_score": FailureCategory.FINAL_RESPONSE_MISMATCH, +} + + +def _statistics(failures: tuple[FailureAttribution, ...]) -> AttributionStatistics: + counts = Counter(failure.primary for failure in failures) + return AttributionStatistics( + total_failures=len(failures), + primary_category_counts=dict(counts), + ) + + +def _safe_text(value: Any, max_chars: int) -> str: + return sanitized_text(value, max_text_chars=max_chars) + + +def _tool_calls(invocation: dict[str, Any]) -> list[tuple[str, Any]]: + parsed = Invocation.model_validate(invocation) + return [(str(call.name), call.args) for call in get_all_tool_calls(parsed.intermediate_data)] + + +def _tool_differences(case: Any, max_chars: int) -> list[tuple[FailureCategory, str]]: + differences: list[tuple[FailureCategory, str]] = [] + for run in case.runs: + for trace in run.trace: + actual = _tool_calls(trace["actual"]) + expected_payload = trace["expected"] + if expected_payload is None: + continue + expected = _tool_calls(expected_payload) + actual_names = [name for name, _ in actual] + expected_names = [name for name, _ in expected] + if actual_names != expected_names: + differences.append(( + FailureCategory.TOOL_CALL_ERROR, + _safe_text({ + "expectedToolNames": expected_names, + "actualToolNames": actual_names + }, max_chars), + )) + continue + argument_diffs = [{ + "tool": actual_item[0], + "expected": expected_item[1], + "actual": actual_item[1] + } for actual_item, expected_item in zip(actual, expected) if actual_item[1] != expected_item[1]] + if argument_diffs: + differences.append((FailureCategory.TOOL_ARGUMENT_ERROR, _safe_text(argument_diffs, max_chars))) + return differences + + +def _fallback_category(reason: str) -> FailureCategory: + lowered = reason.casefold() + if "format" in lowered or "schema" in lowered: + return FailureCategory.FORMAT_VIOLATION + if "argument" in lowered or "parameter" in lowered: + return FailureCategory.TOOL_ARGUMENT_ERROR + if "tool" in lowered or "function" in lowered: + return FailureCategory.TOOL_CALL_ERROR + if "knowledge" in lowered or "recall" in lowered or "retriev" in lowered: + return FailureCategory.KNOWLEDGE_RECALL_INSUFFICIENT + if "rubric" in lowered or "judge" in lowered: + return FailureCategory.LLM_RUBRIC_NOT_MET + if "response" in lowered or "mismatch" in lowered or "answer" in lowered: + return FailureCategory.FINAL_RESPONSE_MISMATCH + return FailureCategory.UNKNOWN + + +def attribute_failures( + snapshot: EvaluationSnapshot, + config: AttributionConfig, + *, + max_text_chars: int, +) -> AttributionSnapshot: + """Attribute each failed case using the documented precedence order.""" + + failures: list[FailureAttribution] = [] + for case in snapshot.cases: + if case.passed: + continue + candidates: list[tuple[FailureCategory, str]] = [] + reasons: list[str] = [] + evidence: list[str] = [] + trigger_metrics: list[str] = [] + trigger_rubrics: list[str] = [] + + def add_candidate(category: FailureCategory, confidence: str) -> None: + if all(existing != category for existing, _ in candidates): + candidates.append((category, confidence)) + + if case.error: + add_candidate(FailureCategory.EVALUATION_ERROR, "high") + reasons.append("Evaluator reported an execution error.") + evidence.append(_safe_text(case.error, max_text_chars)) + + tool_differences = _tool_differences(case, max_text_chars) + structured_tool_categories = {category for category, _ in tool_differences} + for category, item_evidence in tool_differences: + add_candidate(category, "high") + reasons.append("Structured expected and actual tool trajectories differ.") + evidence.append(item_evidence) + + for run in case.runs: + for metric in run.metrics: + if metric.passed: + continue + trigger_metrics.append(metric.metric_name) + reason = metric.reason or ( + f"Metric {metric.metric_name} scored {metric.score:g} below threshold {metric.threshold:g}.") + reasons.append(_safe_text(reason, max_text_chars)) + evidence.append( + _safe_text( + { + "metric": metric.metric_name, + "score": metric.score, + "threshold": metric.threshold, + }, + max_text_chars, + )) + for rubric in metric.rubrics: + if rubric.passed: + continue + trigger_rubrics.append(rubric.id) + if rubric.id in config.rubric_categories: + add_candidate(config.rubric_categories[rubric.id], "high") + if metric.metric_name in config.metric_categories: + add_candidate(config.metric_categories[metric.metric_name], "high") + + semantic = _fallback_category(reason) + if semantic not in { + FailureCategory.UNKNOWN, + FailureCategory.FINAL_RESPONSE_MISMATCH, + }: + add_candidate(semantic, "medium") + if metric.metric_name in _KNOWN_METRICS and not (metric.metric_name == "tool_trajectory_avg_score" + and structured_tool_categories): + add_candidate(_KNOWN_METRICS[metric.metric_name], "medium") + if semantic == FailureCategory.FINAL_RESPONSE_MISMATCH: + add_candidate(semantic, "medium") + + if not candidates: + candidates.append((FailureCategory.UNKNOWN, "low")) + primary, confidence = candidates[0] + categories = tuple(category for category, _ in candidates) + failures.append( + FailureAttribution( + case_id=case.case_id, + primary=primary, + secondary=categories[1:], + reasons=tuple(dict.fromkeys(reasons or ["The failure has no mapped deterministic cause."])), + trigger_metrics=tuple(dict.fromkeys(trigger_metrics)), + trigger_rubrics=tuple(dict.fromkeys(trigger_rubrics)), + evidence=tuple(dict.fromkeys(evidence or ["No structured evidence was available."])), + confidence=confidence, + )) + attributed = tuple(failures) + return AttributionSnapshot( + split=snapshot.split, + phase=snapshot.phase, + failures=attributed, + statistics=_statistics(attributed), + ) + + +def attribute_pair( + snapshots: SnapshotPair, + config: AttributionConfig, + *, + max_text_chars: int, +) -> AttributionPair: + """Attribute a complete train/validation phase without side effects.""" + + if snapshots.train is None or snapshots.validation is None: + raise ValueError("attribution requires complete train and validation snapshots") + return AttributionPair( + train=attribute_failures( + snapshots.train, + config, + max_text_chars=max_text_chars, + ), + validation=attribute_failures( + snapshots.validation, + config, + max_text_chars=max_text_chars, + ), + ) + + +def select_attribution( + snapshot: AttributionSnapshot, + case_ids: set[str] | frozenset[str], +) -> AttributionSnapshot: + """Restrict attribution facts to the cases visible to a downstream stage.""" + + selected = tuple(failure for failure in snapshot.failures if failure.case_id in case_ids) + return AttributionSnapshot( + split=snapshot.split, + phase=snapshot.phase, + failures=selected, + statistics=_statistics(selected), + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/backends.py b/examples/optimization/eval_optimize_loop/pipeline/backends.py new file mode 100644 index 000000000..9c0c92a7c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/backends.py @@ -0,0 +1,674 @@ +"""SDK-backed fake, trace and live adapters plus candidate generators.""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import sys +from copy import deepcopy +from pathlib import Path +from typing import Awaitable, Callable, Optional + +from trpc_agent_sdk.evaluation import ( + AgentEvaluator, + EvalConfig, + EvalSet, + EvalSetAggregateResult, + EvaluatorRegistry, + EvaluateConfig, + EvaluateRequest, + EvaluateResult, + InMemoryEvalSetsManager, + InferenceConfig, + InferenceRequest, + LocalEvalService, + OptimizeResult, + RemoteEvalService, + TargetPrompt, +) + +from .contracts import CandidateGenerator, EvaluationBackend +from .live_adapter import LiveAdapterSpec +from .models import ( + AttributionSnapshot, + CandidateProposal, + CandidateRound, + CostSource, + Phase, + Split, +) +from .offline_evaluation import prepare_offline_evaluation +from .schema import add_exception_note, parse_strict_json, sanitized_text +from .trace_fixture import TraceFixture + +CallAgent = Callable[[str], Awaitable[str]] +_DEFAULT_APP_NAME = "test_app" +_REPOSITORY_ROOT = Path(__file__).resolve().parents[4] +_WORKER_STDERR_TAIL_BYTES = 4000 + + +async def _evaluate_with_sdk( + eval_set: EvalSet, + eval_config: EvalConfig, + *, + call_agent: Optional[CallAgent], + runtime_dir: str, + evaluator_registry: Optional[EvaluatorRegistry] = None, +) -> EvaluateResult: + """Use AgentEvaluator by default and services only for a run-local registry.""" + + dataset = deepcopy(eval_set) + config = deepcopy(eval_config) + runtime_root = Path(runtime_dir) + if not runtime_root.is_dir(): + raise FileNotFoundError(f"backend audit directory does not exist: {runtime_root}") + if config.num_runs < 1: + raise ValueError("evaluation num_runs must be at least one") + + app_name = dataset.app_name or _DEFAULT_APP_NAME + if evaluator_registry is None: + _, _, _, results_by_case = await AgentEvaluator.evaluate_eval_set( + dataset, + call_agent=call_agent, + eval_config=config, + num_runs=config.num_runs, + print_detailed_results=False, + ) + return EvaluateResult( + results_by_eval_set_id={ + dataset.eval_set_id: + EvalSetAggregateResult( + eval_results_by_eval_id=results_by_case, + num_runs=config.num_runs, + ) + }) + + # AgentEvaluator has no per-call registry hook. Keep custom offline judges + # scoped to this run by composing the exported services directly. + manager = InMemoryEvalSetsManager() + manager.create_eval_set(app_name=app_name, eval_set_id=dataset.eval_set_id) + for eval_case in dataset.eval_cases: + manager.add_eval_case( + app_name=app_name, + eval_set_id=dataset.eval_set_id, + eval_case=eval_case, + ) + + if call_agent is None: + service = LocalEvalService( + root_agent=None, + eval_sets_manager=manager, + evaluator_registry=evaluator_registry, + ) + else: + service = RemoteEvalService( + call_agent=call_agent, + eval_sets_manager=manager, + evaluator_registry=evaluator_registry, + ) + + inference_results = [] + inference_request = InferenceRequest( + app_name=app_name, + eval_set_id=dataset.eval_set_id, + inference_config=InferenceConfig(), + ) + for run_id in range(1, config.num_runs + 1): + async for inference_result in service.perform_inference(inference_request): + inference_result.run_id = run_id + inference_results.append(inference_result) + + results_by_case = {} + evaluate_request = EvaluateRequest( + inference_results=inference_results, + evaluate_config=EvaluateConfig(eval_metrics=config.get_eval_metrics()), + ) + async for case_result in service.evaluate(evaluate_request): + results_by_case.setdefault(case_result.eval_id, []).append(case_result) + + return EvaluateResult( + results_by_eval_set_id={ + dataset.eval_set_id: EvalSetAggregateResult( + eval_results_by_eval_id=results_by_case, + num_runs=config.num_runs, + ) + }) + + +async def _evaluate_offline( + eval_set: EvalSet, + eval_config: EvalConfig, + *, + call_agent: Optional[CallAgent], + runtime_dir: str, +) -> EvaluateResult: + """Evaluate using a run-local deterministic replacement registry.""" + + offline_config, registry = prepare_offline_evaluation(eval_config) + return await _evaluate_with_sdk( + eval_set, + offline_config, + call_agent=call_agent, + runtime_dir=runtime_dir, + evaluator_registry=registry, + ) + + +def _prompt_profile(prompts: dict[str, str]) -> bool: + return any("BEHAVIOR_PROFILE=precision-v2" in text for text in prompts.values()) + + +def deterministic_fake_response(query: str, prompts: dict[str, str]) -> str: + """Offline agent used by the public example; behavior is data-driven, not ID-driven.""" + + fields: dict[str, str] = {} + for component in query.split(";"): + if ":" in component: + key, value = component.split(":", 1) + fields[key.strip().upper()] = value.strip() + behavior = fields.get("BEHAVIOR") + answer = fields.get("ANSWER") + if behavior not in {"improve", "regress", "stable"} or answer is None: + return "INVALID_REQUEST" + candidate = _prompt_profile(prompts) + if behavior == "improve": + return answer if candidate else "INCORRECT" + if behavior == "regress": + return "INCORRECT" if candidate else answer + return answer + + +class FakeEvaluationBackend: + """Deterministic evaluator that never reads API credentials or uses the network.""" + + async def evaluate( + self, + *, + eval_set: EvalSet, + eval_config: EvalConfig, + prompts: dict[str, str], + split: Split, + phase: Phase, + audit_dir: str, + ) -> EvaluateResult: + del split, phase + prompt_copy = deepcopy(prompts) + + async def call_agent(query: str) -> str: + return deterministic_fake_response(query, prompt_copy) + + return await _evaluate_offline( + eval_set, + eval_config, + call_agent=call_agent, + runtime_dir=audit_dir, + ) + + +class TraceEvaluationBackend: + """Replay actual conversations from a hash-pinned fixture through LocalEvalService.""" + + def __init__( + self, + fixture_path: str, + dataset_hashes: dict[str, str], + fixture_hash: Optional[str] = None, + ) -> None: + self._fixture = TraceFixture(fixture_path, dataset_hashes, fixture_hash) + + def validate_fixture(self, train: EvalSet, validation: EvalSet) -> None: + """Fail before run creation when any pinned trace input has drifted.""" + + self._fixture.validate(train, validation) + + def _trace_eval_set(self, eval_set: EvalSet, split: Split, phase: Phase) -> EvalSet: + return self._fixture.eval_set(eval_set, split, phase) + + async def evaluate( + self, + *, + eval_set: EvalSet, + eval_config: EvalConfig, + prompts: dict[str, str], + split: Split, + phase: Phase, + audit_dir: str, + ) -> EvaluateResult: + del prompts + traced = self._trace_eval_set(deepcopy(eval_set), split, phase) + return await _evaluate_offline( + traced, + eval_config, + call_agent=None, + runtime_dir=audit_dir, + ) + + +class LiveEvaluationBackend: + """Run an injected async business callback through RemoteEvalService.""" + + def __init__(self, call_agent: CallAgent) -> None: + if not inspect.iscoroutinefunction(call_agent): + raise TypeError("live call_agent must be an async function") + self._call_agent = call_agent + + async def evaluate( + self, + *, + eval_set: EvalSet, + eval_config: EvalConfig, + prompts: dict[str, str], + split: Split, + phase: Phase, + audit_dir: str, + ) -> EvaluateResult: + del prompts, split, phase + return await _evaluate_with_sdk( + eval_set, + eval_config, + call_agent=self._call_agent, + runtime_dir=audit_dir, + ) + + +class DeterministicCandidateGenerator: + """Generate one auditable candidate solely from inner-train failure facts.""" + + async def generate( + self, + *, + target_prompt: TargetPrompt, + baseline_prompts: dict[str, str], + train_attribution: AttributionSnapshot, + inner_train_path: str, + inner_selection_path: str, + config_path: str, + output_dir: str, + ) -> CandidateProposal: + del inner_train_path, inner_selection_path, config_path, output_dir + if set(target_prompt.names()) != set(baseline_prompts): + raise ValueError("candidate target fields do not match baseline prompts") + changed = bool(train_attribution.failures) + prompts = dict(baseline_prompts) + rounds: tuple[CandidateRound, ...] = () + if changed: + for name in prompts: + prompts[name] = prompts[name].rstrip() + "\n\nBEHAVIOR_PROFILE=precision-v2\n" + rounds = (CandidateRound( + round=1, + candidate_prompts=prompts, + accepted=True, + score=1.0, + kind="deterministic", + optimized_fields=tuple(prompts), + acceptance_reason="deterministic failure-attribution rewrite", + cost_usd=0, + ), ) + proposal = CandidateProposal( + algorithm="deterministic_failure_rewrite", + baseline_prompts=dict(baseline_prompts), + prompts=prompts, + changed=changed, + stop_reason="completed", + rounds=rounds, + cost_sources=(CostSource(name="deterministic", cost_usd=0, model_calls=0, metric_calls=0), ), + duration_seconds=0, + ) + return proposal + + +class LiveCandidateGenerator: + """Adapt AgentOptimizer output into the strict candidate-only contract.""" + + def __init__( + self, + live_adapter: LiveAdapterSpec, + *, + shutdown_timeout_seconds: float = 10.0, + verbose: int = 0, + ) -> None: + if shutdown_timeout_seconds <= 0: + raise ValueError("optimizer shutdown timeout must be positive") + self._live_adapter = live_adapter + self._shutdown_timeout_seconds = shutdown_timeout_seconds + self._verbose = verbose + + @staticmethod + def _request_stop(output_dir: str, cancellation: BaseException) -> None: + # The SDK's GepaReflectiveOptimizer registers a FileStopper for this + # exact path. Forced process termination remains the bounded fallback. + stop_path = Path(output_dir) / "optimize.stop" + try: + stop_path.parent.mkdir(parents=True, exist_ok=True) + stop_path.write_text("cancel requested\n", encoding="utf-8") + except OSError as stop_error: + add_exception_note(cancellation, f"could not request optimizer stop: {stop_error}") + + @staticmethod + async def _capture_worker_stderr(stderr: Optional[asyncio.StreamReader]) -> str: + """Drain worker stderr continuously and retain only a sanitized tail.""" + + if stderr is None: + return "" + tail = bytearray() + truncated = False + try: + while True: + chunk = await stderr.read(8192) + if not chunk: + break + if len(chunk) >= _WORKER_STDERR_TAIL_BYTES: + tail[:] = chunk[-_WORKER_STDERR_TAIL_BYTES:] + truncated = True + continue + tail.extend(chunk) + overflow = len(tail) - _WORKER_STDERR_TAIL_BYTES + if overflow > 0: + del tail[:overflow] + truncated = True + except asyncio.CancelledError: + raise + except Exception as capture_error: + detail = sanitized_text(capture_error, max_text_chars=1000) + return f"stderr capture failed with {type(capture_error).__name__}: {detail}" + if not tail: + return "" + text = tail.decode("utf-8", errors="replace") + if truncated: + line_end = text.find("\n") + if line_end < 0: + return "[stderr tail omitted: final line exceeded capture limit]" + text = text[line_end + 1:] + return sanitized_text(text, max_text_chars=_WORKER_STDERR_TAIL_BYTES) + + async def _wait_for_worker( + self, + process: asyncio.subprocess.Process, + cancellation: BaseException, + wait_task: Optional[asyncio.Task[int]], + ) -> tuple[bool, Optional[asyncio.Task[int]]]: + if wait_task is None: + wait_task = asyncio.create_task(process.wait()) + try: + await asyncio.wait_for( + asyncio.shield(wait_task), + timeout=self._shutdown_timeout_seconds, + ) + return True, None + except asyncio.TimeoutError: + return False, wait_task + except asyncio.CancelledError: + raise + except Exception as wait_error: + detail = sanitized_text(wait_error, max_text_chars=1000) + add_exception_note( + cancellation, + f"optimizer worker wait failed with {type(wait_error).__name__}: {detail}", + ) + return process.returncode is not None, None + + @staticmethod + def _signal_worker( + process: asyncio.subprocess.Process, + signal_name: str, + cancellation: BaseException, + ) -> bool: + if process.returncode is not None: + return False + try: + getattr(process, signal_name)() + return True + except ProcessLookupError: + return False + except OSError as signal_error: + detail = sanitized_text(signal_error, max_text_chars=1000) + add_exception_note( + cancellation, + f"optimizer worker {signal_name} failed with {type(signal_error).__name__}: {detail}", + ) + return False + + async def _stop_worker( + self, + process: asyncio.subprocess.Process, + *, + output_dir: str, + cancellation: BaseException, + ) -> None: + self._request_stop(output_dir, cancellation) + wait_task: Optional[asyncio.Task[int]] = None + forced = False + try: + exited, wait_task = await self._wait_for_worker(process, cancellation, wait_task) + if exited: + return + forced = self._signal_worker(process, "terminate", cancellation) + exited, wait_task = await self._wait_for_worker(process, cancellation, wait_task) + if exited: + if forced: + add_exception_note(cancellation, "optimizer worker required forced termination") + return + forced = self._signal_worker(process, "kill", cancellation) or forced + exited, wait_task = await self._wait_for_worker(process, cancellation, wait_task) + if not exited: + add_exception_note(cancellation, "optimizer worker did not report exit after kill") + if forced: + add_exception_note(cancellation, "optimizer worker required forced termination") + finally: + if wait_task is not None: + wait_task.cancel() + try: + await wait_task + except BaseException: + pass + + async def _complete_worker_stop( + self, + process: asyncio.subprocess.Process, + *, + output_dir: str, + cancellation: BaseException, + stderr_capture: asyncio.Task[str], + ) -> None: + """Finish bounded child cleanup even if the parent task is canceled again.""" + + async def stop_and_discard_stderr() -> None: + try: + await self._stop_worker( + process, + output_dir=output_dir, + cancellation=cancellation, + ) + finally: + if not stderr_capture.done(): + stderr_capture.cancel() + try: + await stderr_capture + except asyncio.CancelledError: + pass + + cleanup = asyncio.create_task(stop_and_discard_stderr()) + repeated_cancellations = 0 + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + repeated_cancellations += 1 + continue + except BaseException: + break + if repeated_cancellations: + add_exception_note( + cancellation, + f"worker cleanup resisted {repeated_cancellations} additional cancellation request(s)", + ) + if cleanup.cancelled(): + add_exception_note(cancellation, "optimizer worker cleanup task was canceled before completion") + return + try: + cleanup.result() + except BaseException as cleanup_error: + detail = sanitized_text(cleanup_error, max_text_chars=1000) + add_exception_note( + cancellation, + f"optimizer worker cleanup failed with {type(cleanup_error).__name__}: {detail}", + ) + + async def _optimize_in_worker(self, **kwargs) -> OptimizeResult: + output_dir = kwargs["output_dir"] + request_path = Path(output_dir).parent / "optimizer-worker-request.json" + prompt_paths = {name: kwargs["target_prompt"].describe_source(name) for name in kwargs["target_prompt"].names()} + request_path.write_text( + json.dumps( + { + "callbackSpec": self._live_adapter.import_path, + "callbackSourcePath": str(self._live_adapter.source_path), + "callbackSourceSha256": self._live_adapter.source_sha256, + "callbackCallableSha256": self._live_adapter.callable_sha256, + "promptPaths": prompt_paths, + "configPath": kwargs["config_path"], + "trainPath": kwargs["train_dataset_path"], + "validationPath": kwargs["validation_dataset_path"], + "outputDir": output_dir, + "verbose": kwargs["verbose"], + }, + ensure_ascii=False, + allow_nan=False, + ), + encoding="utf-8", + ) + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "examples.optimization.eval_optimize_loop.pipeline.optimizer_worker", + str(request_path), + cwd=str(_REPOSITORY_ROOT), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + stderr_capture = asyncio.create_task(self._capture_worker_stderr(getattr(process, "stderr", None))) + try: + return_code = await process.wait() + except asyncio.CancelledError as cancellation: + await self._complete_worker_stop( + process, + output_dir=output_dir, + cancellation=cancellation, + stderr_capture=stderr_capture, + ) + raise + stderr_detail = await stderr_capture + result_path = Path(output_dir) / "result.json" + if return_code != 0: + error_path = Path(output_dir) / "worker_error.json" + detail = "optimizer worker failed without an error artifact" + if error_path.is_file(): + payload = parse_strict_json(error_path.read_text(encoding="utf-8")) + detail = str(payload.get("message", detail)) + elif stderr_detail: + detail = f"{detail}; sanitized stderr tail: {stderr_detail}" + raise RuntimeError(f"AgentOptimizer worker failed with exit code {return_code}: {detail}") + if not result_path.is_file(): + raise RuntimeError("AgentOptimizer worker completed without result.json") + return OptimizeResult.from_file(str(result_path)) + + async def generate( + self, + *, + target_prompt: TargetPrompt, + baseline_prompts: dict[str, str], + train_attribution: AttributionSnapshot, + inner_train_path: str, + inner_selection_path: str, + config_path: str, + output_dir: str, + ) -> CandidateProposal: + del train_attribution + optimize_kwargs = { + "config_path": config_path, + "target_prompt": target_prompt, + "train_dataset_path": inner_train_path, + "validation_dataset_path": inner_selection_path, + "output_dir": output_dir, + "update_source": False, + "verbose": self._verbose, + } + result = await self._optimize_in_worker(**optimize_kwargs) + reported_baseline = dict(result.baseline_prompts) + result_prompts = dict(result.best_prompts) + adapter_errors: list[str] = [] + if result.status == "SUCCEEDED": + if reported_baseline != baseline_prompts: + adapter_errors.append("optimizer baseline prompts differ from the workspace baseline") + if set(result_prompts) != set(baseline_prompts): + adapter_errors.append("optimizer best prompt keys differ from the workspace schema") + if result.status != "SUCCEEDED" or adapter_errors: + result_prompts = dict(baseline_prompts) + rounds = tuple( + CandidateRound( + round=round_.round, + candidate_prompts=dict(round_.candidate_prompts), + accepted=round_.accepted, + score=(round_.validation_pass_rate if round_.candidate_prompts and not getattr( + round_, "skip_reason", None) and not getattr(round_, "error_message", None) else None), + kind=getattr(round_, "kind", "reflective"), + optimized_fields=tuple(getattr(round_, "optimized_field_names", ())), + metric_scores=dict(getattr(round_, "metric_breakdown", {})), + acceptance_reason=getattr(round_, "acceptance_reason", None) or None, + skip_reason=getattr(round_, "skip_reason", None), + error_message=getattr(round_, "error_message", None), + cost_usd=round_.round_llm_cost, + token_usage=dict(round_.round_token_usage), + duration_seconds=round_.duration_seconds, + ) for round_ in result.rounds) + proposal = CandidateProposal( + status=result.status, + error_message=result.error_message or None, + adapter_error="; ".join(adapter_errors) or None, + algorithm=result.algorithm, + baseline_prompts=dict(baseline_prompts), + prompts=result_prompts, + changed=result_prompts != baseline_prompts, + stop_reason=result.stop_reason, + rounds=rounds, + cost_sources=(CostSource( + name="optimizer_reported", + cost_usd=result.total_llm_cost, + model_calls=result.total_reflection_lm_calls, + token_usage=dict(result.total_token_usage), + ), ), + duration_seconds=result.duration_seconds, + ) + return proposal + + +def create_backends( + mode: str, + *, + live_adapter: Optional[LiveAdapterSpec] = None, + trace_fixture_path: Optional[str] = None, + trace_fixture_hash: Optional[str] = None, + dataset_hashes: Optional[dict[str, str]] = None, + optimizer_shutdown_timeout_seconds: float = 10.0, +) -> tuple[EvaluationBackend, CandidateGenerator]: + if mode == "fake": + return FakeEvaluationBackend(), DeterministicCandidateGenerator() + if mode == "trace": + if not trace_fixture_path or dataset_hashes is None: + raise ValueError("trace mode requires a fixture path and dataset hashes") + return ( + TraceEvaluationBackend(trace_fixture_path, dataset_hashes, trace_fixture_hash), + DeterministicCandidateGenerator(), + ) + if mode == "live": + if live_adapter is None: + raise ValueError("live mode requires a validated LiveAdapterSpec") + return ( + LiveEvaluationBackend(live_adapter.callback), + LiveCandidateGenerator( + live_adapter, + shutdown_timeout_seconds=optimizer_shutdown_timeout_seconds, + ), + ) + raise ValueError(f"unsupported mode: {mode!r}") diff --git a/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py b/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py new file mode 100644 index 000000000..818619058 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/candidate_runtime.py @@ -0,0 +1,135 @@ +"""Isolated candidate-generation workspace and sanitized artifact ingestion.""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path + +from trpc_agent_sdk.evaluation import EvalConfig, EvalSet, OptimizeConfig + +from .artifacts import AuditSink +from .attribution import select_attribution +from .contracts import CandidateGenerator +from .evaluation import ( + dataset_audit_fingerprint, + dataset_contract_payload, + dataset_fingerprint, + split_train_dataset, +) +from .models import ( + AttributionSnapshot, + CandidateProposal, + EvaluationSnapshot, + InnerSplit, +) +from .prompt_workspace import PromptWorkspace +from .schema import add_exception_note, validate_secret_free_text + + +def prepare_candidate_inputs( + *, + train: EvalSet, + baseline: EvaluationSnapshot, + attribution: AttributionSnapshot, + seed: int, + selection_ratio: float, + sink: AuditSink, +) -> tuple[InnerSplit, AttributionSnapshot, Path, Path]: + """Create and persist the optimizer-only inner split and attribution.""" + + inner_train, inner_selection = split_train_dataset( + train, + baseline, + seed=seed, + selection_ratio=selection_ratio, + ) + train_path = sink.write_json( + "inner_train.evalset.json", + dataset_contract_payload(inner_train), + ) + selection_path = sink.write_json( + "inner_selection.evalset.json", + dataset_contract_payload(inner_selection), + ) + inner = InnerSplit( + train_case_ids=tuple(case.eval_id for case in inner_train.eval_cases), + selection_case_ids=tuple(case.eval_id for case in inner_selection.eval_cases), + train_hash=dataset_fingerprint(inner_train), + selection_hash=dataset_fingerprint(inner_selection), + audit_hashes={ + "train": dataset_audit_fingerprint(inner_train), + "selection": dataset_audit_fingerprint(inner_selection), + }, + train_path=train_path.relative_to(sink.run_dir).as_posix(), + selection_path=selection_path.relative_to(sink.run_dir).as_posix(), + ) + selected = select_attribution(attribution, frozenset(inner.train_case_ids)) + sink.write_json( + "inner_train.attribution.json", + selected.model_dump(mode="json", by_alias=True), + ) + return inner, selected, train_path, selection_path + + +async def generate_candidate( + *, + generator: CandidateGenerator, + workspace: PromptWorkspace, + sink: AuditSink, + eval_config: EvalConfig, + optimize_config: OptimizeConfig, + train_attribution: AttributionSnapshot, + inner_train_path: Path, + inner_selection_path: Path, +) -> CandidateProposal: + """Run a generator outside the audit tree, then import only sanitized output.""" + + sink.phase_dir("candidate_generation") + with tempfile.TemporaryDirectory(prefix="trpc-agent-eval-optimize-") as runtime_value: + runtime_root = Path(runtime_value) + candidate_target = workspace.create_candidate_target(str(runtime_root / "prompt_sandbox")) + runtime_config = runtime_root / "optimizer.json" + runtime_config.write_text( + json.dumps( + { + "evaluate": eval_config.model_dump(mode="json", by_alias=True), + "optimize": optimize_config.model_dump(mode="json", by_alias=True), + }, + ensure_ascii=False, + allow_nan=False, + ), + encoding="utf-8", + ) + optimizer_output = runtime_root / "optimizer-output" + try: + proposal = await generator.generate( + target_prompt=candidate_target, + baseline_prompts=workspace.baseline, + train_attribution=train_attribution, + inner_train_path=str(inner_train_path), + inner_selection_path=str(inner_selection_path), + config_path=str(runtime_config), + output_dir=str(optimizer_output), + ) + except BaseException as primary_error: + try: + sink.import_tree(optimizer_output, "candidate_generation/optimizer") + except BaseException as import_error: + add_exception_note( + primary_error, + f"optimizer artifact import also failed with " + f"{type(import_error).__name__}: {import_error}", + ) + raise + else: + sink.import_tree(optimizer_output, "candidate_generation/optimizer") + validated = CandidateProposal.model_validate(proposal) + if validated.baseline_prompts != workspace.baseline: + raise ValueError("candidate generator baseline prompts differ from the workspace baseline") + for name, text in validated.prompts.items(): + validate_secret_free_text(text, name=f"candidate prompt {name!r}") + for round_ in validated.rounds: + for name, text in round_.candidate_prompts.items(): + validate_secret_free_text(text, name=f"candidate round {round_.round} prompt {name!r}") + return validated diff --git a/examples/optimization/eval_optimize_loop/pipeline/configuration.py b/examples/optimization/eval_optimize_loop/pipeline/configuration.py new file mode 100644 index 000000000..e3196a669 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/configuration.py @@ -0,0 +1,164 @@ +"""Validated input and policy configuration contracts.""" + +from __future__ import annotations + +from typing import Optional + +from pydantic import ( + Field, + StrictBool, + StrictFloat, + StrictInt, + field_validator, + model_validator, +) + +from trpc_agent_sdk.evaluation import EvalConfig, EvalSet, OptimizeConfig + +from .live_adapter import LiveAdapterSpec +from .models import FailureCategory, RunMode +from .schema import ( + StrictModel, + finite_number as _finite, + non_negative_number as _non_negative, +) + + +class AttributionConfig(StrictModel): + metric_categories: dict[str, FailureCategory] = Field(default_factory=dict) + rubric_categories: dict[str, FailureCategory] = Field(default_factory=dict) + + +class GateConfig(StrictModel): + min_validation_score_delta: StrictFloat = 0.0 + min_validation_pass_rate_delta: StrictFloat = 0.0 + max_new_hard_failures: StrictInt = Field(default=0, ge=0) + critical_case_min_delta: StrictFloat = 0.0 + metric_max_regression: dict[str, StrictFloat] = Field(default_factory=dict) + max_cost_usd: Optional[StrictFloat] = None + max_duration_seconds: Optional[StrictFloat] = 180.0 + epsilon: StrictFloat = 1e-9 + + @field_validator( + "min_validation_score_delta", + "min_validation_pass_rate_delta", + "critical_case_min_delta", + ) + @classmethod + def finite_threshold(cls, value: float) -> float: + return _finite(value, "gate threshold") + + @field_validator("epsilon") + @classmethod + def valid_epsilon(cls, value: float) -> float: + return _non_negative(value, "epsilon") + + @field_validator("max_cost_usd", "max_duration_seconds") + @classmethod + def valid_budget(cls, value: Optional[float]) -> Optional[float]: + return None if value is None else _non_negative(value, "budget") + + @field_validator("metric_max_regression") + @classmethod + def valid_metric_regression(cls, value: dict[str, float]) -> dict[str, float]: + for metric, limit in value.items(): + if not metric: + raise ValueError("metric regression keys must be non-empty") + _non_negative(limit, f"metric_max_regression[{metric!r}]") + return value + + +class PipelineSettings(StrictModel): + mode: RunMode = "fake" + seed: StrictInt = 42 + inner_selection_ratio: StrictFloat = 0.34 + apply_candidate: StrictBool = False + artifact_root: str = "artifacts" + trace_fixture: str = "traces/trace_cases.json" + run_id: Optional[str] = None + prompt_paths: dict[str, str] = Field(default_factory=lambda: {"system": "prompts/system.md"}) + critical_case_ids: tuple[str, ...] = () + hard_case_ids: tuple[str, ...] = () + metric_weights: dict[str, StrictFloat] = Field(default_factory=dict) + train_case_weights: dict[str, StrictFloat] = Field(default_factory=dict) + validation_case_weights: dict[str, StrictFloat] = Field(default_factory=dict) + max_text_chars: StrictInt = Field(default=4000, ge=64, le=100_000) + max_audit_file_bytes: StrictInt = Field(default=25 * 1024 * 1024, ge=1024, le=500 * 1024 * 1024) + live_agent_call_max_cost_usd: Optional[StrictFloat] = None + live_metric_call_max_cost_usd: Optional[StrictFloat] = None + optimizer_shutdown_timeout_seconds: StrictFloat = Field(default=10.0, ge=0.1, le=300.0) + max_import_files: StrictInt = Field(default=256, ge=1, le=10_000) + max_import_file_bytes: StrictInt = Field(default=5 * 1024 * 1024, ge=1, le=100 * 1024 * 1024) + max_import_total_bytes: StrictInt = Field(default=25 * 1024 * 1024, ge=1, le=500 * 1024 * 1024) + attribution: AttributionConfig = Field(default_factory=AttributionConfig) + gate: GateConfig = Field(default_factory=GateConfig) + + @model_validator(mode="after") + def valid_import_budget(self) -> "PipelineSettings": + if self.max_import_total_bytes < self.max_import_file_bytes: + raise ValueError("max_import_total_bytes must be at least max_import_file_bytes") + return self + + @model_validator(mode="after") + def valid_live_cost_bounds(self) -> "PipelineSettings": + bounds = ( + self.live_agent_call_max_cost_usd, + self.live_metric_call_max_cost_usd, + ) + if (bounds[0] is None) != (bounds[1] is None): + raise ValueError("live Agent and metric call cost bounds must be configured together") + for value in bounds: + if value is not None: + _non_negative(value, "live call cost bound") + return self + + @field_validator("inner_selection_ratio") + @classmethod + def valid_ratio(cls, value: float) -> float: + value = _finite(value, "inner_selection_ratio") + if not 0 < value < 1: + raise ValueError("inner_selection_ratio must be in (0, 1)") + return value + + @field_validator("metric_weights", "train_case_weights", "validation_case_weights") + @classmethod + def positive_weights(cls, value: dict[str, float]) -> dict[str, float]: + for key, weight in value.items(): + if not key: + raise ValueError("weight keys must be non-empty") + if _finite(weight, f"weight[{key!r}]") <= 0: + raise ValueError("weights must be positive") + return value + + @field_validator("prompt_paths") + @classmethod + def non_empty_prompts(cls, value: dict[str, str]) -> dict[str, str]: + if not value: + raise ValueError("at least one prompt path is required") + return value + + +class PipelineConfig(StrictModel): + evaluate: EvalConfig + optimize: OptimizeConfig + pipeline: PipelineSettings = Field(default_factory=PipelineSettings) + + +class ValidatedRunConfig(StrictModel): + root_dir: str + config_path: str + train_path: str + validation_path: str + trace_fixture_path: Optional[str] + artifact_root: str + run_id: str + config: PipelineConfig + train: EvalSet + validation: EvalSet + input_hashes: dict[str, str] + audit_hashes: dict[str, str] + prompt_paths: dict[str, str] + prompt_hashes: dict[str, str] + adapter_identity: str + reproducibility_paths: tuple[str, ...] + live_adapter: Optional[LiveAdapterSpec] = Field(default=None, exclude=True, repr=False) diff --git a/examples/optimization/eval_optimize_loop/pipeline/contracts.py b/examples/optimization/eval_optimize_loop/pipeline/contracts.py new file mode 100644 index 000000000..e7677e6d3 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/contracts.py @@ -0,0 +1,42 @@ +"""The two deliberate substitution boundaries used by the pipeline.""" + +from __future__ import annotations + +from typing import Protocol + +from trpc_agent_sdk.evaluation import EvalConfig, EvalSet, EvaluateResult, TargetPrompt + +from .models import AttributionSnapshot, CandidateProposal, Phase, Split + + +class EvaluationBackend(Protocol): + """Evaluate an immutable request and return the SDK's raw result object.""" + + async def evaluate( # noqa: E704 + self, + *, + eval_set: EvalSet, + eval_config: EvalConfig, + prompts: dict[str, str], + split: Split, + phase: Phase, + audit_dir: str, + ) -> EvaluateResult: + ... + + +class CandidateGenerator(Protocol): + """Propose prompts from inner-training evidence without making the gate decision.""" + + async def generate( # noqa: E704 + self, + *, + target_prompt: TargetPrompt, + baseline_prompts: dict[str, str], + train_attribution: AttributionSnapshot, + inner_train_path: str, + inner_selection_path: str, + config_path: str, + output_dir: str, + ) -> CandidateProposal: + ... diff --git a/examples/optimization/eval_optimize_loop/pipeline/costing.py b/examples/optimization/eval_optimize_loop/pipeline/costing.py new file mode 100644 index 000000000..32d7821aa --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/costing.py @@ -0,0 +1,39 @@ +"""Completed-call cost ledger and deterministic source merge.""" + +from __future__ import annotations + +from .models import CostSource, CostSummary + + +class CostLedger: + """Run-local ledger; callers record a source only after its call completes.""" + + def __init__(self) -> None: + self._sources: list[CostSource] = [] + + def record( + self, + name: str, + *, + cost_usd: float | None, + upper_bound: bool = False, + model_calls: int | None = None, + metric_calls: int | None = None, + token_usage: dict[str, int] | None = None, + ) -> None: + if any(source.name == name for source in self._sources): + raise ValueError(f"cost source already recorded: {name!r}") + self._sources.append( + CostSource( + name=name, + cost_usd=cost_usd, + upper_bound=upper_bound, + model_calls=model_calls, + metric_calls=metric_calls, + token_usage=dict(token_usage or {}), + )) + + def summary(self) -> CostSummary: + known = all(source.cost_usd is not None for source in self._sources) + total = sum(source.cost_usd or 0 for source in self._sources) if known else None + return CostSummary(sources=tuple(self._sources), total_cost_usd=total) diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluation.py b/examples/optimization/eval_optimize_loop/pipeline/evaluation.py new file mode 100644 index 000000000..ef40c618c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluation.py @@ -0,0 +1,516 @@ +"""Pure dataset validation, deterministic splitting, normalization and comparison.""" + +from __future__ import annotations + +import hashlib +import json +import math +from copy import deepcopy +from typing import Any, Iterable, Mapping + +from trpc_agent_sdk.evaluation import EvalConfig, EvalSet, EvalStatus, EvaluateResult + +from .models import ( + AttributionSnapshot, + CaseComparison, + CaseRun, + CaseSnapshot, + ComparisonSnapshot, + EvaluationSnapshot, + FailureAttribution, + MetricDelta, + MetricRun, + MetricSnapshot, + Phase, + RubricOutcome, + Split, + Transition, +) +from .schema import sanitize + +_VOLATILE_KEYS = { + "invocationId", + "invocation_id", + "callId", + "call_id", + "toolCallId", + "tool_call_id", + "executionId", + "execution_id", + "eventId", + "event_id", + "creationTimestamp", + "creation_timestamp", +} + + +def canonical_json(value: Any) -> str: + """Return deterministic JSON suitable for content hashing.""" + + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def sha256_json(value: Any) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def _without_execution_ids(value: Any) -> Any: + if isinstance(value, dict): + return {key: _without_execution_ids(item) for key, item in value.items() if key not in _VOLATILE_KEYS} + if isinstance(value, list): + return [_without_execution_ids(item) for item in value] + return value + + +def case_content_fingerprint(case: Any) -> str: + payload = case.model_dump(mode="json", by_alias=True) + payload.pop("evalId", None) + return sha256_json(_without_execution_ids(payload)) + + +def case_input_fingerprint(case: Any) -> str: + """Fingerprint only model-visible inputs so changed labels cannot hide leakage.""" + + invocations = case.conversation or case.actual_conversation or [] + payload = { + "userContents": [invocation.user_content.model_dump(mode="json", by_alias=True) for invocation in invocations], + "conversationScenario": (case.conversation_scenario.model_dump(mode="json", by_alias=True) + if case.conversation_scenario is not None else None), + "sessionInput": + (case.session_input.model_dump(mode="json", by_alias=True) if case.session_input is not None else None), + "contextMessages": [item.model_dump(mode="json", by_alias=True) for item in (case.context_messages or [])], + } + return sha256_json(_without_execution_ids(payload)) + + +def dataset_contract_payload(eval_set: EvalSet) -> dict[str, Any]: + """Return the submitted dataset contract without injected defaults.""" + + return eval_set.model_dump(mode="json", by_alias=True, exclude_unset=True) + + +def dataset_fingerprint_payload(payload: Mapping[str, Any]) -> str: + """Hash a serialized dataset contract after removing execution-only IDs.""" + + return sha256_json(_without_execution_ids(payload)) + + +def dataset_fingerprint(eval_set: EvalSet) -> str: + """Hash the source dataset contract used for replay and trace validation.""" + + return dataset_fingerprint_payload(dataset_contract_payload(eval_set)) + + +def dataset_audit_payload(eval_set: EvalSet) -> dict[str, Any]: + """Return the exact credential-redacted payload persisted in the audit tree.""" + + return sanitize(dataset_contract_payload(eval_set), max_text_chars=None) + + +def dataset_audit_fingerprint(eval_set: EvalSet) -> str: + """Hash the redacted dataset artifact independently from its source hash.""" + + return dataset_fingerprint_payload(dataset_audit_payload(eval_set)) + + +def _validate_case_ids(eval_set: EvalSet, label: str, minimum: int) -> set[str]: + if len(eval_set.eval_cases) < minimum: + raise ValueError(f"{label} must contain at least {minimum} cases") + case_ids = [case.eval_id for case in eval_set.eval_cases] + if any(not case_id or case_id.strip() != case_id for case_id in case_ids): + raise ValueError(f"{label} case IDs must be non-empty and already trimmed") + if len(case_ids) != len(set(case_ids)): + raise ValueError(f"{label} case IDs must be unique") + return set(case_ids) + + +def validate_datasets( + train: EvalSet, + validation: EvalSet, + *, + train_path: str, + validation_path: str, + configured_metrics: Iterable[str], + critical_case_ids: Iterable[str] = (), + hard_case_ids: Iterable[str] = (), + metric_weights: Mapping[str, float] | None = None, + train_case_weights: Mapping[str, float] | None = None, + validation_case_weights: Mapping[str, float] | None = None, +) -> None: + """Validate split isolation and all configured references.""" + + if train_path.casefold() == validation_path.casefold(): + raise ValueError("train and validation paths must differ") + train_ids = _validate_case_ids(train, "train", 3) + validation_ids = _validate_case_ids(validation, "validation", 1) + overlap = train_ids & validation_ids + if overlap: + raise ValueError(f"train and validation share case IDs: {sorted(overlap)}") + + train_fingerprints = {case_content_fingerprint(case) for case in train.eval_cases} + validation_fingerprints = {case_content_fingerprint(case) for case in validation.eval_cases} + if train_fingerprints & validation_fingerprints: + raise ValueError("train and validation contain duplicate case content") + train_inputs = {case_input_fingerprint(case) for case in train.eval_cases} + validation_inputs = {case_input_fingerprint(case) for case in validation.eval_cases} + if train_inputs & validation_inputs: + raise ValueError("train and validation contain duplicate model inputs") + + validation_references = set(critical_case_ids) | set(hard_case_ids) + unknown_validation = validation_references - validation_ids + if unknown_validation: + raise ValueError(f"critical/hard cases are missing from validation: {sorted(unknown_validation)}") + + metric_list = list(configured_metrics) + if not metric_list or any(not name or name != name.strip() for name in metric_list): + raise ValueError("configured metric names must be non-empty and trimmed") + if len(metric_list) != len(set(metric_list)): + raise ValueError("configured metric names must be unique") + metrics = set(metric_list) + unknown_metrics = set(metric_weights or {}) - metrics + if unknown_metrics: + raise ValueError(f"metric weights reference unknown metrics: {sorted(unknown_metrics)}") + unknown_train = set(train_case_weights or {}) - train_ids + if unknown_train: + raise ValueError(f"train weights reference unknown cases: {sorted(unknown_train)}") + unknown_val = set(validation_case_weights or {}) - validation_ids + if unknown_val: + raise ValueError(f"validation weights reference unknown cases: {sorted(unknown_val)}") + + +def split_train_dataset( + train: EvalSet, + baseline: EvaluationSnapshot, + *, + seed: int, + selection_ratio: float, +) -> tuple[EvalSet, EvalSet]: + """Return deterministic inner-train and selection sets, stratified when possible.""" + + case_ids = [case.eval_id for case in train.eval_cases] + if tuple(case_ids) != baseline.case_ids: + raise ValueError("baseline train snapshot does not match train dataset") + if len(case_ids) < 3: + raise ValueError("inner split requires at least three train cases") + selection_count = 1 if len(case_ids) == 3 else round(len(case_ids) * selection_ratio) + selection_count = max(1, min(len(case_ids) - 1, selection_count)) + + passed = {case.case_id: case.passed for case in baseline.cases} + + def rank(case_id: str) -> str: + return hashlib.sha256(f"{seed}:{case_id}".encode("utf-8")).hexdigest() + + groups = { + False: sorted((case_id for case_id in case_ids if not passed[case_id]), key=rank), + True: sorted((case_id for case_id in case_ids if passed[case_id]), key=rank), + } + selection: list[str] = [] + if selection_count >= 2 and groups[False] and groups[True]: + selection.extend((groups[False].pop(0), groups[True].pop(0))) + remaining = sorted(groups[False] + groups[True], key=rank) + selection.extend(remaining[:selection_count - len(selection)]) + selection_ids = set(selection) + + train_cases = [deepcopy(case) for case in train.eval_cases if case.eval_id not in selection_ids] + selection_cases = [deepcopy(case) for case in train.eval_cases if case.eval_id in selection_ids] + inner_train_update = {"eval_set_id": f"{train.eval_set_id}_inner_train", "eval_cases": train_cases} + inner_selection_update = {"eval_set_id": f"{train.eval_set_id}_inner_selection", "eval_cases": selection_cases} + inner_train = train.model_copy(update=inner_train_update, deep=True) + inner_selection = train.model_copy(update=inner_selection_update, deep=True) + return inner_train, inner_selection + + +def _metric_config(eval_config: EvalConfig) -> tuple[list[str], dict[str, float]]: + metrics = eval_config.get_eval_metrics() + names = [metric.metric_name for metric in metrics] + if not names or len(names) != len(set(names)): + raise ValueError("configured metric names must be non-empty and unique") + thresholds = {metric.metric_name: float(metric.threshold) for metric in metrics} + if any(not math.isfinite(value) for value in thresholds.values()): + raise ValueError("configured metric thresholds must be finite") + return names, thresholds + + +def _status_passed(status: EvalStatus) -> bool: + if status == EvalStatus.NOT_EVALUATED: + raise ValueError("metric status cannot be NOT_EVALUATED") + return status == EvalStatus.PASSED + + +def _same_model(left: Any, right: Any) -> bool: + if left is None or right is None: + return left is right + return left.model_dump(mode="json", by_alias=True) == right.model_dump(mode="json", by_alias=True) + + +def _normalize_run( + result: Any, + *, + expected_case: Any, + expected_run_id: int, + eval_set_id: str, + metric_names: list[str], + thresholds: dict[str, float], +) -> CaseRun: + if result.eval_set_id != eval_set_id or result.eval_id != expected_case.eval_id: + raise ValueError("run eval-set/case identity drift") + if result.run_id != expected_run_id: + raise ValueError("run IDs must be contiguous and 1-based") + overall = result.overall_eval_metric_results + if [item.metric_name for item in overall] != metric_names: + raise ValueError("overall metric schema drift") + + has_reference = bool(expected_case.conversation) + expected_invocations = expected_case.conversation or expected_case.actual_conversation or [] + per_invocation = result.eval_metric_result_per_invocation + if len(per_invocation) != len(expected_invocations): + raise ValueError("invocation count drift") + trace: list[dict[str, Any]] = [] + for index, invocation_result in enumerate(per_invocation): + expected = expected_invocations[index] + if has_reference: + if not _same_model(invocation_result.expected_invocation, expected): + raise ValueError("expected invocation drift") + else: + placeholder = invocation_result.expected_invocation + if (placeholder is None or placeholder.final_response is not None + or placeholder.intermediate_data is not None + or not _same_model(placeholder.user_content, expected.user_content)): + raise ValueError("reference-free expected invocation drift") + if not _same_model(invocation_result.actual_invocation.user_content, expected.user_content): + raise ValueError("actual user content drift") + per_metric = invocation_result.eval_metric_results + if [item.metric_name for item in per_metric] != metric_names: + raise ValueError("per-invocation metric schema drift") + for item in per_metric: + if item.score is None or not math.isfinite(float(item.score)): + raise ValueError("per-invocation metric score must be finite") + if float(item.threshold) != thresholds[item.metric_name]: + raise ValueError("per-invocation metric threshold drift") + if _status_passed(item.eval_status) != (float(item.score) >= float(item.threshold)): + raise ValueError("per-invocation metric status drift") + trace.append({ + "actual": invocation_result.actual_invocation.model_dump(mode="json", by_alias=True), + "expected": (expected.model_dump(mode="json", by_alias=True) if has_reference else None), + }) + + metrics: list[MetricRun] = [] + for item in overall: + if item.score is None or not math.isfinite(float(item.score)): + raise ValueError("overall metric score must be finite") + if float(item.threshold) != thresholds[item.metric_name]: + raise ValueError("overall metric threshold drift") + passed = _status_passed(item.eval_status) + if passed != (float(item.score) >= float(item.threshold)): + raise ValueError("overall metric status drift") + details = item.details + rubrics: list[RubricOutcome] = [] + if details and details.rubric_scores: + for rubric in details.rubric_scores: + if isinstance(rubric, Mapping): + rubric_id = rubric.get("rubricId") or rubric.get("rubric_id") or rubric.get("id") + rubric_score = rubric.get("score") + rubric_reason = rubric.get("reason") + else: + rubric_id = getattr(rubric, "rubric_id", None) or getattr(rubric, "id", None) + rubric_score = getattr(rubric, "score", None) + rubric_reason = getattr(rubric, "reason", None) + if not rubric_id or rubric_score is None: + raise ValueError("rubric result requires a non-empty id and score") + score = float(rubric_score) + rubrics.append( + RubricOutcome( + id=str(rubric_id), + score=score, + passed=score >= float(item.threshold), + reason=str(rubric_reason) if rubric_reason is not None else None, + )) + metrics.append( + MetricRun( + metric_name=item.metric_name, + score=float(item.score), + threshold=float(item.threshold), + passed=passed, + reason=details.reason if details else None, + rubrics=tuple(rubrics), + )) + final_passed = _status_passed(result.final_eval_status) + expected_passed = result.error_message is None and all(metric.passed for metric in metrics) + if final_passed != expected_passed: + raise ValueError("final run status drift") + return CaseRun( + run_id=expected_run_id, + passed=final_passed, + metrics=tuple(metrics), + error=result.error_message, + trace=tuple(trace), + ) + + +def _weighted_average(values: Iterable[tuple[float, float]]) -> float: + pairs = list(values) + total_weight = sum(weight for _, weight in pairs) + if not pairs or total_weight <= 0: + raise ValueError("weighted average requires positive total weight") + return sum(value * weight for value, weight in pairs) / total_weight + + +def normalize_result( + raw: EvaluateResult, + eval_set: EvalSet, + eval_config: EvalConfig, + *, + split: Split, + phase: Phase, + metric_weights: Mapping[str, float] | None = None, + case_weights: Mapping[str, float] | None = None, +) -> EvaluationSnapshot: + """Validate and unfold an SDK EvaluateResult exactly once.""" + + if set(raw.results_by_eval_set_id) != {eval_set.eval_set_id}: + raise ValueError("EvaluateResult must contain exactly the expected eval set") + aggregate = raw.results_by_eval_set_id[eval_set.eval_set_id] + if aggregate.num_runs != eval_config.num_runs: + raise ValueError("aggregate run count does not match eval config") + expected_ids = [case.eval_id for case in eval_set.eval_cases] + if set(aggregate.eval_results_by_eval_id) != set(expected_ids): + raise ValueError("EvaluateResult case IDs do not match input dataset") + metric_names, thresholds = _metric_config(eval_config) + metric_weight_map = {name: float((metric_weights or {}).get(name, 1.0)) for name in metric_names} + + cases: list[CaseSnapshot] = [] + for expected_case in eval_set.eval_cases: + raw_runs = aggregate.eval_results_by_eval_id[expected_case.eval_id] + if len(raw_runs) != eval_config.num_runs: + raise ValueError("case run count does not match eval config") + by_run_id = {run.run_id: run for run in raw_runs} + if set(by_run_id) != set(range(1, eval_config.num_runs + 1)): + raise ValueError("case run IDs are missing or duplicated") + runs = tuple( + _normalize_run( + by_run_id[run_id], + expected_case=expected_case, + expected_run_id=run_id, + eval_set_id=eval_set.eval_set_id, + metric_names=metric_names, + thresholds=thresholds, + ) for run_id in range(1, eval_config.num_runs + 1)) + metric_snapshots: list[MetricSnapshot] = [] + for metric_name in metric_names: + run_metrics = [next(metric for metric in run.metrics if metric.metric_name == metric_name) for run in runs] + score = sum(metric.score for metric in run_metrics) / len(run_metrics) + metric_snapshots.append( + MetricSnapshot( + metric_name=metric_name, + score=score, + threshold=thresholds[metric_name], + passed=all(metric.passed for metric in run_metrics), + )) + case_score = _weighted_average( + (metric.score, metric_weight_map[metric.metric_name]) for metric in metric_snapshots) + cases.append( + CaseSnapshot( + case_id=expected_case.eval_id, + passed=all(run.passed for run in runs), + score=case_score, + metrics=tuple(metric_snapshots), + runs=runs, + error=next((run.error for run in runs if run.error), None), + )) + + case_weight_map = {case.case_id: float((case_weights or {}).get(case.case_id, 1.0)) for case in cases} + dataset_score = _weighted_average((case.score, case_weight_map[case.case_id]) for case in cases) + metric_scores = { + metric_name: + _weighted_average(( + next(metric.score for metric in case.metrics if metric.metric_name == metric_name), + case_weight_map[case.case_id], + ) for case in cases) + for metric_name in metric_names + } + return EvaluationSnapshot( + split=split, + phase=phase, + dataset_score=dataset_score, + pass_rate=sum(case.passed for case in cases) / len(cases), + metric_scores=metric_scores, + case_ids=tuple(expected_ids), + cases=tuple(cases), + ) + + +def compare_snapshots( + baseline: EvaluationSnapshot, + candidate: EvaluationSnapshot, + *, + epsilon: float, + critical_case_ids: Iterable[str] = (), + hard_case_ids: Iterable[str] = (), + baseline_attribution: AttributionSnapshot | None = None, + candidate_attribution: AttributionSnapshot | None = None, +) -> ComparisonSnapshot: + """Compare schema-identical snapshots and classify every case transition.""" + + if baseline.split != candidate.split or baseline.case_ids != candidate.case_ids: + raise ValueError("baseline and candidate case schemas differ") + if set(baseline.metric_scores) != set(candidate.metric_scores): + raise ValueError("baseline and candidate metric schemas differ") + baseline_failures = _attribution_map(baseline_attribution) + candidate_failures = _attribution_map(candidate_attribution) + critical = set(critical_case_ids) + hard = set(hard_case_ids) + comparisons: list[CaseComparison] = [] + for base_case, new_case in zip(baseline.cases, candidate.cases): + if base_case.case_id != new_case.case_id: + raise ValueError("baseline and candidate case ordering differs") + base_metrics = {metric.metric_name: metric for metric in base_case.metrics} + new_metrics = {metric.metric_name: metric for metric in new_case.metrics} + if set(base_metrics) != set(new_metrics): + raise ValueError("baseline and candidate per-case metric schemas differ") + delta = new_case.score - base_case.score + if not base_case.passed and new_case.passed: + transition = Transition.NEW_PASS + elif base_case.passed and not new_case.passed: + transition = Transition.NEW_FAIL + elif delta > epsilon: + transition = Transition.IMPROVED + elif delta < -epsilon: + transition = Transition.REGRESSED + else: + transition = Transition.UNCHANGED + comparisons.append( + CaseComparison( + case_id=base_case.case_id, + baseline_passed=base_case.passed, + candidate_passed=new_case.passed, + baseline_score=base_case.score, + candidate_score=new_case.score, + delta=delta, + metrics=tuple( + MetricDelta( + metric_name=name, + baseline=base_metrics[name].score, + candidate=new_metrics[name].score, + delta=new_metrics[name].score - base_metrics[name].score, + ) for name in base_metrics), + transition=transition, + critical=base_case.case_id in critical, + hard=base_case.case_id in hard, + baseline_attribution=baseline_failures.get(base_case.case_id), + candidate_attribution=candidate_failures.get(base_case.case_id), + new_hard_failure=base_case.case_id in hard and base_case.passed and not new_case.passed, + )) + return ComparisonSnapshot( + split=baseline.split, + score_delta=candidate.dataset_score - baseline.dataset_score, + pass_rate_delta=candidate.pass_rate - baseline.pass_rate, + metric_deltas={ + name: candidate.metric_scores[name] - baseline.metric_scores[name] + for name in baseline.metric_scores + }, + cases=tuple(comparisons), + ) + + +def _attribution_map(snapshot: AttributionSnapshot | None) -> dict[str, FailureAttribution]: + return {} if snapshot is None else {failure.case_id: failure for failure in snapshot.failures} diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py b/examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py new file mode 100644 index 000000000..447a72942 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluation_runtime.py @@ -0,0 +1,159 @@ +"""Side-effecting evaluation stage runner around the pure normalizer.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import Callable, Optional + +from trpc_agent_sdk.evaluation import EvalConfig, EvalSet + +from .artifacts import AuditSink +from .configuration import ValidatedRunConfig +from .contracts import EvaluationBackend +from .costing import CostLedger +from .evaluation import normalize_result +from .models import EvaluationSnapshot, Phase, SnapshotPair, Split + + +def create_evaluation_runtime( + *, + validated: ValidatedRunConfig, + backend: EvaluationBackend, + custom_backend: bool, + sink: AuditSink, + ledger: CostLedger, +) -> "EvaluationRuntime": + """Construct accounting policy once at the composition boundary.""" + + settings = validated.config.pipeline + offline = not custom_backend and settings.mode in {"fake", "trace"} + live_with_bounds = (not custom_backend and settings.mode == "live" + and settings.live_agent_call_max_cost_usd is not None + and settings.live_metric_call_max_cost_usd is not None) + calls_per_invocation = (1 if not custom_backend and settings.mode in {"fake", "live"} else + 0 if not custom_backend and settings.mode == "trace" else None) + return EvaluationRuntime( + backend=backend, + sink=sink, + ledger=ledger, + eval_config=validated.config.evaluate, + agent_call_cost_usd=(0 if offline else settings.live_agent_call_max_cost_usd if live_with_bounds else None), + metric_call_cost_usd=(0 if offline else settings.live_metric_call_max_cost_usd if live_with_bounds else None), + cost_is_upper_bound=live_with_bounds, + model_calls_per_invocation=calls_per_invocation, + metric_weights=settings.metric_weights, + train_case_weights=settings.train_case_weights, + validation_case_weights=settings.validation_case_weights, + ) + + +@dataclass +class EvaluationRuntime: + """Own backend invocation, completed-call accounting and snapshot persistence.""" + + backend: EvaluationBackend + sink: AuditSink + ledger: CostLedger + eval_config: EvalConfig + agent_call_cost_usd: float | None + metric_call_cost_usd: float | None + cost_is_upper_bound: bool + model_calls_per_invocation: int | None + metric_weights: dict[str, float] + train_case_weights: dict[str, float] + validation_case_weights: dict[str, float] + + async def evaluate( + self, + *, + eval_set: EvalSet, + prompts: dict[str, str], + split: Split, + phase: Phase, + ) -> EvaluationSnapshot: + stage = f"{phase.value}_{split.value}" + phase_dir = self.sink.phase_dir(stage) + metric_count = len(self.eval_config.get_eval_metrics()) + invocations = sum(len(case.conversation or case.actual_conversation or []) for case in eval_set.eval_cases) + metric_calls = invocations * self.eval_config.num_runs * metric_count + model_calls = None + if self.model_calls_per_invocation is not None: + model_calls = invocations * self.eval_config.num_runs * self.model_calls_per_invocation + cost_usd = None + if self.agent_call_cost_usd is not None and self.metric_call_cost_usd is not None: + cost_usd = ( + invocations * self.eval_config.num_runs * self.agent_call_cost_usd + + metric_calls * self.metric_call_cost_usd + ) + try: + raw = await self.backend.evaluate( + eval_set=deepcopy(eval_set), + eval_config=deepcopy(self.eval_config), + prompts=deepcopy(prompts), + split=split, + phase=phase, + audit_dir=str(phase_dir), + ) + except BaseException: + self.ledger.record( + stage, + cost_usd=(0 if self.agent_call_cost_usd == self.metric_call_cost_usd == 0 else None), + model_calls=(0 if self.model_calls_per_invocation == 0 else None), + metric_calls=None, + ) + raise + self.ledger.record( + stage, + cost_usd=cost_usd, + upper_bound=self.cost_is_upper_bound, + model_calls=model_calls, + metric_calls=metric_calls, + ) + snapshot = normalize_result( + raw, + eval_set, + self.eval_config, + split=split, + phase=phase, + metric_weights=self.metric_weights, + case_weights=(self.train_case_weights if split == Split.TRAIN else self.validation_case_weights), + ) + self.sink.write_json( + f"{stage}/snapshot.json", + snapshot.model_dump(mode="json", by_alias=True), + ) + return snapshot + + async def evaluate_phase( + self, + *, + train: EvalSet, + validation: EvalSet, + prompts: dict[str, str], + phase: Phase, + on_stage: Optional[Callable[[str], None]] = None, + on_progress: Optional[Callable[[Phase, SnapshotPair], None]] = None, + ) -> SnapshotPair: + """Evaluate train then validation for one phase in the fixed order.""" + + snapshots = {} + for split, eval_set in ( + (Split.TRAIN, train), + (Split.VALIDATION, validation), + ): + if on_stage is not None: + on_stage(f"{phase.value}_{split.value}") + snapshots[split] = await self.evaluate( + eval_set=eval_set, + prompts=prompts, + split=split, + phase=phase, + ) + progress = SnapshotPair( + train=snapshots.get(Split.TRAIN), + validation=snapshots.get(Split.VALIDATION), + ) + if on_progress is not None: + on_progress(phase, progress) + return progress diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/eval_optimize_loop/pipeline/gate.py new file mode 100644 index 000000000..4d134ac0d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/gate.py @@ -0,0 +1,203 @@ +"""Pure deterministic acceptance gate with fixed AND semantics.""" + +from __future__ import annotations + +import math +from typing import Any + +from .configuration import GateConfig +from .models import ( + CandidateProposal, + ComparisonSnapshot, + CostSummary, + Decision, + GateCheck, + GateDecision, +) + + +def _check(code: str, passed: bool, observed: Any, threshold: Any, message: str) -> GateCheck: + return GateCheck(code=code, passed=passed, observed=observed, threshold=threshold, message=message) + + +def _invalid_input(message: str) -> GateDecision: + return GateDecision( + decision=Decision.ERROR, + accepted=False, + checks=(_check("REPORT_COMPLETE", False, "invalid", "complete", message), ), + reasons=("GATE_INPUT_INVALID", ), + ) + + +def evaluate_gate( + *, + train: ComparisonSnapshot, + validation: ComparisonSnapshot, + candidate: CandidateProposal, + cost: CostSummary, + duration_seconds: float, + baseline_prompt_hashes: dict[str, str], + candidate_prompt_hashes: dict[str, str], + config: GateConfig, + report_complete: bool = True, +) -> GateDecision: + """Evaluate all ten checks in their stable order.""" + + try: + if train.split.value != "train" or validation.split.value != "validation": + raise ValueError("comparison splits are misplaced") + if set(baseline_prompt_hashes) != set(candidate_prompt_hashes) or not baseline_prompt_hashes: + raise ValueError("prompt hash schemas are incomplete") + if not math.isfinite(duration_seconds) or duration_seconds < 0: + raise ValueError("duration must be finite and non-negative") + for value in ( + train.score_delta, + train.pass_rate_delta, + validation.score_delta, + validation.pass_rate_delta, + *train.metric_deltas.values(), + *validation.metric_deltas.values(), + ): + if not math.isfinite(value): + raise ValueError("gate inputs must be finite") + missing_metrics = set(config.metric_max_regression) - set(validation.metric_deltas) + if missing_metrics: + raise ValueError(f"configured metrics are missing: {sorted(missing_metrics)}") + except (AttributeError, TypeError, ValueError) as error: + return _invalid_input(str(error)) + + checks: list[GateCheck] = [] + failed_reasons: list[str] = [] + + def add( + code: str, + passed: bool, + observed: Any, + threshold: Any, + message: str, + reason: str, + ) -> None: + checks.append(_check(code, passed, observed, threshold, message)) + if not passed: + failed_reasons.append(reason) + + add( + "REPORT_COMPLETE", + report_complete, + report_complete, + True, + "All required regression snapshots and report inputs are present.", + "REPORT_INCOMPLETE", + ) + hashes_changed = baseline_prompt_hashes != candidate_prompt_hashes + add( + "CANDIDATE_CHANGED", + candidate.changed and hashes_changed, + hashes_changed, + True, + "Candidate prompt hashes differ from baseline.", + "CANDIDATE_UNCHANGED", + ) + score_passed = validation.score_delta + config.epsilon >= config.min_validation_score_delta + add( + "VALIDATION_SCORE_DELTA", + score_passed, + validation.score_delta, + config.min_validation_score_delta, + "Held-out validation score delta meets the configured minimum.", + "VALIDATION_SCORE_DELTA_BELOW_MINIMUM", + ) + pass_rate_passed = validation.pass_rate_delta + config.epsilon >= config.min_validation_pass_rate_delta + add( + "VALIDATION_PASS_RATE_DELTA", + pass_rate_passed, + validation.pass_rate_delta, + config.min_validation_pass_rate_delta, + "Held-out validation pass-rate delta meets the configured minimum.", + "VALIDATION_PASS_RATE_DELTA_BELOW_MINIMUM", + ) + new_hard_failures = sum(case.new_hard_failure for case in validation.cases) + add( + "NO_NEW_HARD_FAIL", + new_hard_failures <= config.max_new_hard_failures, + new_hard_failures, + config.max_new_hard_failures, + "New held-out hard failures stay within budget.", + "NEW_HARD_FAIL_BUDGET_EXCEEDED", + ) + + def is_critical_regression(case) -> bool: + new_failure = case.baseline_passed and not case.candidate_passed + below_delta = case.delta + config.epsilon < config.critical_case_min_delta + return case.critical and (new_failure or below_delta) + + critical_regressions = [case.case_id for case in validation.cases if is_critical_regression(case)] + add( + "CRITICAL_CASE_NON_REGRESSION", + not critical_regressions, + critical_regressions, + config.critical_case_min_delta, + "Critical held-out cases do not regress.", + "CRITICAL_CASE_REGRESSION", + ) + metric_regressions = { + metric: validation.metric_deltas[metric] + for metric, limit in config.metric_max_regression.items() + if validation.metric_deltas[metric] + config.epsilon < -limit + } + add( + "METRIC_NON_REGRESSION", + not metric_regressions, + metric_regressions, + config.metric_max_regression, + "Configured validation metrics stay within regression limits.", + "METRIC_REGRESSION", + ) + if config.max_cost_usd is None: + cost_passed, cost_observed, cost_reason = True, cost.total_cost_usd, "COST_BUDGET_EXCEEDED" + elif cost.total_cost_usd is None or any(source.cost_usd is None for source in cost.sources): + cost_passed, cost_observed, cost_reason = False, None, "COST_UNAVAILABLE" + else: + cost_passed = cost.total_cost_usd <= config.max_cost_usd + config.epsilon + cost_observed, cost_reason = cost.total_cost_usd, "COST_BUDGET_EXCEEDED" + add( + "COST_BUDGET", + cost_passed, + cost_observed, + config.max_cost_usd, + "Known enabled costs stay within budget.", + cost_reason, + ) + duration_passed = (config.max_duration_seconds is None + or duration_seconds <= config.max_duration_seconds + config.epsilon) + add( + "DURATION_BUDGET", + duration_passed, + duration_seconds, + config.max_duration_seconds, + "Gate-observed duration stays within budget.", + "DURATION_BUDGET_EXCEEDED", + ) + validation_regressed = (validation.score_delta < -config.epsilon or validation.pass_rate_delta < -config.epsilon) + train_improved = train.score_delta > config.epsilon or train.pass_rate_delta > config.epsilon + overfit = train_improved and validation_regressed + add( + "OVERFIT_GUARD", + not overfit, + { + "trainScoreDelta": train.score_delta, + "trainPassRateDelta": train.pass_rate_delta, + "validationScoreDelta": validation.score_delta, + "validationPassRateDelta": validation.pass_rate_delta, + }, + "train improvement cannot accompany held-out validation regression", + "Training improvement with held-out regression is rejected as overfitting.", + "OVERFIT_TRAIN_UP_VALIDATION_DOWN", + ) + accepted = all(check.passed for check in checks) + return GateDecision( + decision=Decision.ACCEPT if accepted else Decision.REJECT, + accepted=accepted, + checks=tuple(checks), + reasons=tuple(dict.fromkeys(failed_reasons)), + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/live_adapter.py b/examples/optimization/eval_optimize_loop/pipeline/live_adapter.py new file mode 100644 index 000000000..b7c92ec6d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/live_adapter.py @@ -0,0 +1,88 @@ +"""Single importable callback contract for live evaluation and optimization.""" + +from __future__ import annotations + +import importlib +import hashlib +import inspect +import marshal +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +def load_callback(spec: str) -> Any: + """Load one MODULE:FUNCTION callback and reject ambiguous specifications.""" + + if spec.count(":") != 1: + raise ValueError("live callback must use MODULE:FUNCTION syntax") + module_name, function_name = spec.split(":", 1) + if not module_name or not function_name or "." in function_name: + raise ValueError("live callback must use MODULE:FUNCTION syntax") + callback = getattr(importlib.import_module(module_name), function_name) + if not inspect.iscoroutinefunction(callback): + raise TypeError("live callback must be an async function") + return callback + + +def _source_identity(callback: Any) -> tuple[Path, str, str]: + unwrapped = inspect.unwrap(callback) + code = getattr(unwrapped, "__code__", None) + if code is None: + raise TypeError("live callback must expose Python function code") + source = inspect.getsourcefile(unwrapped) or inspect.getfile(unwrapped) + if not source: + raise ValueError("live callback source is unavailable") + source_path = Path(source).resolve() + if not source_path.is_file(): + raise ValueError("live callback source is not a file") + return ( + source_path, + hashlib.sha256(source_path.read_bytes()).hexdigest(), + hashlib.sha256(marshal.dumps(code)).hexdigest(), + ) + + +def load_verified_callback( + spec: str, + *, + expected_source_path: str, + expected_source_sha256: str, + expected_callable_sha256: str, +) -> Any: + """Load a worker callback and prove it is the preflight-resolved source.""" + + callback = load_callback(spec) + source_path, source_sha256, callable_sha256 = _source_identity(callback) + if source_path != Path(expected_source_path).resolve(): + raise ValueError("worker callback source path differs from preflight") + if source_sha256 != expected_source_sha256: + raise ValueError("worker callback source hash differs from preflight") + if callable_sha256 != expected_callable_sha256: + raise ValueError("worker callback code differs from preflight") + return callback + + +@dataclass(frozen=True) +class LiveAdapterSpec: + """Bind the parent callable, worker import path and tracked source as one fact.""" + + import_path: str + callback: Any + source_path: Path + source_sha256: str + callable_sha256: str + + @classmethod + def resolve(cls, import_path: str, callback: Any) -> "LiveAdapterSpec": + loaded = load_callback(import_path) + if inspect.unwrap(loaded) is not inspect.unwrap(callback): + raise ValueError("call_agent and callback_spec must resolve to the same function") + source_path, source_sha256, callable_sha256 = _source_identity(loaded) + return cls( + import_path=import_path, + callback=loaded, + source_path=source_path, + source_sha256=source_sha256, + callable_sha256=callable_sha256, + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py new file mode 100644 index 000000000..7eddc1931 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -0,0 +1,725 @@ +"""Strict, immutable stage contracts for the evaluation/optimization loop.""" + +from __future__ import annotations + +import math +from enum import Enum +from typing import Any, Literal, Optional + +from pydantic import Field, field_validator, model_validator + +from .schema import ( + StrictModel, + finite_number as _finite, + non_negative_number as _non_negative, +) + + +class Split(str, Enum): + TRAIN = "train" + VALIDATION = "validation" + + +class Phase(str, Enum): + BASELINE = "baseline" + CANDIDATE = "candidate" + + +class Transition(str, Enum): + NEW_PASS = "NEW_PASS" + NEW_FAIL = "NEW_FAIL" + IMPROVED = "IMPROVED" + REGRESSED = "REGRESSED" + UNCHANGED = "UNCHANGED" + + +class Decision(str, Enum): + ACCEPT = "ACCEPT" + REJECT = "REJECT" + ERROR = "ERROR" + + +class FailureCategory(str, Enum): + EVALUATION_ERROR = "EVALUATION_ERROR" + TOOL_CALL_ERROR = "TOOL_CALL_ERROR" + TOOL_ARGUMENT_ERROR = "TOOL_ARGUMENT_ERROR" + KNOWLEDGE_RECALL_INSUFFICIENT = "KNOWLEDGE_RECALL_INSUFFICIENT" + FORMAT_VIOLATION = "FORMAT_VIOLATION" + LLM_RUBRIC_NOT_MET = "LLM_RUBRIC_NOT_MET" + FINAL_RESPONSE_MISMATCH = "FINAL_RESPONSE_MISMATCH" + UNKNOWN = "UNKNOWN" + + +Confidence = Literal["low", "medium", "high"] +RunMode = Literal["fake", "trace", "live"] + + +class RubricOutcome(StrictModel): + id: str + score: float + passed: bool + reason: Optional[str] = None + + @field_validator("id") + @classmethod + def valid_id(cls, value: str) -> str: + if not value or value != value.strip(): + raise ValueError("rubric outcome id must be non-empty and trimmed") + return value + + @field_validator("score") + @classmethod + def finite_score(cls, value: float) -> float: + return _finite(value, "rubric score") + + +class MetricRun(StrictModel): + metric_name: str + score: float + threshold: float + passed: bool + reason: Optional[str] = None + rubrics: tuple[RubricOutcome, ...] = () + + @field_validator("score", "threshold") + @classmethod + def finite_score(cls, value: float) -> float: + return _finite(value, "metric value") + + @model_validator(mode="after") + def consistent_status(self) -> "MetricRun": + if self.passed != (self.score >= self.threshold): + raise ValueError("metric passed status does not match score >= threshold") + for rubric in self.rubrics: + if rubric.passed != (rubric.score >= self.threshold): + raise ValueError(f"rubric {rubric.id!r} passed status does not match metric threshold") + return self + + +class CaseRun(StrictModel): + run_id: int = Field(ge=1) + passed: bool + metrics: tuple[MetricRun, ...] + error: Optional[str] = None + trace: tuple[dict[str, Any], ...] = () + + @model_validator(mode="after") + def consistent_status(self) -> "CaseRun": + if not self.metrics: + raise ValueError("a case run must contain metrics") + expected = self.error is None and all(metric.passed for metric in self.metrics) + if self.passed != expected: + raise ValueError("case run passed status does not match metric statuses") + return self + + +class MetricSnapshot(StrictModel): + metric_name: str + score: float + threshold: float + passed: bool + + @field_validator("score", "threshold") + @classmethod + def finite_score(cls, value: float) -> float: + return _finite(value, "metric aggregate") + + +class CaseSnapshot(StrictModel): + case_id: str + passed: bool + score: float + metrics: tuple[MetricSnapshot, ...] + runs: tuple[CaseRun, ...] + error: Optional[str] = None + + @field_validator("score") + @classmethod + def finite_score(cls, value: float) -> float: + return _finite(value, "case score") + + @model_validator(mode="after") + def consistent_aggregate(self) -> "CaseSnapshot": + if not self.metrics or not self.runs: + raise ValueError("case snapshot requires metrics and runs") + metric_names = tuple(metric.metric_name for metric in self.metrics) + if len(set(metric_names)) != len(metric_names): + raise ValueError("case snapshot metric names must be unique") + if tuple(run.run_id for run in self.runs) != tuple(range(1, len(self.runs) + 1)): + raise ValueError("case snapshot run IDs must be contiguous and 1-based") + for run in self.runs: + if tuple(metric.metric_name for metric in run.metrics) != metric_names: + raise ValueError("case snapshot run metric schemas must match the aggregate") + if self.passed != all(run.passed for run in self.runs): + raise ValueError("case snapshot passed status does not match run statuses") + expected_error = next((run.error for run in self.runs if run.error), None) + if self.error != expected_error: + raise ValueError("case snapshot error does not match the first run error") + scores = [metric.score for metric in self.metrics] + if self.score < min(scores) - 1e-12 or self.score > max(scores) + 1e-12: + raise ValueError("case score must be a positive-weight aggregate of metric scores") + return self + + +class EvaluationSnapshot(StrictModel): + split: Split + phase: Phase + dataset_score: float + pass_rate: float + metric_scores: dict[str, float] + case_ids: tuple[str, ...] + cases: tuple[CaseSnapshot, ...] + + @field_validator("dataset_score", "pass_rate") + @classmethod + def finite_aggregate(cls, value: float) -> float: + return _finite(value, "evaluation aggregate") + + @field_validator("metric_scores") + @classmethod + def finite_metric_scores(cls, value: dict[str, float]) -> dict[str, float]: + for name, score in value.items(): + _finite(score, f"metric_scores[{name!r}]") + return value + + @model_validator(mode="after") + def complete_cases(self) -> "EvaluationSnapshot": + if not self.cases: + raise ValueError("evaluation snapshot requires at least one case") + if self.case_ids != tuple(case.case_id for case in self.cases): + raise ValueError("case_ids must match cases in order") + if len(set(self.case_ids)) != len(self.case_ids): + raise ValueError("evaluation snapshot case IDs must be unique") + metric_names = set(self.metric_scores) + if not metric_names or any(not name or name != name.strip() for name in metric_names): + raise ValueError("evaluation snapshot metric names must be non-empty and trimmed") + for case in self.cases: + if {metric.metric_name for metric in case.metrics} != metric_names: + raise ValueError("evaluation snapshot case metric schemas must match") + expected_pass_rate = sum(case.passed for case in self.cases) / len(self.cases) + if not math.isclose(self.pass_rate, expected_pass_rate, rel_tol=0, abs_tol=1e-12): + raise ValueError("pass_rate does not match case statuses") + case_scores = [case.score for case in self.cases] + if self.dataset_score < min(case_scores) - 1e-12 or self.dataset_score > max(case_scores) + 1e-12: + raise ValueError("dataset score must be a positive-weight aggregate of case scores") + for name, aggregate in self.metric_scores.items(): + scores = [ + next(metric.score for metric in case.metrics if metric.metric_name == name) for case in self.cases + ] + if aggregate < min(scores) - 1e-12 or aggregate > max(scores) + 1e-12: + raise ValueError(f"metric score {name!r} must be a positive-weight aggregate of case metrics") + return self + + +class FailureAttribution(StrictModel): + case_id: str + primary: FailureCategory + secondary: tuple[FailureCategory, ...] = () + reasons: tuple[str, ...] + trigger_metrics: tuple[str, ...] = () + trigger_rubrics: tuple[str, ...] = () + evidence: tuple[str, ...] + confidence: Confidence + + @model_validator(mode="after") + def explanatory(self) -> "FailureAttribution": + if not self.reasons or not self.evidence: + raise ValueError("failure attribution needs reasons and evidence") + if self.primary in self.secondary: + raise ValueError("primary category cannot also be secondary") + return self + + +class AttributionStatistics(StrictModel): + total_failures: int = Field(default=0, ge=0) + primary_category_counts: dict[FailureCategory, int] = Field(default_factory=dict) + + @model_validator(mode="after") + def consistent_totals(self) -> "AttributionStatistics": + if any(count < 1 for count in self.primary_category_counts.values()): + raise ValueError("attribution category counts must be positive") + if self.total_failures != sum(self.primary_category_counts.values()): + raise ValueError("attribution total must equal primary category counts") + return self + + +class AttributionSnapshot(StrictModel): + split: Split + phase: Phase + failures: tuple[FailureAttribution, ...] + statistics: AttributionStatistics = Field(default_factory=AttributionStatistics) + + @model_validator(mode="after") + def statistics_match_failures(self) -> "AttributionSnapshot": + expected = { + category: sum(failure.primary == category for failure in self.failures) + for category in FailureCategory + } + expected = {category: count for category, count in expected.items() if count} + if self.statistics.total_failures != len(self.failures): + raise ValueError("attribution statistics total does not match failures") + if self.statistics.primary_category_counts != expected: + raise ValueError("attribution category statistics do not match failures") + return self + + +class InnerSplit(StrictModel): + train_case_ids: tuple[str, ...] + selection_case_ids: tuple[str, ...] + train_hash: str + selection_hash: str + audit_hashes: dict[str, str] = Field(default_factory=dict) + train_path: str + selection_path: str + + +class CostSource(StrictModel): + name: str + cost_usd: Optional[float] + upper_bound: bool = False + model_calls: Optional[int] = Field(default=None, ge=0) + metric_calls: Optional[int] = Field(default=None, ge=0) + token_usage: dict[str, int] = Field(default_factory=dict) + + @field_validator("name") + @classmethod + def valid_name(cls, value: str) -> str: + if not value or value != value.strip(): + raise ValueError("cost source name must be non-empty and trimmed") + return value + + @field_validator("cost_usd") + @classmethod + def valid_cost(cls, value: Optional[float]) -> Optional[float]: + return None if value is None else _non_negative(value, "cost") + + @model_validator(mode="after") + def valid_upper_bound(self) -> "CostSource": + if self.upper_bound and self.cost_usd is None: + raise ValueError("unknown cost cannot be marked as an upper bound") + return self + + @field_validator("token_usage") + @classmethod + def valid_tokens(cls, value: dict[str, int]) -> dict[str, int]: + if any(amount < 0 for amount in value.values()): + raise ValueError("token usage must be non-negative") + return value + + +def _unreported_candidate_cost() -> tuple[CostSource, ...]: + return (CostSource(name="unreported", cost_usd=None), ) + + +class CandidateRound(StrictModel): + round: int = Field(ge=1) + candidate_prompts: dict[str, str] + accepted: bool + score: Optional[float] = None + kind: Literal["deterministic", "reflective", "merge"] = "reflective" + optimized_fields: tuple[str, ...] = () + metric_scores: dict[str, float] = Field(default_factory=dict) + acceptance_reason: Optional[str] = None + skip_reason: Optional[str] = None + error_message: Optional[str] = None + cost_usd: Optional[float] = None + token_usage: dict[str, int] = Field(default_factory=dict) + duration_seconds: float = 0.0 + + @field_validator("score") + @classmethod + def valid_score(cls, value: Optional[float]) -> Optional[float]: + if value is None: + return None + value = _finite(value, "round score") + if not 0 <= value <= 1: + raise ValueError("round score must be in [0, 1]") + return value + + @field_validator("optimized_fields") + @classmethod + def valid_optimized_fields(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if len(set(value)) != len(value) or any(not name or name != name.strip() for name in value): + raise ValueError("optimized fields must be non-empty and unique") + return value + + @field_validator("metric_scores") + @classmethod + def valid_metric_scores(cls, value: dict[str, float]) -> dict[str, float]: + for name, score in value.items(): + if not name or name != name.strip(): + raise ValueError("round metric names must be non-empty and trimmed") + _finite(score, f"round metric score {name!r}") + return value + + @field_validator("cost_usd") + @classmethod + def valid_cost(cls, value: Optional[float]) -> Optional[float]: + return None if value is None else _non_negative(value, "round cost") + + @field_validator("token_usage") + @classmethod + def valid_tokens(cls, value: dict[str, int]) -> dict[str, int]: + if any(amount < 0 for amount in value.values()): + raise ValueError("round token usage must be non-negative") + return value + + @field_validator("duration_seconds") + @classmethod + def valid_duration(cls, value: float) -> float: + return _non_negative(value, "round duration") + + @model_validator(mode="after") + def consistent_outcome(self) -> "CandidateRound": + if self.skip_reason and self.error_message: + raise ValueError("round cannot be both skipped and errored") + if self.accepted and (not self.candidate_prompts or self.skip_reason or self.error_message): + raise ValueError("accepted round must contain a normal candidate") + if self.candidate_prompts and not (self.skip_reason or self.error_message) and self.score is None: + raise ValueError("candidate round requires a validation score") + if not self.candidate_prompts and not (self.skip_reason or self.error_message): + raise ValueError("empty candidate round requires a skip or error reason") + if not self.candidate_prompts and self.optimized_fields: + raise ValueError("empty candidate round cannot report optimized fields") + return self + + +class CandidateProposal(StrictModel): + status: Literal["SUCCEEDED", "FAILED", "CANCELED"] = "SUCCEEDED" + error_message: Optional[str] = None + adapter_error: Optional[str] = None + algorithm: str + baseline_prompts: dict[str, str] + prompts: dict[str, str] + changed: bool + stop_reason: Optional[str] = None + rounds: tuple[CandidateRound, ...] = () + cost_sources: tuple[CostSource, ...] = Field(default_factory=_unreported_candidate_cost) + duration_seconds: float = 0.0 + + @field_validator("duration_seconds") + @classmethod + def valid_duration(cls, value: float) -> float: + return _non_negative(value, "candidate duration") + + @model_validator(mode="after") + def valid_round_history(self) -> "CandidateProposal": + if self.status == "SUCCEEDED" and self.error_message: + raise ValueError("successful candidate generation cannot contain an error") + if self.adapter_error and self.status != "SUCCEEDED": + raise ValueError("adapter errors only describe rejected successful backend results") + if self.status == "FAILED" and not self.error_message: + raise ValueError("failed candidate generation requires an error message") + if set(self.baseline_prompts) != set(self.prompts): + raise ValueError("candidate prompt keys must equal baseline keys") + if self.changed != (self.prompts != self.baseline_prompts): + raise ValueError("candidate changed flag is inconsistent") + if tuple(round_.round for round_ in self.rounds) != tuple(range(1, len(self.rounds) + 1)): + raise ValueError("candidate rounds must be contiguous and 1-based") + source_names = tuple(source.name for source in self.cost_sources) + if not source_names or len(set(source_names)) != len(source_names): + raise ValueError("candidate cost source names must be non-empty and unique") + registered = set(self.prompts) + for round_ in self.rounds: + if round_.candidate_prompts and set(round_.candidate_prompts) != registered: + raise ValueError("candidate round prompt keys must equal registered prompt keys") + if not set(round_.optimized_fields) <= registered: + raise ValueError("round optimized fields must reference registered prompt keys") + final_was_accepted = any(round_.accepted and round_.candidate_prompts == self.prompts for round_ in self.rounds) + if self.changed and not final_was_accepted: + raise ValueError("changed final candidate must equal an accepted round") + return self + + +class MetricDelta(StrictModel): + metric_name: str + baseline: float + candidate: float + delta: float + + @field_validator("baseline", "candidate", "delta") + @classmethod + def finite_value(cls, value: float) -> float: + return _finite(value, "metric delta") + + @model_validator(mode="after") + def consistent_delta(self) -> "MetricDelta": + if not math.isclose( + self.delta, + self.candidate - self.baseline, + rel_tol=0, + abs_tol=1e-12, + ): + raise ValueError("metric delta does not match candidate minus baseline") + return self + + +class CaseComparison(StrictModel): + case_id: str + baseline_passed: bool + candidate_passed: bool + baseline_score: float + candidate_score: float + delta: float + metrics: tuple[MetricDelta, ...] + transition: Transition + critical: bool = False + hard: bool = False + baseline_attribution: Optional[FailureAttribution] = None + candidate_attribution: Optional[FailureAttribution] = None + new_hard_failure: bool = False + + @field_validator("baseline_score", "candidate_score", "delta") + @classmethod + def finite_score(cls, value: float) -> float: + return _finite(value, "case comparison score") + + @model_validator(mode="after") + def consistent_transition(self) -> "CaseComparison": + if not math.isclose( + self.delta, + self.candidate_score - self.baseline_score, + rel_tol=0, + abs_tol=1e-12, + ): + raise ValueError("case delta does not match candidate minus baseline") + metric_names = tuple(metric.metric_name for metric in self.metrics) + if len(set(metric_names)) != len(metric_names): + raise ValueError("case comparison metric names must be unique") + expected_new_pass = not self.baseline_passed and self.candidate_passed + expected_new_fail = self.baseline_passed and not self.candidate_passed + if expected_new_pass != (self.transition == Transition.NEW_PASS): + raise ValueError("NEW_PASS transition does not match case statuses") + if expected_new_fail != (self.transition == Transition.NEW_FAIL): + raise ValueError("NEW_FAIL transition does not match case statuses") + expected_hard_failure = self.hard and expected_new_fail + if self.new_hard_failure != expected_hard_failure: + raise ValueError("new hard failure flag does not match case statuses") + for attribution in (self.baseline_attribution, self.candidate_attribution): + if attribution is not None and attribution.case_id != self.case_id: + raise ValueError("case comparison attribution ID does not match the case") + return self + + +class ComparisonSnapshot(StrictModel): + split: Split + score_delta: float + pass_rate_delta: float + metric_deltas: dict[str, float] + cases: tuple[CaseComparison, ...] + + @field_validator("score_delta", "pass_rate_delta") + @classmethod + def finite_value(cls, value: float) -> float: + return _finite(value, "comparison delta") + + @field_validator("metric_deltas") + @classmethod + def finite_metric_deltas(cls, value: dict[str, float]) -> dict[str, float]: + for name, delta in value.items(): + if not name or name != name.strip(): + raise ValueError("comparison metric names must be non-empty and trimmed") + _finite(delta, f"comparison metric delta {name!r}") + return value + + @model_validator(mode="after") + def consistent_aggregate(self) -> "ComparisonSnapshot": + if not self.cases: + raise ValueError("comparison snapshot requires at least one case") + case_ids = tuple(case.case_id for case in self.cases) + if len(set(case_ids)) != len(case_ids): + raise ValueError("comparison case IDs must be unique") + metric_names = set(self.metric_deltas) + if not metric_names: + raise ValueError("comparison snapshot requires metric deltas") + for case in self.cases: + if {metric.metric_name for metric in case.metrics} != metric_names: + raise ValueError("comparison case metric schemas must match aggregate metrics") + expected_pass_delta = (sum(case.candidate_passed + for case in self.cases) - sum(case.baseline_passed + for case in self.cases)) / len(self.cases) + if not math.isclose( + self.pass_rate_delta, + expected_pass_delta, + rel_tol=0, + abs_tol=1e-12, + ): + raise ValueError("pass-rate delta does not match case statuses") + case_deltas = [case.delta for case in self.cases] + if self.score_delta < min(case_deltas) - 1e-12 or self.score_delta > max(case_deltas) + 1e-12: + raise ValueError("score delta must be a positive-weight aggregate of case deltas") + for name, aggregate in self.metric_deltas.items(): + deltas = [ + next(metric.delta for metric in case.metrics if metric.metric_name == name) for case in self.cases + ] + if aggregate < min(deltas) - 1e-12 or aggregate > max(deltas) + 1e-12: + raise ValueError(f"metric delta {name!r} must be a positive-weight aggregate of case metrics") + return self + + +class CostSummary(StrictModel): + sources: tuple[CostSource, ...] + total_cost_usd: Optional[float] + + @field_validator("total_cost_usd") + @classmethod + def valid_total(cls, value: Optional[float]) -> Optional[float]: + return None if value is None else _non_negative(value, "total cost") + + @model_validator(mode="after") + def consistent_total(self) -> "CostSummary": + names = tuple(source.name for source in self.sources) + if len(set(names)) != len(names): + raise ValueError("cost source names must be unique") + has_unknown = any(source.cost_usd is None for source in self.sources) + if has_unknown and self.total_cost_usd is not None: + raise ValueError("total cost must be unknown when any source cost is unknown") + if not has_unknown and self.total_cost_usd is None: + raise ValueError("total cost must be reported when all source costs are known") + if not has_unknown and not math.isclose( + self.total_cost_usd or 0, + sum(source.cost_usd or 0 for source in self.sources), + rel_tol=0, + abs_tol=1e-12, + ): + raise ValueError("total cost does not equal the sum of cost sources") + return self + + +class GateCheck(StrictModel): + code: str + passed: bool + observed: Any + threshold: Any + message: str + + +class GateDecision(StrictModel): + decision: Decision + accepted: bool + checks: tuple[GateCheck, ...] + reasons: tuple[str, ...] + + @model_validator(mode="after") + def consistent_decision(self) -> "GateDecision": + if self.accepted != (self.decision == Decision.ACCEPT): + raise ValueError("accepted flag must match ACCEPT decision") + return self + + +class SnapshotPair(StrictModel): + train: Optional[EvaluationSnapshot] = None + validation: Optional[EvaluationSnapshot] = None + + @model_validator(mode="after") + def has_completed_snapshot(self) -> "SnapshotPair": + if self.train is None and self.validation is None: + raise ValueError("snapshot pair requires at least one completed split") + if self.train is not None and self.train.split != Split.TRAIN: + raise ValueError("train snapshot must use the train split") + if self.validation is not None and self.validation.split != Split.VALIDATION: + raise ValueError("validation snapshot must use the validation split") + return self + + +class ComparisonPair(StrictModel): + train: Optional[ComparisonSnapshot] = None + validation: Optional[ComparisonSnapshot] = None + + @model_validator(mode="after") + def has_completed_comparison(self) -> "ComparisonPair": + if self.train is None and self.validation is None: + raise ValueError("comparison pair requires at least one completed split") + if self.train is not None and self.train.split != Split.TRAIN: + raise ValueError("train comparison must use the train split") + if self.validation is not None and self.validation.split != Split.VALIDATION: + raise ValueError("validation comparison must use the validation split") + return self + + +class AttributionPair(StrictModel): + train: Optional[AttributionSnapshot] = None + validation: Optional[AttributionSnapshot] = None + + @model_validator(mode="after") + def has_completed_attribution(self) -> "AttributionPair": + if self.train is None and self.validation is None: + raise ValueError("attribution pair requires at least one completed split") + if self.train is not None and self.train.split != Split.TRAIN: + raise ValueError("train attribution must use the train split") + if self.validation is not None and self.validation.split != Split.VALIDATION: + raise ValueError("validation attribution must use the validation split") + return self + + +class Reproducibility(StrictModel): + reproducible: bool + command: Optional[str] + reason: Optional[str] + git_commit: Optional[str] + git_dirty: Optional[bool] + + +class SourceApplication(StrictModel): + requested: bool + applied: bool + baseline_hashes: dict[str, str] + final_hashes: dict[str, str] + + +class ArtifactRecord(StrictModel): + path: str + sha256: str + byte_size: int = Field(ge=0) + + +class RunError(StrictModel): + stage: str + error_type: str + message: str + + +class OptimizationReport(StrictModel): + schema_version: Literal["v2"] = "v2" + run_id: str + status: Decision + mode: RunMode + stage: str + started_at: str + finished_at: str + duration_seconds: float + reproducibility: Reproducibility + inputs: dict[str, Any] + prompts: dict[str, Any] + baseline: Optional[SnapshotPair] = None + candidate: Optional[SnapshotPair] = None + delta: Optional[ComparisonPair] = None + failure_attribution: Optional[AttributionPair] = None + candidate_failure_attribution: Optional[AttributionPair] = None + optimization: Optional[dict[str, Any]] = None + cost: Optional[CostSummary] = None + gate_decision: Optional[GateDecision] = None + source_application: SourceApplication + artifacts: tuple[ArtifactRecord, ...] = () + errors: tuple[RunError, ...] = () + + @field_validator("duration_seconds") + @classmethod + def valid_duration(cls, value: float) -> float: + return _non_negative(value, "duration") + + @model_validator(mode="after") + def terminal_consistency(self) -> "OptimizationReport": + if self.status in (Decision.ACCEPT, Decision.REJECT): + if self.gate_decision is None or self.gate_decision.decision != self.status: + raise ValueError("terminal report status must match gate decision") + for pair, name in ( + (self.baseline, "baseline"), + (self.candidate, "candidate"), + (self.delta, "delta"), + ): + if pair is None or pair.train is None or pair.validation is None: + raise ValueError(f"terminal report requires complete {name} results") + if self.status == Decision.ERROR and not self.errors: + raise ValueError("ERROR report must include at least one error") + return self diff --git a/examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py b/examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py new file mode 100644 index 000000000..d887a577e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/offline_evaluation.py @@ -0,0 +1,222 @@ +"""Deterministic replacements for LLM judge metrics in offline modes.""" + +from __future__ import annotations + +import json +import statistics +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import ClassVar, Optional + +from trpc_agent_sdk.evaluation import ( + EvalConfig, + EvalMetric, + EvalStatus, + EvaluationResult, + Evaluator, + EvaluatorRegistry, + FinalResponseEvaluator, + Invocation, + LLM_METRIC_NAMES, + PerInvocationResult, + RubricScore, + get_all_tool_responses, + get_llm_criterion_from_metric, +) + +_RESPONSE_RULES = frozenset({ + "OFFLINE_RESPONSE_CONTAINS", + "OFFLINE_RESPONSE_EQUALS", + "OFFLINE_RESPONSE_EXACT_REFERENCE", + "OFFLINE_RESPONSE_NON_EMPTY", +}) +_KNOWLEDGE_RULES = frozenset({ + "OFFLINE_KNOWLEDGE_CONTAINS", + "OFFLINE_KNOWLEDGE_NON_EMPTY", +}) + + +@dataclass(frozen=True) +class _OfflineRubric: + id: str + rule: str + operand: str + + +def _response_text(invocation: Optional[Invocation]) -> str: + response = invocation.final_response if invocation is not None else None + if response is None: + return "" + return "".join(part.text or "" for part in response.parts or [] if getattr(part, "text", None) is not None) + + +def _parse_rubrics(eval_metric: EvalMetric, supported: frozenset[str]) -> tuple[_OfflineRubric, ...]: + criterion = get_llm_criterion_from_metric(eval_metric) + if criterion is None or not criterion.rubrics: + raise ValueError(f"{eval_metric.metric_name} requires criterion.llmJudge.rubrics") + parsed = [] + ids = set() + for rubric in criterion.rubrics: + rubric_id = rubric.id + if not rubric_id or rubric_id != rubric_id.strip() or rubric_id in ids: + raise ValueError("offline rubric ids must be non-empty, trimmed, and unique") + ids.add(rubric_id) + rule = rubric.type.strip().upper() + if rule not in supported: + allowed = ", ".join(sorted(supported)) + raise ValueError(f"offline rubric {rubric_id!r} requires an explicit type; " + f"supported types: {allowed}") + content = rubric.content + operand = content.text if content is not None else "" + if rule.endswith("_EQUALS") and content is None: + raise ValueError(f"offline rubric {rubric_id!r} requires content.text") + if rule.endswith("_CONTAINS") and not operand: + raise ValueError(f"offline rubric {rubric_id!r} requires non-empty content.text") + parsed.append(_OfflineRubric(id=rubric_id, rule=rule, operand=operand)) + return tuple(parsed) + + +def _evidence_fragments(value: object) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + return (value, ) if value.strip() else () + if isinstance(value, Mapping): + return tuple(fragment for key in sorted(value, key=str) for fragment in _evidence_fragments(value[key])) + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + return tuple(fragment for item in value for fragment in _evidence_fragments(item)) + return (json.dumps(value, ensure_ascii=False, sort_keys=True, allow_nan=False), ) + + +class _OfflineRubricEvaluator(Evaluator): + requires_reference = False + supported_rules: ClassVar[frozenset[str]] = frozenset() + + def __init__(self, eval_metric: Optional[EvalMetric] = None) -> None: + if eval_metric is None: + raise ValueError("eval_metric is required for an offline rubric evaluator") + criterion = get_llm_criterion_from_metric(eval_metric) + if criterion is None: + raise ValueError(f"{eval_metric.metric_name} requires criterion.llmJudge") + self._threshold = eval_metric.threshold + self._rubrics = _parse_rubrics(eval_metric, self.supported_rules) + self._knowledge_tool_names = frozenset(criterion.get_knowledge_tool_names()) + + def _score_rubric( + self, + rubric: _OfflineRubric, + actual: Invocation, + expected: Optional[Invocation], + ) -> tuple[float, str]: + raise NotImplementedError + + def evaluate_invocations( + self, + actual_invocations: list[Invocation], + expected_invocations: Optional[list[Invocation]], + ) -> EvaluationResult: + per_invocation = [] + invocation_scores = [] + for index, actual in enumerate(actual_invocations): + expected = (expected_invocations[index] + if expected_invocations is not None and index < len(expected_invocations) else None) + results = [(rubric, *self._score_rubric(rubric, actual, expected)) for rubric in self._rubrics] + score = statistics.fmean(result[1] for result in results) + reason = "; ".join(result[2] for result in results) + invocation_scores.append(score) + per_invocation.append( + PerInvocationResult( + actual_invocation=actual, + expected_invocation=expected, + score=score, + eval_status=(EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED), + reason=reason, + rubric_scores=[ + RubricScore(id=rubric.id, score=item_score, reason=item_reason) + for rubric, item_score, item_reason in results + ], + )) + if not per_invocation: + return EvaluationResult( + overall_score=0.0, + overall_eval_status=EvalStatus.FAILED, + per_invocation_results=[], + ) + overall_score = statistics.fmean(invocation_scores) + return EvaluationResult( + overall_score=overall_score, + overall_eval_status=(EvalStatus.PASSED if overall_score >= self._threshold else EvalStatus.FAILED), + per_invocation_results=per_invocation, + ) + + +class _OfflineResponseEvaluator(_OfflineRubricEvaluator): + supported_rules = _RESPONSE_RULES + + def _score_rubric( + self, + rubric: _OfflineRubric, + actual: Invocation, + expected: Optional[Invocation], + ) -> tuple[float, str]: + actual_text = _response_text(actual) + if rubric.rule == "OFFLINE_RESPONSE_EXACT_REFERENCE": + if expected is None or expected.final_response is None: + raise ValueError(f"offline rubric {rubric.id!r} requires a reference final response") + matched = actual_text == _response_text(expected) + elif rubric.rule == "OFFLINE_RESPONSE_EQUALS": + matched = actual_text == rubric.operand + elif rubric.rule == "OFFLINE_RESPONSE_CONTAINS": + matched = rubric.operand in actual_text + else: + matched = bool(actual_text.strip()) + return float(matched), f"{rubric.rule} {'matched' if matched else 'did not match'}" + + +class _OfflineKnowledgeEvaluator(_OfflineRubricEvaluator): + supports_remote = False + supported_rules = _KNOWLEDGE_RULES + + def _score_rubric( + self, + rubric: _OfflineRubric, + actual: Invocation, + expected: Optional[Invocation], + ) -> tuple[float, str]: + del expected + fragments = tuple(fragment for response in get_all_tool_responses(actual.intermediate_data) + if response.name in self._knowledge_tool_names + for fragment in _evidence_fragments(response.response)) + matched = (rubric.operand in "\n".join(fragments) + if rubric.rule == "OFFLINE_KNOWLEDGE_CONTAINS" else bool(fragments)) + return float(matched), f"{rubric.rule} {'matched' if matched else 'did not match'}" + + +def prepare_offline_evaluation(eval_config: EvalConfig, ) -> tuple[EvalConfig, Optional[EvaluatorRegistry]]: + """Build a run-local deterministic config and registry for offline modes.""" + + configured = eval_config.get_eval_metrics() + llm_names = {metric.metric_name for metric in configured if metric.metric_name in LLM_METRIC_NAMES} + if not llm_names: + return eval_config, None + metrics = [] + for metric in configured: + payload = metric.model_dump(mode="json", by_alias=True) + if metric.metric_name == "llm_final_response": + payload["criterion"] = {"finalResponse": {"text": {"match": "exact"}}} + metrics.append(payload) + payload = eval_config.model_dump(mode="python", by_alias=False) + payload.update({"criteria": {}, "metrics": metrics}) + offline_config = EvalConfig.model_validate(payload) + registry = EvaluatorRegistry() + replacements = { + "llm_final_response": FinalResponseEvaluator, + "llm_rubric_response": _OfflineResponseEvaluator, + "llm_rubric_knowledge_recall": _OfflineKnowledgeEvaluator, + } + for metric_name in llm_names: + registry.register(metric_name, replacements[metric_name]) + for metric in offline_config.get_eval_metrics(): + if metric.metric_name in llm_names and metric.metric_name != "llm_final_response": + registry.get_evaluator(metric) + return offline_config, registry diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py b/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py new file mode 100644 index 000000000..b898bf242 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/optimizer_worker.py @@ -0,0 +1,79 @@ +"""Isolated process entry point for a live AgentOptimizer invocation.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +from pathlib import Path + +from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt + +from .live_adapter import load_verified_callback +from .schema import parse_strict_json, sanitize, sanitized_exception_message + + +async def _run(request_path: Path) -> None: + request = parse_strict_json(request_path.read_text(encoding="utf-8")) + callback = load_verified_callback( + str(request["callbackSpec"]), + expected_source_path=str(request["callbackSourcePath"]), + expected_source_sha256=str(request["callbackSourceSha256"]), + expected_callable_sha256=str(request["callbackCallableSha256"]), + ) + prompt_paths = request["promptPaths"] + if not isinstance(prompt_paths, dict) or not prompt_paths: + raise ValueError("promptPaths must be a non-empty object") + target = TargetPrompt() + for name, path in prompt_paths.items(): + target.add_path(str(name), str(path)) + result = await AgentOptimizer.optimize( + config_path=str(request["configPath"]), + call_agent=callback, + target_prompt=target, + train_dataset_path=str(request["trainPath"]), + validation_dataset_path=str(request["validationPath"]), + output_dir=str(request["outputDir"]), + update_source=False, + verbose=int(request.get("verbose", 0)), + ) + output_dir = Path(str(request["outputDir"])) + output_dir.mkdir(parents=True, exist_ok=True) + result.dump_to(str(output_dir / "result.json")) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("request") + request_path = Path(parser.parse_args().request).resolve() + try: + asyncio.run(_run(request_path)) + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as error: + try: + request = parse_strict_json(request_path.read_text(encoding="utf-8")) + output_dir = Path(str(request["outputDir"])) + output_dir.mkdir(parents=True, exist_ok=True) + payload = sanitize( + { + "errorType": type(error).__name__, + "message": sanitized_exception_message( + error, + max_text_chars=4000, + ), + }, + max_text_chars=None, + ) + (output_dir / "worker_error.json").write_text( + json.dumps(payload, ensure_ascii=False, indent=2, allow_nan=False) + "\n", + encoding="utf-8", + ) + except Exception: + pass + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py new file mode 100644 index 000000000..7d199ddcd --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/orchestrator.py @@ -0,0 +1,430 @@ +"""Sole composition root and lifecycle coordinator for the loop example.""" + +from __future__ import annotations + +import asyncio +import time +from typing import Callable, Optional + +from trpc_agent_sdk.evaluation import TargetPrompt + +from .artifacts import AuditPersistenceError, AuditSink +from .attribution import attribute_failures +from .backends import CallAgent, TraceEvaluationBackend, create_backends +from .candidate_runtime import generate_candidate, prepare_candidate_inputs +from .configuration import ValidatedRunConfig +from .contracts import CandidateGenerator, EvaluationBackend +from .costing import CostLedger +from .evaluation import compare_snapshots, dataset_contract_payload +from .evaluation_runtime import create_evaluation_runtime +from .gate import evaluate_gate +from .models import ( + AttributionPair, + CandidateProposal, + ComparisonPair, + Decision, + GateDecision, + InnerSplit, + OptimizationReport, + Phase, + RunError, + SnapshotPair, +) +from .preflight import preflight_run +from .prompt_workspace import ( + PromptRestoreError, + PromptRunLock, + PromptWorkspace, + prompt_hashes, +) +from .reporting import ( + build_optimization_report, + create_report_context, + persist_report, + persist_terminal_report, + utc_now, +) +from .schema import add_exception_note, sanitized_exception_message, sanitized_text + +FaultInjector = Callable[[str], None] + + +def _fault(stage: str, injector: Optional[FaultInjector]) -> None: + if injector: + injector(stage) + + +async def run_pipeline( + root_dir: str, + *, + config_path: Optional[str] = None, + train_path: Optional[str] = None, + validation_path: Optional[str] = None, + mode: Optional[str] = None, + run_id: Optional[str] = None, + apply_candidate: Optional[bool] = None, + call_agent: Optional[CallAgent] = None, + callback_spec: Optional[str] = None, + backend: Optional[EvaluationBackend] = None, + candidate_generator: Optional[CandidateGenerator] = None, + fault_injector: Optional[FaultInjector] = None, +) -> OptimizationReport: + """Run the audited lifecycle; REJECT returns normally and ERROR is reported.""" + + started_at = utc_now() + started_clock = time.monotonic() + validated = preflight_run( + root_dir, + config_path=config_path, + train_path=train_path, + validation_path=validation_path, + mode=mode, + run_id=run_id, + apply_candidate=apply_candidate, + call_agent=call_agent, + callback_spec=callback_spec, + backend=backend, + candidate_generator=candidate_generator, + ) + run_lock = PromptRunLock(tuple(validated.prompt_paths.values()), ) + with run_lock: + return await _run_validated_pipeline( + validated, + started_at=started_at, + started_clock=started_clock, + backend=backend, + candidate_generator=candidate_generator, + fault_injector=fault_injector, + ) + + +async def _run_validated_pipeline( + validated: ValidatedRunConfig, + *, + started_at: str, + started_clock: float, + backend: Optional[EvaluationBackend], + candidate_generator: Optional[CandidateGenerator], + fault_injector: Optional[FaultInjector], +) -> OptimizationReport: + """Execute the lifecycle while the caller owns the prompt-set lock.""" + + settings = validated.config.pipeline + custom_backend = backend is not None + custom_components = custom_backend or candidate_generator is not None + if backend is None or candidate_generator is None: + default_backend, default_generator = create_backends( + settings.mode, + live_adapter=validated.live_adapter, + trace_fixture_path=validated.trace_fixture_path, + trace_fixture_hash=validated.input_hashes.get("trace"), + dataset_hashes={ + "train": validated.input_hashes["train"], + "validation": validated.input_hashes["validation"], + }, + optimizer_shutdown_timeout_seconds=settings.optimizer_shutdown_timeout_seconds, + ) + backend = backend or default_backend + candidate_generator = candidate_generator or default_generator + if custom_backend and isinstance(backend, TraceEvaluationBackend): + backend.validate_fixture(validated.train, validated.validation) + + target = TargetPrompt() + for name, path in validated.prompt_paths.items(): + target.add_path(name, path) + workspace = PromptWorkspace(target) + await workspace.initialize() + if workspace.baseline_hashes != validated.prompt_hashes: + raise ValueError("prompt changed between preflight and workspace initialization") + sink = AuditSink( + validated.artifact_root, + validated.run_id, + publication_root=validated.root_dir, + max_file_bytes=settings.max_audit_file_bytes, + max_import_files=settings.max_import_files, + max_import_file_bytes=settings.max_import_file_bytes, + max_import_total_bytes=settings.max_import_total_bytes, + ) + sink.create() + ledger = CostLedger() + evaluation_runtime = create_evaluation_runtime( + validated=validated, + backend=backend, + custom_backend=custom_backend, + sink=sink, + ledger=ledger, + ) + stage = "audit_config" + applied = False + candidate_started = False + baseline_pair = candidate_pair = comparisons = None + baseline_attribution_pair = candidate_attribution_pair = None + proposal: Optional[CandidateProposal] = None + inner: Optional[InnerSplit] = None + gate_decision = None + report_context = create_report_context( + validated, + started_at=started_at, + started_clock=started_clock, + baseline_prompts=workspace.baseline, + baseline_hashes=workspace.baseline_hashes, + callback_spec=(validated.live_adapter.import_path if validated.live_adapter else None), + programmatic_component=custom_components, + ) + + def build_report(status: Decision, report_stage: str, errors: tuple[RunError, ...] = ()) -> OptimizationReport: + candidate_hashes = prompt_hashes(proposal.prompts) if proposal else None + return build_optimization_report( + report_context, + status=status, + stage=report_stage, + applied=applied, + proposal=proposal, + candidate_hashes=candidate_hashes, + baseline=baseline_pair, + candidate=candidate_pair, + delta=comparisons, + baseline_attribution=baseline_attribution_pair, + candidate_attribution=candidate_attribution_pair, + inner=inner, + cost=ledger.summary(), + gate_decision=gate_decision, + artifacts=sink.records(include_report_files=False), + errors=errors, + ) + + def enter_stage(next_stage: str) -> None: + nonlocal stage + stage = next_stage + _fault(stage, fault_injector) + + def retain_progress(phase: Phase, progress: SnapshotPair) -> None: + nonlocal baseline_pair, candidate_pair + nonlocal baseline_attribution_pair, candidate_attribution_pair + if phase == Phase.BASELINE: + baseline_pair = progress + else: + candidate_pair = progress + attribution = AttributionPair( + train=(attribute_failures( + progress.train, + settings.attribution, + max_text_chars=settings.max_text_chars, + ) if progress.train is not None else None), + validation=(attribute_failures( + progress.validation, + settings.attribution, + max_text_chars=settings.max_text_chars, + ) if progress.validation is not None else None), + ) + if phase == Phase.BASELINE: + baseline_attribution_pair = attribution + else: + candidate_attribution_pair = attribution + + try: + enter_stage(stage) + sink.write_json("config.json", validated.config.model_dump(mode="json", by_alias=True)) + sink.write_json("train.evalset.json", dataset_contract_payload(validated.train)) + sink.write_json("val.evalset.json", dataset_contract_payload(validated.validation)) + for name, content in workspace.baseline.items(): + sink.write_text(f"baseline_prompts/{name}.md", content) + + baseline_pair = await evaluation_runtime.evaluate_phase( + train=validated.train, + validation=validated.validation, + prompts=workspace.baseline, + phase=Phase.BASELINE, + on_stage=enter_stage, + on_progress=retain_progress, + ) + if baseline_attribution_pair is None: + raise RuntimeError("baseline attribution was not retained") + baseline_train = baseline_pair.train + baseline_validation = baseline_pair.validation + baseline_train_attr = baseline_attribution_pair.train + baseline_val_attr = baseline_attribution_pair.validation + if (baseline_train is None or baseline_validation is None or baseline_train_attr is None + or baseline_val_attr is None): + raise RuntimeError("baseline evaluation did not produce complete snapshots and attribution") + + enter_stage("inner_split") + inner, inner_train_attr, inner_train_path, inner_selection_path = prepare_candidate_inputs( + train=validated.train, + baseline=baseline_train, + attribution=baseline_train_attr, + seed=settings.seed, + selection_ratio=settings.inner_selection_ratio, + sink=sink, + ) + + enter_stage("candidate_generation") + candidate_started = True + proposal = await generate_candidate( + generator=candidate_generator, + workspace=workspace, + sink=sink, + eval_config=validated.config.evaluate, + optimize_config=validated.config.optimize, + train_attribution=inner_train_attr, + inner_train_path=inner_train_path, + inner_selection_path=inner_selection_path, + ) + for source in proposal.cost_sources: + ledger.record( + f"{stage}.{source.name}", + cost_usd=source.cost_usd, + upper_bound=source.upper_bound, + model_calls=source.model_calls, + metric_calls=source.metric_calls, + token_usage=source.token_usage, + ) + sink.write_json("candidate.json", proposal.model_dump(mode="json", by_alias=True)) + if proposal.adapter_error: + raise RuntimeError(f"candidate adapter rejected optimizer result: {proposal.adapter_error}") + if proposal.status == "CANCELED": + raise asyncio.CancelledError(proposal.error_message or "AgentOptimizer canceled") + if proposal.status == "FAILED": + raise RuntimeError(f"AgentOptimizer candidate generation failed: {proposal.error_message}") + for name, content in proposal.prompts.items(): + sink.write_text(f"candidate_prompts/{name}.md", content) + + async with workspace.temporary(proposal.prompts): + candidate_pair = await evaluation_runtime.evaluate_phase( + train=validated.train, + validation=validated.validation, + prompts=proposal.prompts, + phase=Phase.CANDIDATE, + on_stage=enter_stage, + on_progress=retain_progress, + ) + if candidate_attribution_pair is None: + raise RuntimeError("candidate attribution was not retained") + candidate_train = candidate_pair.train + candidate_validation = candidate_pair.validation + candidate_train_attr = candidate_attribution_pair.train + candidate_val_attr = candidate_attribution_pair.validation + if (candidate_train is None or candidate_validation is None or candidate_train_attr is None + or candidate_val_attr is None): + raise RuntimeError("candidate evaluation did not produce complete snapshots and attribution") + + enter_stage("comparison") + train_comparison = compare_snapshots( + baseline_train, + candidate_train, + epsilon=settings.gate.epsilon, + baseline_attribution=baseline_train_attr, + candidate_attribution=candidate_train_attr, + ) + comparisons = ComparisonPair(train=train_comparison) + validation_comparison = compare_snapshots( + baseline_validation, + candidate_validation, + epsilon=settings.gate.epsilon, + critical_case_ids=settings.critical_case_ids, + hard_case_ids=settings.hard_case_ids, + baseline_attribution=baseline_val_attr, + candidate_attribution=candidate_val_attr, + ) + comparisons = ComparisonPair(train=train_comparison, validation=validation_comparison) + sink.write_json("comparison.json", comparisons.model_dump(mode="json", by_alias=True)) + + def decide(duration_seconds: float) -> GateDecision: + return evaluate_gate( + train=train_comparison, + validation=validation_comparison, + candidate=proposal, + cost=ledger.summary(), + duration_seconds=duration_seconds, + baseline_prompt_hashes=workspace.baseline_hashes, + candidate_prompt_hashes=prompt_hashes(proposal.prompts), + config=settings.gate, + ) + + enter_stage("gate") + gate_decision = decide(time.monotonic() - started_clock) + if gate_decision.decision == Decision.ERROR: + raise ValueError("gate rejected structurally invalid regression inputs") + + enter_stage("pre_apply_report") + pre_apply_report = build_report(gate_decision.decision, stage) + persist_report(sink, pre_apply_report) + + if gate_decision.accepted and settings.apply_candidate: + enter_stage("apply") + await workspace.apply(proposal.prompts) + applied = True + + enter_stage("final_report") + terminal_duration = time.monotonic() - started_clock + gate_decision = decide(terminal_duration) + if gate_decision.decision == Decision.ERROR: + raise ValueError("terminal gate rejected structurally invalid regression inputs") + if applied and not gate_decision.accepted: + await workspace.restore() + applied = False + terminal_duration = time.monotonic() - started_clock + gate_decision = decide(terminal_duration) + final_report = build_report(gate_decision.decision, "complete") + return persist_terminal_report(sink, final_report, duration_seconds=terminal_duration) + except BaseException as error: + if candidate_started and proposal is None: + ledger.record( + "candidate_generation.unreported_failure", + cost_usd=None, + model_calls=None, + metric_calls=None, + ) + fatal_restore: Optional[PromptRestoreError] = None + try: + await workspace.restore() + applied = False + except PromptRestoreError as restore_error: + original_stage = stage + original_message = sanitized_text(str(error), max_text_chars=settings.max_text_chars) + add_exception_note( + restore_error, + f"pipeline failure before final restoration at stage {original_stage!r}: " + f"{type(error).__name__}: {original_message}", + ) + for note in getattr(error, "__notes__", ()): + add_exception_note( + restore_error, + f"prior pipeline diagnostic: {sanitized_text(str(note), max_text_chars=settings.max_text_chars)}", + ) + fatal_restore = restore_error + stage = "restore" + error = restore_error + message = sanitized_exception_message( + error, + max_text_chars=settings.max_text_chars, + ) + run_error = RunError( + stage=stage, + error_type=type(error).__name__, + message=message, + ) + error_report = build_report(Decision.ERROR, stage, (run_error, )) + persistence_error: Optional[BaseException] = None + try: + error_report = persist_terminal_report( + sink, + error_report, + duration_seconds=time.monotonic() - started_clock, + ) + except BaseException as persist_error: + persistence_error = persist_error + if fatal_restore is not None: + if persistence_error is not None: + add_exception_note(fatal_restore, f"terminal audit persistence also failed: {persistence_error}") + raise fatal_restore + if isinstance(error, (asyncio.CancelledError, KeyboardInterrupt, SystemExit)): + if persistence_error is not None: + add_exception_note(error, f"terminal audit persistence also failed: {persistence_error}") + raise + if persistence_error is not None: + audit_error = AuditPersistenceError( + f"terminal audit persistence failed at stage {stage!r}: {persistence_error}") + add_exception_note(audit_error, f"original pipeline error: {type(error).__name__}: {message}") + raise audit_error from persistence_error + return error_report diff --git a/examples/optimization/eval_optimize_loop/pipeline/preflight.py b/examples/optimization/eval_optimize_loop/pipeline/preflight.py new file mode 100644 index 000000000..619a98bef --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/preflight.py @@ -0,0 +1,212 @@ +"""Validated, content-addressed run inputs for the optimization example.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any, Optional + +from trpc_agent_sdk.evaluation import EvalSet + +from .artifacts import load_strict_json +from .evaluation import ( + canonical_json, + dataset_audit_fingerprint, + dataset_fingerprint, + validate_datasets, +) +from .configuration import PipelineConfig, ValidatedRunConfig +from .live_adapter import LiveAdapterSpec +from .schema import validate_safe_component, validate_secret_free_text +from .prompt_workspace import prompt_hashes +from .trace_fixture import TraceFixture + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def inside_example_root(root: Path, relative: str, label: str) -> Path: + """Resolve a configured path while preventing escape from the example root.""" + + raw = Path(relative) + if raw.is_absolute(): + raise ValueError(f"{label} must be relative to the example root") + resolved = (root / raw).resolve() + if root != resolved and root not in resolved.parents: + raise ValueError(f"{label} escapes the example root") + return resolved + + +def _component_identity(component: object | None, default: str) -> str: + if component is None: + return default + component_type = type(component) + return f"{component_type.__module__}.{component_type.__qualname__}" + + +def _adapter_identity( + mode: str, + *, + live_adapter: Optional[LiveAdapterSpec], + backend: object | None, + candidate_generator: object | None, +) -> str: + """Stable identity for the components that can change execution behavior.""" + + return canonical_json({ + "mode": + mode, + "backend": + _component_identity(backend, f"default:{mode}:evaluation:v1"), + "candidateGenerator": + _component_identity(candidate_generator, f"default:{mode}:candidate-generator:v1"), + "callback": + ({ + "importPath": live_adapter.import_path, + "sourceSha256": live_adapter.source_sha256, + "callableSha256": live_adapter.callable_sha256, + } if live_adapter is not None else None), + }) + + +def preflight_run( + root_dir: str, + *, + config_path: Optional[str], + train_path: Optional[str], + validation_path: Optional[str], + mode: Optional[str], + run_id: Optional[str], + apply_candidate: Optional[bool], + call_agent: object | None, + callback_spec: Optional[str], + backend: object | None, + candidate_generator: object | None, +) -> ValidatedRunConfig: + """Load all inputs and derive the immutable run identity before side effects.""" + + root = Path(root_dir).resolve() + config_file = Path(config_path).resolve() if config_path else root / "optimizer.json" + train_file = Path(train_path).resolve() if train_path else root / "train.evalset.json" + validation_file = Path(validation_path).resolve() if validation_path else root / "val.evalset.json" + config = PipelineConfig.model_validate(load_strict_json(config_file)) + settings = config.pipeline + updates: dict[str, Any] = {} + if mode is not None: + updates["mode"] = mode + if run_id is not None: + updates["run_id"] = run_id + if apply_candidate is not None: + updates["apply_candidate"] = apply_candidate + if updates: + settings = type(settings).model_validate({**settings.model_dump(mode="python", by_alias=False), **updates}) + config = config.model_copy(update={"pipeline": settings}) + + train = EvalSet.model_validate(load_strict_json(train_file)) + validation = EvalSet.model_validate(load_strict_json(validation_file)) + metric_names = [metric.metric_name for metric in config.evaluate.get_eval_metrics()] + validate_datasets( + train, + validation, + train_path=str(train_file), + validation_path=str(validation_file), + configured_metrics=metric_names, + critical_case_ids=settings.critical_case_ids, + hard_case_ids=settings.hard_case_ids, + metric_weights=settings.metric_weights, + train_case_weights=settings.train_case_weights, + validation_case_weights=settings.validation_case_weights, + ) + + prompt_paths: dict[str, str] = {} + prompts: dict[str, str] = {} + for name, relative in settings.prompt_paths.items(): + validate_safe_component(name, name="prompt field") + path = inside_example_root(root, relative, f"prompt path {name!r}") + prompts[name] = validate_secret_free_text( + path.read_text(encoding="utf-8"), + name=f"prompt {name!r}", + ) + prompt_paths[name] = str(path) + + hashes = { + "config": + _sha256_file(config_file), + "effectiveConfig": + hashlib.sha256(canonical_json(config.model_dump(mode="json", by_alias=True)).encode("utf-8")).hexdigest(), + "train": + dataset_fingerprint(train), + "validation": + dataset_fingerprint(validation), + } + audit_hashes = { + "train": dataset_audit_fingerprint(train), + "validation": dataset_audit_fingerprint(validation), + } + trace_path: Optional[Path] = None + if settings.mode == "trace": + trace_path = inside_example_root(root, settings.trace_fixture, "trace fixture") + hashes["trace"] = _sha256_file(trace_path) + TraceFixture( + trace_path, + { + "train": hashes["train"], + "validation": hashes["validation"], + }, + hashes["trace"], + ).validate(train, validation) + + live_adapter: Optional[LiveAdapterSpec] = None + if settings.mode == "live": + if backend is None and call_agent is None: + raise ValueError("default live evaluation requires call_agent") + if callback_spec is not None: + if call_agent is None: + raise ValueError("callback_spec requires call_agent for identity verification") + live_adapter = LiveAdapterSpec.resolve(callback_spec, call_agent) + if (backend is None or candidate_generator is None) and live_adapter is None: + raise ValueError("default live components require an importable callback_spec") + + adapter_identity = _adapter_identity( + settings.mode, + live_adapter=live_adapter, + backend=backend, + candidate_generator=candidate_generator, + ) + reproducibility_paths = [config_file, train_file, validation_file] + reproducibility_paths.extend(Path(path) for path in prompt_paths.values()) + if trace_path is not None: + reproducibility_paths.append(trace_path) + if live_adapter is not None: + reproducibility_paths.append(live_adapter.source_path) + hashes["callback"] = live_adapter.source_sha256 + hashes["callbackCallable"] = live_adapter.callable_sha256 + derived = ("run-" + hashlib.sha256( + canonical_json({ + "inputHashes": hashes, + "promptHashes": prompt_hashes(prompts), + "adapterIdentity": adapter_identity, + }).encode("utf-8")).hexdigest()[:16]) + effective_run_id = settings.run_id or derived + validate_safe_component(effective_run_id, name="run ID") + artifact_root = inside_example_root(root, settings.artifact_root, "artifact root") + return ValidatedRunConfig( + root_dir=str(root), + config_path=str(config_file), + train_path=str(train_file), + validation_path=str(validation_file), + trace_fixture_path=str(trace_path) if trace_path else None, + artifact_root=str(artifact_root), + run_id=effective_run_id, + config=config, + train=train, + validation=validation, + input_hashes=hashes, + audit_hashes=audit_hashes, + prompt_paths=prompt_paths, + prompt_hashes=prompt_hashes(prompts), + adapter_identity=adapter_identity, + reproducibility_paths=tuple(str(path.resolve()) for path in reproducibility_paths), + live_adapter=live_adapter, + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py b/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py new file mode 100644 index 000000000..7f3f10604 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/prompt_workspace.py @@ -0,0 +1,224 @@ +"""Verified TargetPrompt mutation, restoration and controlled application.""" + +from __future__ import annotations + +import hashlib +import os +import tempfile +from contextlib import asynccontextmanager +from pathlib import Path +from typing import AsyncIterator + +from trpc_agent_sdk.evaluation import TargetPrompt + +from .schema import add_exception_note, validate_safe_component + + +class PromptRestoreError(RuntimeError): + """The baseline prompt state could not be restored and verified.""" + + +class PromptRunLock: + """Cross-process exclusive lock for one set of file-backed prompts.""" + + def __init__( + self, + prompt_paths: tuple[str, ...], + *, + lock_root: str | None = None, + ) -> None: + if not prompt_paths: + raise ValueError("prompt run lock requires at least one prompt path") + identity = "\0".join(sorted(os.path.normcase(str(Path(path).resolve())) for path in prompt_paths)) + digest = hashlib.sha256(identity.encode("utf-8")).hexdigest() + if lock_root is not None: + root = Path(lock_root).resolve() + else: + root = Path(tempfile.gettempdir()).resolve() / "trpc-agent-eval-optimize-locks" + self.path = root / f"{digest}.lock" + self._handle = None + + @staticmethod + def _lock(handle) -> None: + if os.name == "nt": + import msvcrt + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + + @staticmethod + def _unlock(handle) -> None: + handle.seek(0) + if os.name == "nt": + import msvcrt + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + def acquire(self) -> None: + if self._handle is not None: + raise RuntimeError("prompt run lock is already acquired") + self.path.parent.mkdir(parents=True, exist_ok=True) + handle = self.path.open("a+b") + handle.seek(0) + try: + self._lock(handle) + except (OSError, BlockingIOError) as error: + handle.close() + raise RuntimeError("prompt sources are already owned by another pipeline run") from error + try: + if handle.seek(0, os.SEEK_END) == 0: + handle.write(b"\0") + handle.flush() + except BaseException: + try: + self._unlock(handle) + except OSError: + pass + handle.close() + raise + self._handle = handle + + def release(self) -> None: + handle = self._handle + if handle is None: + return + try: + self._unlock(handle) + finally: + handle.close() + self._handle = None + + def __enter__(self) -> "PromptRunLock": + self.acquire() + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.release() + + +def hash_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def prompt_hashes(prompts: dict[str, str]) -> dict[str, str]: + return {name: hash_text(text) for name, text in prompts.items()} + + +class PromptWorkspace: + """Own every prompt mutation and verify all writes by reading them back.""" + + def __init__(self, target: TargetPrompt) -> None: + if not target.names(): + raise ValueError("PromptWorkspace requires at least one TargetPrompt field") + for name in target.names(): + validate_safe_component(name, name="prompt field") + self._target = target + self._baseline: dict[str, str] | None = None + self._baseline_hashes: dict[str, str] | None = None + + async def initialize(self) -> None: + baseline = await self._target.read_all() + self._validate_prompt_map(baseline) + self._baseline = dict(baseline) + self._baseline_hashes = prompt_hashes(baseline) + + @property + def baseline(self) -> dict[str, str]: + self._ensure_initialized() + return dict(self._baseline or {}) + + @property + def baseline_hashes(self) -> dict[str, str]: + self._ensure_initialized() + return dict(self._baseline_hashes or {}) + + async def current_hashes(self) -> dict[str, str]: + self._ensure_initialized() + current = await self._target.read_all() + self._validate_prompt_map(current) + return prompt_hashes(current) + + async def _write_verified(self, prompts: dict[str, str]) -> dict[str, str]: + self._validate_prompt_map(prompts) + await self._target.write_all(dict(prompts)) + actual = await self._target.read_all() + if actual != prompts: + raise IOError("TargetPrompt read-back differs from requested content") + return prompt_hashes(actual) + + async def restore(self) -> None: + self._ensure_initialized() + try: + hashes = await self._write_verified(self.baseline) + if hashes != self.baseline_hashes: + raise IOError("baseline prompt hashes differ after restoration") + except BaseException as error: + restore_error = PromptRestoreError("baseline prompt restoration could not be verified") + add_exception_note( + restore_error, + f"restoration operation failed with {type(error).__name__}: {error}", + ) + raise restore_error from error + + async def apply(self, candidate: dict[str, str]) -> dict[str, str]: + self._ensure_initialized() + try: + return await self._write_verified(candidate) + except BaseException as primary_error: + try: + await self.restore() + except PromptRestoreError as restore_error: + add_exception_note( + restore_error, + f"prompt apply also failed with {type(primary_error).__name__}: {primary_error}", + ) + raise + raise + + @asynccontextmanager + async def temporary(self, candidate: dict[str, str]) -> AsyncIterator[dict[str, str]]: + self._ensure_initialized() + primary: BaseException | None = None + try: + hashes = await self._write_verified(candidate) + yield hashes + except BaseException as error: + primary = error + raise + finally: + try: + await self.restore() + except PromptRestoreError as restore_error: + if primary is not None: + add_exception_note( + restore_error, + f"prompt operation also failed with {type(primary).__name__}: {primary}", + ) + raise + + def create_candidate_target(self, directory: str) -> TargetPrompt: + """Create an independent file-backed TargetPrompt for AgentOptimizer.""" + + self._ensure_initialized() + root = Path(directory) + root.mkdir(parents=True, exist_ok=False) + target = TargetPrompt() + for name, content in self.baseline.items(): + path = root / f"{name}.md" + path.write_text(content, encoding="utf-8") + target.add_path(name, str(path)) + return target + + def _validate_prompt_map(self, prompts: dict[str, str]) -> None: + expected = set(self._target.names()) + if set(prompts) != expected: + raise ValueError(f"prompt keys mismatch; expected {sorted(expected)}, got {sorted(prompts)}") + if any(not isinstance(text, str) for text in prompts.values()): + raise TypeError("all prompt values must be UTF-8 text strings") + + def _ensure_initialized(self) -> None: + if self._baseline is None or self._baseline_hashes is None: + raise RuntimeError("PromptWorkspace.initialize() must be awaited first") diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporting.py b/examples/optimization/eval_optimize_loop/pipeline/reporting.py new file mode 100644 index 000000000..563b0fc50 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/reporting.py @@ -0,0 +1,379 @@ +"""Environment metadata and render/persist operations for OptimizationReport.""" + +from __future__ import annotations + +import platform +import re +import shlex +import subprocess +import sys +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from trpc_agent_sdk.version import __version__ as sdk_version + +from .artifacts import AuditSink +from .configuration import ValidatedRunConfig +from .models import ( + ArtifactRecord, + AttributionPair, + CandidateProposal, + ComparisonPair, + CostSummary, + Decision, + GateDecision, + InnerSplit, + OptimizationReport, + Reproducibility, + RunError, + SnapshotPair, + SourceApplication, +) + +_GIT_OBJECT_ID_PATTERN = re.compile(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})\Z") + + +@dataclass(frozen=True) +class ReportContext: + """Static run facts shared by pre-apply, terminal and error reports.""" + + validated: ValidatedRunConfig + started_at: str + started_clock: float + reproducibility: Reproducibility + baseline_prompts: dict[str, str] + baseline_hashes: dict[str, str] + + +def create_report_context( + validated: ValidatedRunConfig, + *, + started_at: str, + started_clock: float, + baseline_prompts: dict[str, str], + baseline_hashes: dict[str, str], + callback_spec: Optional[str], + programmatic_component: bool, +) -> ReportContext: + settings = validated.config.pipeline + reproducibility = build_reproducibility( + validated.root_dir, + mode=settings.mode, + config_path=validated.config_path, + train_path=validated.train_path, + validation_path=validated.validation_path, + run_id=validated.run_id, + apply_candidate=settings.apply_candidate, + callback_spec=callback_spec, + trace_fixture=validated.trace_fixture_path, + trace_hash=validated.input_hashes.get("trace"), + programmatic_component=programmatic_component, + input_paths=validated.reproducibility_paths, + ) + return ReportContext( + validated=validated, + started_at=started_at, + started_clock=started_clock, + reproducibility=reproducibility, + baseline_prompts=baseline_prompts, + baseline_hashes=baseline_hashes, + ) + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def build_reproducibility( + repo_root: str, + *, + mode: str, + config_path: str, + train_path: str, + validation_path: str, + run_id: str, + apply_candidate: bool, + callback_spec: Optional[str] = None, + trace_fixture: Optional[str] = None, + trace_hash: Optional[str] = None, + programmatic_component: bool = False, + input_paths: tuple[str, ...] = (), +) -> Reproducibility: + commit: Optional[str] = None + dirty: Optional[bool] = None + reason: Optional[str] = None + git_root: Optional[Path] = None + try: + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + dirty = bool( + subprocess.run( + ["git", "status", "--porcelain"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip()) + git_root = Path( + subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip()).resolve() + except (OSError, subprocess.SubprocessError): + reason = "git_metadata_unavailable" + # Git repositories may use SHA-1 or SHA-256 object IDs. ``rev-parse`` + # already verified that HEAD resolves; this check rejects malformed output. + if reason is None and (commit is None or _GIT_OBJECT_ID_PATTERN.fullmatch(commit) is None): + reason = "git_commit_invalid" + elif reason is None and dirty: + reason = "worktree_dirty" + elif reason is None and programmatic_component: + reason = "programmatic_custom_component" + elif reason is None and mode == "live" and not callback_spec: + reason = "live_callback_not_importable" + elif reason is None and mode == "trace" and (not trace_fixture or not trace_hash): + reason = "trace_fixture_not_pinned" + elif reason is None and git_root is None: + reason = "git_root_unavailable" + if reason is None: + for input_path in input_paths: + resolved = Path(input_path).resolve() + try: + relative = resolved.relative_to(git_root) + except ValueError: + reason = "input_outside_git" + break + tracked = subprocess.run( + ["git", "ls-files", "--error-unmatch", "--", + relative.as_posix()], + cwd=git_root, + check=False, + capture_output=True, + text=True, + ) + if tracked.returncode != 0: + reason = "input_not_tracked" + break + reproducible = reason is None + command = None + if reproducible: + if git_root is None: + raise RuntimeError("reproducible replay requires a resolved Git root") + + def replay_path(path: str | Path) -> str: + resolved = Path(path).resolve() + try: + return resolved.relative_to(git_root).as_posix() + except ValueError: + return str(resolved) + + args = [ + # Keep the report portable across checkouts; environment metadata + # records the interpreter version used for the original run. + "python", + replay_path(Path(repo_root) / "run_pipeline.py"), + "--mode", + mode, + "--config", + replay_path(config_path), + "--train", + replay_path(train_path), + "--validation", + replay_path(validation_path), + "--run-id", + f"{run_id}-replay", + "--apply-candidate" if apply_candidate else "--no-apply-candidate", + ] + if callback_spec: + args.extend(("--call-agent", callback_spec)) + command = shlex.join(args) + return Reproducibility( + reproducible=reproducible, + command=command, + reason=reason, + git_commit=commit, + git_dirty=dirty, + ) + + +def environment_metadata() -> dict[str, str]: + return { + "python": sys.version.split()[0], + "sdk": sdk_version, + "platform": platform.platform(), + } + + +def _report_path(path: str, root: Path) -> str: + resolved = Path(path) + try: + return resolved.relative_to(root).as_posix() + except ValueError: + return str(resolved) + + +def build_optimization_report( + context: ReportContext, + *, + status: Decision, + stage: str, + applied: bool, + proposal: Optional[CandidateProposal], + candidate_hashes: Optional[dict[str, str]], + baseline: Optional[SnapshotPair], + candidate: Optional[SnapshotPair], + delta: Optional[ComparisonPair], + baseline_attribution: Optional[AttributionPair], + candidate_attribution: Optional[AttributionPair], + inner: Optional[InnerSplit], + cost: CostSummary, + gate_decision: Optional[GateDecision], + artifacts: tuple[ArtifactRecord, ...], + errors: tuple[RunError, ...] = (), +) -> OptimizationReport: + """Assemble report-model facts without rerunning aggregation or gate logic.""" + + validated = context.validated + settings = validated.config.pipeline + report_root = Path(validated.root_dir) + final_hashes = candidate_hashes if applied and candidate_hashes else context.baseline_hashes + return OptimizationReport( + run_id=validated.run_id, + status=status, + mode=settings.mode, + stage=stage, + started_at=context.started_at, + finished_at=utc_now(), + duration_seconds=time.monotonic() - context.started_clock, + reproducibility=context.reproducibility, + inputs={ + "hashes": validated.input_hashes, + "auditHashes": validated.audit_hashes, + "paths": { + "config": _report_path(validated.config_path, report_root), + "train": _report_path(validated.train_path, report_root), + "validation": _report_path(validated.validation_path, report_root), + }, + "environment": environment_metadata(), + "seed": settings.seed, + "adapterIdentity": validated.adapter_identity, + }, + prompts={ + "baseline": context.baseline_prompts, + "baselineHashes": context.baseline_hashes, + "candidate": proposal.prompts if proposal else None, + "candidateHashes": candidate_hashes, + }, + baseline=baseline, + candidate=candidate, + delta=delta, + failure_attribution=baseline_attribution, + candidate_failure_attribution=candidate_attribution, + optimization=({ + "proposal": proposal.model_dump(mode="json", by_alias=True), + "innerSplit": inner.model_dump(mode="json", by_alias=True) if inner else None, + } if proposal else None), + cost=cost, + gate_decision=gate_decision, + source_application=SourceApplication( + requested=settings.apply_candidate, + applied=applied, + baseline_hashes=context.baseline_hashes, + final_hashes=final_hashes, + ), + artifacts=artifacts, + errors=errors, + ) + + +def render_markdown(report: OptimizationReport) -> str: + """Render only report-model facts; no business aggregation occurs here.""" + + lines = [ + f"# Optimization Report: {report.status.value}", + "", + f"- Run ID: `{report.run_id}`", + f"- Mode: `{report.mode}`", + f"- Final stage: `{report.stage}`", + f"- Duration: `{report.duration_seconds:.3f}s`", + f"- Reproducible: `{'yes' if report.reproducibility.reproducible else 'no'}`", + f"- Source prompt applied: `{'yes' if report.source_application.applied else 'no'}`", + ] + if (report.baseline and report.candidate and report.baseline.train and report.baseline.validation + and report.candidate.train and report.candidate.validation): + lines.extend([ + "", + "## Regression", + "", + "| Split | Baseline score | Candidate score | Baseline pass rate | Candidate pass rate |", + "|---|---:|---:|---:|---:|", + (f"| train | {report.baseline.train.dataset_score:.4f} | " + f"{report.candidate.train.dataset_score:.4f} | " + f"{report.baseline.train.pass_rate:.4f} | {report.candidate.train.pass_rate:.4f} |"), + (f"| validation | {report.baseline.validation.dataset_score:.4f} | " + f"{report.candidate.validation.dataset_score:.4f} | " + f"{report.baseline.validation.pass_rate:.4f} | " + f"{report.candidate.validation.pass_rate:.4f} |"), + ]) + if report.delta and report.delta.validation: + lines.extend(["", "## Validation Transitions", ""]) + for case in report.delta.validation.cases: + lines.append(f"- `{case.case_id}`: `{case.transition.value}` ({case.delta:+.4f})") + if report.gate_decision: + failed = [check.code for check in report.gate_decision.checks if not check.passed] + lines.extend([ + "", + "## Gate", + "", + f"- Decision: `{report.gate_decision.decision.value}`", + f"- Failed checks: `{', '.join(failed) if failed else 'none'}`", + f"- Reasons: `{', '.join(report.gate_decision.reasons) if report.gate_decision.reasons else 'none'}`", + ("- Overfit detected: `yes`" + if "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in report.gate_decision.reasons else "- Overfit detected: `no`"), + ]) + if report.cost: + cost = "unknown" if report.cost.total_cost_usd is None else f"${report.cost.total_cost_usd:.6f}" + lines.extend(["", "## Cost", "", f"- Total: `{cost}`"]) + if report.errors: + lines.extend(["", "## Errors", ""]) + for error in report.errors: + lines.append(f"- `{error.stage}/{error.error_type}`: {error.message}") + return "\n".join(lines) + "\n" + + +def persist_report(sink: AuditSink, report: OptimizationReport) -> None: + sink.write_json("optimization_report.json", report.model_dump(mode="json", by_alias=True)) + sink.write_text("optimization_report.md", render_markdown(report)) + + +def persist_terminal_report( + sink: AuditSink, + report: OptimizationReport, + *, + duration_seconds: float, +) -> OptimizationReport: + """Commit one authoritative terminal report and publish its latest snapshots.""" + + completed = OptimizationReport.model_validate({ + **report.model_dump(mode="python", by_alias=False), + "finished_at": + utc_now(), + "duration_seconds": + duration_seconds, + }) + persist_report(sink, completed) + sink.write_manifest() + sink.publish_latest_snapshot("optimization_report.json", "optimization_report.json") + sink.publish_latest_snapshot("optimization_report.md", "optimization_report.md") + return completed diff --git a/examples/optimization/eval_optimize_loop/pipeline/schema.py b/examples/optimization/eval_optimize_loop/pipeline/schema.py new file mode 100644 index 000000000..6285131bd --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/schema.py @@ -0,0 +1,215 @@ +"""Shared strict-schema primitives with no pipeline dependencies.""" + +from __future__ import annotations + +import json +import math +import re +from typing import Any + +from pydantic import BaseModel, ConfigDict +from pydantic.alias_generators import to_camel + +_WINDOWS_RESERVED = { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{i}" for i in range(1, 10)), + *(f"LPT{i}" for i in range(1, 10)), +} +_SAFE_COMPONENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_SECRET_ASSIGNMENT = re.compile(r"(?i)(?P(?P[\"']?)[A-Za-z0-9_.-]*" + r"(?:api[_-]?key|access[_-]?token|refresh[_-]?token|password|authorization|" + r"cookie|secret|credential|private[_-]?key|secret[_-]?access[_-]?key)" + r"(?P=key_quote)\s*[:=]\s*)" + r"(?P\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*'|" + r"(?:bearer\s+)?[^\s,;}\]]+)") +_BEARER_TOKEN = re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]+") +_SECRET_SUFFIXES = ( + "apikey", + "token", + "password", + "authorization", + "cookie", + "secret", + "credential", + "accesstoken", + "refreshtoken", + "privatekey", + "secretaccesskey", +) + + +def parse_strict_json(text: str) -> dict[str, Any]: + """Parse a JSON object while rejecting duplicates and non-finite values.""" + + def object_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key!r}") + result[key] = value + return result + + def invalid_constant(value: str) -> Any: + raise ValueError(f"non-finite JSON number: {value}") + + def finite_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError(f"non-finite JSON number: {value}") + return parsed + + payload = json.loads( + text, + object_pairs_hook=object_pairs, + parse_constant=invalid_constant, + parse_float=finite_float, + ) + if not isinstance(payload, dict): + raise ValueError("JSON root must be an object") + return payload + + +def add_exception_note(error: BaseException, note: str) -> None: + """Attach diagnostics without dropping Python 3.10 compatibility.""" + + add_note = getattr(error, "add_note", None) + if callable(add_note): + add_note(note) + return + notes = list(getattr(error, "__notes__", ())) + notes.append(note) + try: + error.__notes__ = notes + except (AttributeError, TypeError): + pass + + +def validate_safe_component(value: str, *, name: str = "component") -> str: + """Validate one portable path component.""" + + if not value or value != value.strip() or not _SAFE_COMPONENT.fullmatch(value): + raise ValueError(f"{name} must be a non-empty portable path component") + if ".." in value or value.endswith((".", " ")): + raise ValueError(f"{name} contains an unsafe path component") + stem = value.split(".", 1)[0].upper() + if stem in _WINDOWS_RESERVED: + raise ValueError(f"{name} is reserved on Windows") + return value + + +def _is_secret_key(value: str) -> bool: + normalized = re.sub(r"[^a-z0-9]", "", value.casefold()) + return normalized in _SECRET_SUFFIXES or normalized.endswith(_SECRET_SUFFIXES) + + +def _redact_assignment(match: re.Match[str]) -> str: + value = match.group("value") + quote = value[0] if value and value[0] in "\"'" else "" + return f"{match.group('prefix')}{quote}[REDACTED]{quote}" + + +def sanitize(value: Any, *, max_text_chars: int | None) -> Any: + """Recursively redact credentials and optionally bound individual strings.""" + + if isinstance(value, dict): + return { + str(key): ("[REDACTED]" if _is_secret_key(str(key)) else sanitize(item, max_text_chars=max_text_chars)) + for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [sanitize(item, max_text_chars=max_text_chars) for item in value] + if isinstance(value, str): + text = _SECRET_ASSIGNMENT.sub(_redact_assignment, value) + text = _BEARER_TOKEN.sub(lambda match: match.group(1) + "[REDACTED]", text) + if max_text_chars is None or len(text) <= max_text_chars: + return text + return text[:max_text_chars] + "...[truncated]" + return value + + +def validate_secret_free_text(value: str, *, name: str) -> str: + """Reject prompt content that would be changed by audit redaction.""" + + if sanitize(value, max_text_chars=None) != value: + raise ValueError(f"{name} contains credential-shaped content") + return value + + +def sanitized_text(value: Any, *, max_text_chars: int) -> str: + """Render one bounded text value after structured recursive sanitization.""" + + if isinstance(value, BaseException): + return sanitized_exception_message(value, max_text_chars=max_text_chars) + if isinstance(value, str) and len(value) > max_text_chars: + marker = "...[truncated]" + value = value[:max(0, max_text_chars - len(marker))] + marker + clean = sanitize(value, max_text_chars=max_text_chars) + if isinstance(clean, str): + text = clean + elif isinstance(clean, (dict, list)): + text = json.dumps(clean, ensure_ascii=False, sort_keys=True, default=str) + else: + text = str(clean) + text = str(sanitize(text, max_text_chars=max_text_chars)) + return text if len(text) <= max_text_chars else text[:max_text_chars] + "...[truncated]" + + +def sanitized_exception_message( + error: BaseException, + *, + max_text_chars: int, + max_parts: int = 8, +) -> str: + """Render an exception chain with bounded count, text and total size.""" + + notes = tuple(getattr(error, "__notes__", ())) + all_parts = [("primary", str(error))] + all_parts.extend((f"diagnostic-{index}", str(note)) for index, note in enumerate(notes, start=1)) + omitted = 0 + if len(all_parts) > max_parts: + keep_notes = max_parts - 1 + first = max(1, keep_notes // 2) + last = keep_notes - first + selected = [all_parts[0], *all_parts[1:1 + first]] + if last: + selected.extend(all_parts[-last:]) + omitted = len(all_parts) - len(selected) + all_parts = selected + omitted_text = f"; {omitted} diagnostics omitted" if omitted else "" + overhead = sum(len(label) + 2 for label, _ in all_parts) + len(omitted_text) + part_budget = max(1, (max_text_chars - overhead) // max(1, len(all_parts))) + rendered: list[str] = [] + for label, raw in all_parts: + marker = "...[truncated]" + bounded = raw if len(raw) <= part_budget else raw[:max(0, part_budget - len(marker))] + marker + clean = sanitize(bounded, max_text_chars=part_budget) + rendered.append(f"{label}: {clean if isinstance(clean, str) else str(clean)}") + result = "\n".join(rendered) + omitted_text + return result[:max_text_chars] + + +class StrictModel(BaseModel): + model_config = ConfigDict( + extra="forbid", + frozen=True, + populate_by_name=True, + alias_generator=to_camel, + arbitrary_types_allowed=True, + ) + + +def finite_number(value: float, name: str) -> float: + value = float(value) + if not math.isfinite(value): + raise ValueError(f"{name} must be finite") + return value + + +def non_negative_number(value: float, name: str) -> float: + value = finite_number(value, name) + if value < 0: + raise ValueError(f"{name} must be non-negative") + return value diff --git a/examples/optimization/eval_optimize_loop/pipeline/trace_fixture.py b/examples/optimization/eval_optimize_loop/pipeline/trace_fixture.py new file mode 100644 index 000000000..ca53dbf24 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/trace_fixture.py @@ -0,0 +1,59 @@ +"""Hash-pinned trace fixture contract shared by preflight and replay.""" + +from __future__ import annotations + +import hashlib +from copy import deepcopy +from pathlib import Path +from typing import Optional + +from trpc_agent_sdk.evaluation import EvalSet + +from .models import Phase, Split +from .schema import parse_strict_json + + +class TraceFixture: + """Parse, pin and validate every phase/split in a trace fixture.""" + + def __init__( + self, + path: str | Path, + dataset_hashes: dict[str, str], + expected_hash: Optional[str] = None, + ) -> None: + self.path = Path(path) + self.dataset_hashes = dict(dataset_hashes) + if not self.path.is_file(): + raise FileNotFoundError(f"trace fixture does not exist: {self.path}") + content = self.path.read_bytes() + self.sha256 = hashlib.sha256(content).hexdigest() + if expected_hash is not None and self.sha256 != expected_hash: + raise ValueError("trace fixture changed after preflight") + self.payload = parse_strict_json(content.decode("utf-8")) + + def validate(self, train: EvalSet, validation: EvalSet) -> None: + for phase in Phase: + self.eval_set(train, Split.TRAIN, phase) + self.eval_set(validation, Split.VALIDATION, phase) + + def eval_set(self, eval_set: EvalSet, split: Split, phase: Phase) -> EvalSet: + if self.payload.get("schemaVersion") != "v1": + raise ValueError("unsupported trace fixture schemaVersion") + if self.payload.get("datasetHashes") != self.dataset_hashes: + raise ValueError("trace fixture dataset hashes do not match validated inputs") + try: + cases = self.payload["phases"][phase.value][split.value] + except (KeyError, TypeError) as error: + raise ValueError(f"trace fixture is missing {phase.value}/{split.value}") from error + expected_ids = [case.eval_id for case in eval_set.eval_cases] + if not isinstance(cases, dict) or set(cases) != set(expected_ids): + raise ValueError("trace fixture case IDs do not match the dataset") + raw = eval_set.model_dump(mode="json", by_alias=True, exclude_none=True) + for case in raw["evalCases"]: + conversation = deepcopy(cases[case["evalId"]]) + if not isinstance(conversation, list) or not conversation: + raise ValueError("each trace fixture conversation must be non-empty") + case["evalMode"] = "trace" + case["actualConversation"] = conversation + return EvalSet.model_validate(raw) diff --git a/examples/optimization/eval_optimize_loop/prompts/system.md b/examples/optimization/eval_optimize_loop/prompts/system.md new file mode 100644 index 000000000..6b4c11388 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/system.md @@ -0,0 +1,3 @@ +You are a deterministic response agent. + +Return the requested answer exactly. Do not add commentary. diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 000000000..8adfdcd1e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,74 @@ +"""Thin CLI for the auditable evaluation and optimization loop.""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from examples.optimization.eval_optimize_loop.pipeline.orchestrator import run_pipeline +from examples.optimization.eval_optimize_loop.pipeline.live_adapter import load_callback +from examples.optimization.eval_optimize_loop.pipeline.schema import sanitized_exception_message + +_CLI_ERROR_MAX_CHARS = 4000 + + +def _format_cli_error(error: BaseException) -> str: + error_type = type(error).__name__[:128] + prefix = f"ERROR: {error_type}: " + summary = sanitized_exception_message( + error, + max_text_chars=_CLI_ERROR_MAX_CHARS - len(prefix), + ) + return prefix + summary + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run the auditable prompt optimization loop.") + parser.add_argument("--mode", choices=("fake", "trace", "live")) + parser.add_argument("--config") + parser.add_argument("--train") + parser.add_argument("--validation") + parser.add_argument("--run-id") + parser.add_argument("--call-agent", metavar="MODULE:FUNCTION") + parser.add_argument( + "--apply-candidate", + action=argparse.BooleanOptionalAction, + default=None, + ) + return parser + + +async def _main(args: argparse.Namespace) -> int: + callback = load_callback(args.call_agent) if args.call_agent else None + report = await run_pipeline( + str(_HERE), + config_path=args.config, + train_path=args.train, + validation_path=args.validation, + mode=args.mode, + run_id=args.run_id, + apply_candidate=args.apply_candidate, + call_agent=callback, + callback_spec=args.call_agent, + ) + print(f"{report.status.value}: run={report.run_id} stage={report.stage}") + if report.gate_decision and report.gate_decision.reasons: + print("reasons=" + ",".join(report.gate_decision.reasons)) + return 1 if report.status.value == "ERROR" else 0 + + +if __name__ == "__main__": + try: + raise SystemExit(asyncio.run(_main(_parser().parse_args()))) + except (KeyboardInterrupt, asyncio.CancelledError): + raise + except Exception as error: + print(_format_cli_error(error), file=sys.stderr) + raise SystemExit(1) diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json new file mode 100644 index 000000000..b64df9ed0 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -0,0 +1,2323 @@ +{ + "artifacts": [ + { + "byteSize": 101, + "path": "baseline_prompts/system.md", + "sha256": "aacc5210d0fe4482263f69f5f83241cdf9252e44be22a70de08107daec05e639" + }, + { + "byteSize": 13921, + "path": "baseline_train/snapshot.json", + "sha256": "e1e7a1d29d8495b2f97d90ddef4179aca40c7da3cd7f28e7fac51b5207273489" + }, + { + "byteSize": 13955, + "path": "baseline_validation/snapshot.json", + "sha256": "c56620490167625ac3aa84b67bf7914b9bbeb17a8ecb56547ac28d2b4990d5d1" + }, + { + "byteSize": 1257, + "path": "candidate.json", + "sha256": "b426767863e0ec1d1b3e7310ad9e05809362926993428e83234ec6214a61fc17" + }, + { + "byteSize": 132, + "path": "candidate_prompts/system.md", + "sha256": "ed2dfc103a7763a126f690e536572c5d06d20e2119ef7ef707c0a72deb162be2" + }, + { + "byteSize": 13860, + "path": "candidate_train/snapshot.json", + "sha256": "0da1efea5c6dfc843ab03d09fc687724bb24596456c1d47692d4fd70a7afb6c7" + }, + { + "byteSize": 13953, + "path": "candidate_validation/snapshot.json", + "sha256": "d7ba2a08412cb7adc5a82ccb408a5e9ad5da3564550f13ec2468389c0993cf9f" + }, + { + "byteSize": 5986, + "path": "comparison.json", + "sha256": "ce6e77862b58a753e4fe61fcf044161dd74b0194f15e61e9cb21c1e9896fd836" + }, + { + "byteSize": 2924, + "path": "config.json", + "sha256": "911831d3f66d621893f03ca83840d507d29365c18a6b2c6dbbb7911d656b632e" + }, + { + "byteSize": 2163, + "path": "inner_selection.evalset.json", + "sha256": "af935deba41f9e95db01dcb4b8db86c17619a2f205d6dddd0addad2415896715" + }, + { + "byteSize": 645, + "path": "inner_train.attribution.json", + "sha256": "77478a288c3f38a77677e169e5da7d1c88509c0e3091e0989c70a26a1c216aa4" + }, + { + "byteSize": 4108, + "path": "inner_train.evalset.json", + "sha256": "b7146ee360286979afab3f62bbace6255b13293037a27757639d294a3fc7bdff" + }, + { + "byteSize": 6045, + "path": "train.evalset.json", + "sha256": "bb66729048d0076e1f6f08b4367641bb28c0c2026ca6278d304e93360a39a945" + }, + { + "byteSize": 6061, + "path": "val.evalset.json", + "sha256": "11df6126e19a1e3cc8126ba6575aa6c69ed8c014d2c0b13484ade19be3d21afb" + } + ], + "baseline": { + "train": { + "caseIds": [ + "train_improve_alpha", + "train_improve_beta", + "train_stable" + ], + "cases": [ + { + "caseId": "train_improve_alpha", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "reason": null, + "rubrics": [], + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.1068149, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "INCORRECT", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 0.0 + }, + { + "caseId": "train_improve_beta", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "reason": null, + "rubrics": [], + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.1068149, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "INCORRECT", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 0.0 + }, + { + "caseId": "train_stable", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.1068149, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + } + ], + "datasetScore": 0.3333333333333333, + "metricScores": { + "final_response_avg_score": 0.3333333333333333 + }, + "passRate": 0.3333333333333333, + "phase": "baseline", + "split": "train" + }, + "validation": { + "caseIds": [ + "validation_improve", + "validation_regress", + "validation_stable" + ], + "cases": [ + { + "caseId": "validation_improve", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "reason": null, + "rubrics": [], + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.1110127, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "INCORRECT", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 0.0 + }, + { + "caseId": "validation_regress", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.1110127, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:regress; ANSWER:DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:regress; ANSWER:DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + }, + { + "caseId": "validation_stable", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.1110127, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + } + ], + "datasetScore": 0.6666666666666666, + "metricScores": { + "final_response_avg_score": 0.6666666666666666 + }, + "passRate": 0.6666666666666666, + "phase": "baseline", + "split": "validation" + } + }, + "candidate": { + "train": { + "caseIds": [ + "train_improve_alpha", + "train_improve_beta", + "train_stable" + ], + "cases": [ + { + "caseId": "train_improve_alpha", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.1273332, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:ALPHA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + }, + { + "caseId": "train_improve_beta", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.1273332, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:BETA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + }, + { + "caseId": "train_stable", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.1273332, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "train-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "train-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:ELEVEN", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + } + ], + "datasetScore": 1.0, + "metricScores": { + "final_response_avg_score": 1.0 + }, + "passRate": 1.0, + "phase": "candidate", + "split": "train" + }, + "validation": { + "caseIds": [ + "validation_improve", + "validation_regress", + "validation_stable" + ], + "cases": [ + { + "caseId": "validation_improve", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.131333, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-1", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:improve; ANSWER:GAMMA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + }, + { + "caseId": "validation_regress", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": false, + "reason": null, + "rubrics": [], + "score": 0.0, + "threshold": 1.0 + } + ], + "passed": false, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.131333, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "INCORRECT", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:regress; ANSWER:DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-2", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:regress; ANSWER:DELTA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 0.0 + }, + { + "caseId": "validation_stable", + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runs": [ + { + "error": null, + "metrics": [ + { + "metricName": "final_response_avg_score", + "passed": true, + "reason": null, + "rubrics": [], + "score": 1.0, + "threshold": 1.0 + } + ], + "passed": true, + "runId": 1, + "trace": [ + { + "actual": { + "creationTimestamp": 1785431480.131333, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": null + }, + "intermediateData": null, + "invocationId": "validation-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + }, + "expected": { + "creationTimestamp": 0.0, + "finalResponse": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "model" + }, + "intermediateData": null, + "invocationId": "validation-input-3", + "userContent": { + "parts": [ + { + "audioTranscription": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "mediaResolution": null, + "partMetadata": null, + "text": "BEHAVIOR:stable; ANSWER:OMEGA", + "thought": null, + "thoughtSignature": null, + "toolCall": null, + "toolResponse": null, + "videoMetadata": null + } + ], + "role": "user" + } + } + } + ] + } + ], + "score": 1.0 + } + ], + "datasetScore": 0.6666666666666666, + "metricScores": { + "final_response_avg_score": 0.6666666666666666 + }, + "passRate": 0.6666666666666666, + "phase": "candidate", + "split": "validation" + } + }, + "candidateFailureAttribution": { + "train": { + "failures": [], + "phase": "candidate", + "split": "train", + "statistics": { + "primaryCategoryCounts": {}, + "totalFailures": 0 + } + }, + "validation": { + "failures": [ + { + "caseId": "validation_regress", + "confidence": "high", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + } + ], + "phase": "candidate", + "split": "validation", + "statistics": { + "primaryCategoryCounts": { + "FINAL_RESPONSE_MISMATCH": 1 + }, + "totalFailures": 1 + } + } + }, + "cost": { + "sources": [ + { + "costUsd": 0.0, + "metricCalls": 3, + "modelCalls": 3, + "name": "baseline_train", + "tokenUsage": {}, + "upperBound": false + }, + { + "costUsd": 0.0, + "metricCalls": 3, + "modelCalls": 3, + "name": "baseline_validation", + "tokenUsage": {}, + "upperBound": false + }, + { + "costUsd": 0.0, + "metricCalls": 0, + "modelCalls": 0, + "name": "candidate_generation.deterministic", + "tokenUsage": {}, + "upperBound": false + }, + { + "costUsd": 0.0, + "metricCalls": 3, + "modelCalls": 3, + "name": "candidate_train", + "tokenUsage": {}, + "upperBound": false + }, + { + "costUsd": 0.0, + "metricCalls": 3, + "modelCalls": 3, + "name": "candidate_validation", + "tokenUsage": {}, + "upperBound": false + } + ], + "totalCostUsd": 0.0 + }, + "delta": { + "train": { + "cases": [ + { + "baselineAttribution": { + "caseId": "train_improve_alpha", + "confidence": "high", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + }, + "baselinePassed": false, + "baselineScore": 0.0, + "candidateAttribution": null, + "candidatePassed": true, + "candidateScore": 1.0, + "caseId": "train_improve_alpha", + "critical": false, + "delta": 1.0, + "hard": false, + "metrics": [ + { + "baseline": 0.0, + "candidate": 1.0, + "delta": 1.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": false, + "transition": "NEW_PASS" + }, + { + "baselineAttribution": { + "caseId": "train_improve_beta", + "confidence": "high", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + }, + "baselinePassed": false, + "baselineScore": 0.0, + "candidateAttribution": null, + "candidatePassed": true, + "candidateScore": 1.0, + "caseId": "train_improve_beta", + "critical": false, + "delta": 1.0, + "hard": false, + "metrics": [ + { + "baseline": 0.0, + "candidate": 1.0, + "delta": 1.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": false, + "transition": "NEW_PASS" + }, + { + "baselineAttribution": null, + "baselinePassed": true, + "baselineScore": 1.0, + "candidateAttribution": null, + "candidatePassed": true, + "candidateScore": 1.0, + "caseId": "train_stable", + "critical": false, + "delta": 0.0, + "hard": false, + "metrics": [ + { + "baseline": 1.0, + "candidate": 1.0, + "delta": 0.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": false, + "transition": "UNCHANGED" + } + ], + "metricDeltas": { + "final_response_avg_score": 0.6666666666666667 + }, + "passRateDelta": 0.6666666666666667, + "scoreDelta": 0.6666666666666667, + "split": "train" + }, + "validation": { + "cases": [ + { + "baselineAttribution": { + "caseId": "validation_improve", + "confidence": "high", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + }, + "baselinePassed": false, + "baselineScore": 0.0, + "candidateAttribution": null, + "candidatePassed": true, + "candidateScore": 1.0, + "caseId": "validation_improve", + "critical": false, + "delta": 1.0, + "hard": false, + "metrics": [ + { + "baseline": 0.0, + "candidate": 1.0, + "delta": 1.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": false, + "transition": "NEW_PASS" + }, + { + "baselineAttribution": null, + "baselinePassed": true, + "baselineScore": 1.0, + "candidateAttribution": { + "caseId": "validation_regress", + "confidence": "high", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + }, + "candidatePassed": false, + "candidateScore": 0.0, + "caseId": "validation_regress", + "critical": false, + "delta": -1.0, + "hard": true, + "metrics": [ + { + "baseline": 1.0, + "candidate": 0.0, + "delta": -1.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": true, + "transition": "NEW_FAIL" + }, + { + "baselineAttribution": null, + "baselinePassed": true, + "baselineScore": 1.0, + "candidateAttribution": null, + "candidatePassed": true, + "candidateScore": 1.0, + "caseId": "validation_stable", + "critical": true, + "delta": 0.0, + "hard": false, + "metrics": [ + { + "baseline": 1.0, + "candidate": 1.0, + "delta": 0.0, + "metricName": "final_response_avg_score" + } + ], + "newHardFailure": false, + "transition": "UNCHANGED" + } + ], + "metricDeltas": { + "final_response_avg_score": 0.0 + }, + "passRateDelta": 0.0, + "scoreDelta": 0.0, + "split": "validation" + } + }, + "durationSeconds": 0.3899999999994179, + "errors": [], + "failureAttribution": { + "train": { + "failures": [ + { + "caseId": "train_improve_alpha", + "confidence": "high", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + }, + { + "caseId": "train_improve_beta", + "confidence": "high", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + } + ], + "phase": "baseline", + "split": "train", + "statistics": { + "primaryCategoryCounts": { + "FINAL_RESPONSE_MISMATCH": 2 + }, + "totalFailures": 2 + } + }, + "validation": { + "failures": [ + { + "caseId": "validation_improve", + "confidence": "high", + "evidence": [ + "{\"metric\": \"final_response_avg_score\", \"score\": 0.0, \"threshold\": 1.0}" + ], + "primary": "FINAL_RESPONSE_MISMATCH", + "reasons": [ + "Metric final_response_avg_score scored 0 below threshold 1." + ], + "secondary": [], + "triggerMetrics": [ + "final_response_avg_score" + ], + "triggerRubrics": [] + } + ], + "phase": "baseline", + "split": "validation", + "statistics": { + "primaryCategoryCounts": { + "FINAL_RESPONSE_MISMATCH": 1 + }, + "totalFailures": 1 + } + } + }, + "finishedAt": "2026-07-30T17:11:20.215369+00:00", + "gateDecision": { + "accepted": false, + "checks": [ + { + "code": "REPORT_COMPLETE", + "message": "All required regression snapshots and report inputs are present.", + "observed": true, + "passed": true, + "threshold": true + }, + { + "code": "CANDIDATE_CHANGED", + "message": "Candidate prompt hashes differ from baseline.", + "observed": true, + "passed": true, + "threshold": true + }, + { + "code": "VALIDATION_SCORE_DELTA", + "message": "Held-out validation score delta meets the configured minimum.", + "observed": 0.0, + "passed": false, + "threshold": 0.05 + }, + { + "code": "VALIDATION_PASS_RATE_DELTA", + "message": "Held-out validation pass-rate delta meets the configured minimum.", + "observed": 0.0, + "passed": true, + "threshold": 0.0 + }, + { + "code": "NO_NEW_HARD_FAIL", + "message": "New held-out hard failures stay within budget.", + "observed": 1, + "passed": false, + "threshold": 0 + }, + { + "code": "CRITICAL_CASE_NON_REGRESSION", + "message": "Critical held-out cases do not regress.", + "observed": [], + "passed": true, + "threshold": 0.0 + }, + { + "code": "METRIC_NON_REGRESSION", + "message": "Configured validation metrics stay within regression limits.", + "observed": {}, + "passed": true, + "threshold": { + "final_response_avg_score": 0.0 + } + }, + { + "code": "COST_BUDGET", + "message": "Known enabled costs stay within budget.", + "observed": 0.0, + "passed": true, + "threshold": null + }, + { + "code": "DURATION_BUDGET", + "message": "Gate-observed duration stays within budget.", + "observed": 0.31199999999807915, + "passed": true, + "threshold": 180.0 + }, + { + "code": "OVERFIT_GUARD", + "message": "Training improvement with held-out regression is rejected as overfitting.", + "observed": { + "trainScoreDelta": 0.6666666666666667, + "validationPassRateDelta": 0.0, + "validationScoreDelta": 0.0 + }, + "passed": true, + "threshold": "train improvement cannot accompany held-out validation regression" + } + ], + "decision": "REJECT", + "reasons": [ + "VALIDATION_SCORE_DELTA_BELOW_MINIMUM", + "NEW_HARD_FAIL_BUDGET_EXCEEDED" + ] + }, + "inputs": { + "adapterIdentity": "{\"backend\":\"default:fake:evaluation:v1\",\"callback\":null,\"candidateGenerator\":\"default:fake:candidate-generator:v1\",\"mode\":\"fake\"}", + "environment": { + "platform": "Windows-11-10.0.22631-SP0", + "python": "3.12.7", + "sdk": "1.1.14" + }, + "hashes": { + "config": "fef5828244ea4f0058084dc427d5c42f60ffa004ee3e3ac38bee94ba08f0667e", + "effectiveConfig": "69cbd345d2e3868ab5baf39f73901fa51e137a59289bdebf6e10d31086d997e1", + "train": "af410bcc6f46c041a373efb1d1dda9f0f0f20258d1ef2146eac8a583cde99b19", + "validation": "5011756743a6211c78c75f2b7cbc435e98cf4caf0e0c484bbac9c5f03951c7d9" + }, + "auditHashes": { + "train": "af410bcc6f46c041a373efb1d1dda9f0f0f20258d1ef2146eac8a583cde99b19", + "validation": "5011756743a6211c78c75f2b7cbc435e98cf4caf0e0c484bbac9c5f03951c7d9" + }, + "paths": { + "config": "optimizer.json", + "train": "train.evalset.json", + "validation": "val.evalset.json" + }, + "seed": 42 + }, + "mode": "fake", + "optimization": { + "innerSplit": { + "selectionCaseIds": [ + "train_improve_beta" + ], + "selectionHash": "ec02d7b1c9840580943cbed756a0d3bc8e0185adf2b2fc7009ad70d61aaca92a", + "selectionPath": "inner_selection.evalset.json", + "auditHashes": { + "selection": "ec02d7b1c9840580943cbed756a0d3bc8e0185adf2b2fc7009ad70d61aaca92a", + "train": "70f85fca6ab8ece3ad0e3a3b1c3a6635eab921bf29efa4ed224b22513e9d9cd2" + }, + "trainCaseIds": [ + "train_improve_alpha", + "train_stable" + ], + "trainHash": "70f85fca6ab8ece3ad0e3a3b1c3a6635eab921bf29efa4ed224b22513e9d9cd2", + "trainPath": "inner_train.evalset.json" + }, + "proposal": { + "algorithm": "deterministic_failure_rewrite", + "baselinePrompts": { + "system": "You are a deterministic response agent.\n\nReturn the requested answer exactly. Do not add commentary.\n" + }, + "changed": true, + "costSources": [ + { + "costUsd": 0.0, + "metricCalls": 0, + "modelCalls": 0, + "name": "deterministic", + "tokenUsage": {}, + "upperBound": false + } + ], + "durationSeconds": 0.0, + "prompts": { + "system": "You are a deterministic response agent.\n\nReturn the requested answer exactly. Do not add commentary.\n\nBEHAVIOR_PROFILE=precision-v2\n" + }, + "rounds": [ + { + "acceptanceReason": "deterministic failure-attribution rewrite", + "accepted": true, + "candidatePrompts": { + "system": "You are a deterministic response agent.\n\nReturn the requested answer exactly. Do not add commentary.\n\nBEHAVIOR_PROFILE=precision-v2\n" + }, + "costUsd": 0.0, + "durationSeconds": 0.0, + "errorMessage": null, + "kind": "deterministic", + "metricScores": {}, + "optimizedFields": [ + "system" + ], + "round": 1, + "score": 1.0, + "skipReason": null, + "tokenUsage": {} + } + ], + "status": "SUCCEEDED", + "stopReason": "completed" + } + }, + "prompts": { + "baseline": { + "system": "You are a deterministic response agent.\n\nReturn the requested answer exactly. Do not add commentary.\n" + }, + "baselineHashes": { + "system": "aacc5210d0fe4482263f69f5f83241cdf9252e44be22a70de08107daec05e639" + }, + "candidate": { + "system": "You are a deterministic response agent.\n\nReturn the requested answer exactly. Do not add commentary.\n\nBEHAVIOR_PROFILE=precision-v2\n" + }, + "candidateHashes": { + "system": "ed2dfc103a7763a126f690e536572c5d06d20e2119ef7ef707c0a72deb162be2" + } + }, + "reproducibility": { + "command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --config examples/optimization/eval_optimize_loop/optimizer.json --train examples/optimization/eval_optimize_loop/train.evalset.json --validation examples/optimization/eval_optimize_loop/val.evalset.json --run-id sample-output-v1-replay --no-apply-candidate", + "gitCommit": "858b196edcfb7d10b8e7283625177ad72fbaf8f4", + "gitDirty": false, + "reason": null, + "reproducible": true + }, + "runId": "sample-output-v1", + "schemaVersion": "v2", + "sourceApplication": { + "applied": false, + "baselineHashes": { + "system": "aacc5210d0fe4482263f69f5f83241cdf9252e44be22a70de08107daec05e639" + }, + "finalHashes": { + "system": "aacc5210d0fe4482263f69f5f83241cdf9252e44be22a70de08107daec05e639" + }, + "requested": false + }, + "stage": "complete", + "startedAt": "2026-07-30T17:11:19.833935+00:00", + "status": "REJECT" +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md new file mode 100644 index 000000000..0372b238d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -0,0 +1,32 @@ +# Optimization Report: REJECT + +- Run ID: `sample-output-v1` +- Mode: `fake` +- Final stage: `complete` +- Duration: `0.390s` +- Reproducible: `yes` +- Source prompt applied: `no` + +## Regression + +| Split | Baseline score | Candidate score | Baseline pass rate | Candidate pass rate | +|---|---:|---:|---:|---:| +| train | 0.3333 | 1.0000 | 0.3333 | 1.0000 | +| validation | 0.6667 | 0.6667 | 0.6667 | 0.6667 | + +## Validation Transitions + +- `validation_improve`: `NEW_PASS` (+1.0000) +- `validation_regress`: `NEW_FAIL` (-1.0000) +- `validation_stable`: `UNCHANGED` (+0.0000) + +## Gate + +- Decision: `REJECT` +- Failed checks: `VALIDATION_SCORE_DELTA, NO_NEW_HARD_FAIL` +- Reasons: `VALIDATION_SCORE_DELTA_BELOW_MINIMUM, NEW_HARD_FAIL_BUDGET_EXCEEDED` +- Overfit detected: `no` + +## Cost + +- Total: `$0.000000` diff --git a/examples/optimization/eval_optimize_loop/traces/trace_cases.json b/examples/optimization/eval_optimize_loop/traces/trace_cases.json new file mode 100644 index 000000000..a20db2e40 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/traces/trace_cases.json @@ -0,0 +1,105 @@ +{ + "schemaVersion": "v1", + "datasetHashes": { + "train": "af410bcc6f46c041a373efb1d1dda9f0f0f20258d1ef2146eac8a583cde99b19", + "validation": "5011756743a6211c78c75f2b7cbc435e98cf4caf0e0c484bbac9c5f03951c7d9" + }, + "phases": { + "baseline": { + "train": { + "train_improve_alpha": [ + { + "invocationId": "trace-baseline-train-1", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:ALPHA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "INCORRECT"}]} + } + ], + "train_improve_beta": [ + { + "invocationId": "trace-baseline-train-2", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:BETA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "INCORRECT"}]} + } + ], + "train_stable": [ + { + "invocationId": "trace-baseline-train-3", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:stable; ANSWER:ELEVEN"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "ELEVEN"}]} + } + ] + }, + "validation": { + "validation_improve": [ + { + "invocationId": "trace-baseline-validation-1", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:GAMMA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "INCORRECT"}]} + } + ], + "validation_regress": [ + { + "invocationId": "trace-baseline-validation-2", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:regress; ANSWER:DELTA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "DELTA"}]} + } + ], + "validation_stable": [ + { + "invocationId": "trace-baseline-validation-3", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:stable; ANSWER:OMEGA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "OMEGA"}]} + } + ] + } + }, + "candidate": { + "train": { + "train_improve_alpha": [ + { + "invocationId": "trace-candidate-train-1", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:ALPHA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "ALPHA"}]} + } + ], + "train_improve_beta": [ + { + "invocationId": "trace-candidate-train-2", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:BETA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "BETA"}]} + } + ], + "train_stable": [ + { + "invocationId": "trace-candidate-train-3", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:stable; ANSWER:ELEVEN"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "ELEVEN"}]} + } + ] + }, + "validation": { + "validation_improve": [ + { + "invocationId": "trace-candidate-validation-1", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:improve; ANSWER:GAMMA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "GAMMA"}]} + } + ], + "validation_regress": [ + { + "invocationId": "trace-candidate-validation-2", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:regress; ANSWER:DELTA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "INCORRECT"}]} + } + ], + "validation_stable": [ + { + "invocationId": "trace-candidate-validation-3", + "userContent": {"role": "user", "parts": [{"text": "BEHAVIOR:stable; ANSWER:OMEGA"}]}, + "finalResponse": {"role": "model", "parts": [{"text": "OMEGA"}]} + } + ] + } + } + } +} diff --git a/examples/optimization/eval_optimize_loop/train.evalset.json b/examples/optimization/eval_optimize_loop/train.evalset.json new file mode 100644 index 000000000..092ffb140 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/train.evalset.json @@ -0,0 +1,57 @@ +{ + "evalSetId": "eval_optimize_loop_train", + "name": "Deterministic inner optimization training cases", + "evalCases": [ + { + "evalId": "train_improve_alpha", + "conversation": [ + { + "invocationId": "train-input-1", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:improve; ANSWER:ALPHA"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "ALPHA"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + }, + { + "evalId": "train_improve_beta", + "conversation": [ + { + "invocationId": "train-input-2", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:improve; ANSWER:BETA"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "BETA"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + }, + { + "evalId": "train_stable", + "conversation": [ + { + "invocationId": "train-input-3", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:stable; ANSWER:ELEVEN"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "ELEVEN"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/val.evalset.json b/examples/optimization/eval_optimize_loop/val.evalset.json new file mode 100644 index 000000000..d3830e19b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/val.evalset.json @@ -0,0 +1,57 @@ +{ + "evalSetId": "eval_optimize_loop_validation", + "name": "Held-out deterministic regression cases", + "evalCases": [ + { + "evalId": "validation_improve", + "conversation": [ + { + "invocationId": "validation-input-1", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:improve; ANSWER:GAMMA"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "GAMMA"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + }, + { + "evalId": "validation_regress", + "conversation": [ + { + "invocationId": "validation-input-2", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:regress; ANSWER:DELTA"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "DELTA"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + }, + { + "evalId": "validation_stable", + "conversation": [ + { + "invocationId": "validation-input-3", + "userContent": { + "role": "user", + "parts": [{"text": "BEHAVIOR:stable; ANSWER:OMEGA"}] + }, + "finalResponse": { + "role": "model", + "parts": [{"text": "OMEGA"}] + } + } + ], + "sessionInput": {"appName": "eval_optimize_loop", "userId": "example", "state": {}} + } + ] +} diff --git a/tests/examples/eval_optimize_loop/test_attribution_gate.py b/tests/examples/eval_optimize_loop/test_attribution_gate.py new file mode 100644 index 000000000..58fc986b5 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_attribution_gate.py @@ -0,0 +1,497 @@ +"""Deterministic attribution and gate policy tests.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from examples.optimization.eval_optimize_loop.pipeline.attribution import ( + attribute_failures, + select_attribution, +) +from examples.optimization.eval_optimize_loop.pipeline.configuration import ( + AttributionConfig, + GateConfig, +) +from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate +from examples.optimization.eval_optimize_loop.pipeline.models import ( + CandidateProposal, + CandidateRound, + CaseComparison, + CaseRun, + CaseSnapshot, + ComparisonSnapshot, + CostSource, + CostSummary, + Decision, + EvaluationSnapshot, + AttributionSnapshot, + AttributionStatistics, + FailureCategory, + MetricDelta, + MetricRun, + MetricSnapshot, + Phase, + RubricOutcome, + Split, + Transition, +) + +_MISSING = object() + + +def _invocation(tool: str | None = None, args: dict | None = None) -> dict: + payload = { + "invocationId": "ignored", + "userContent": { + "role": "user", + "parts": [{ + "text": "query" + }] + }, + "finalResponse": { + "role": "model", + "parts": [{ + "text": "answer" + }] + }, + } + if tool: + payload["intermediateData"] = {"toolUses": [{"name": tool, "args": args or {}}]} + return payload + + +def _failed_snapshot( + metric_name: str = "custom", + *, + reason: str | None = "unmapped failure", + error: str | None = None, + actual: dict | None = None, + expected: dict | None | object = _MISSING, + rubrics: tuple[RubricOutcome, ...] = (), +) -> EvaluationSnapshot: + metric = MetricRun( + metric_name=metric_name, + score=0, + threshold=1, + passed=False, + reason=reason, + rubrics=rubrics, + ) + run = CaseRun( + run_id=1, + passed=False, + metrics=(metric, ), + error=error, + trace=({ + "actual": actual or _invocation(), + "expected": _invocation() if expected is _MISSING else expected, + }, ), + ) + case = CaseSnapshot( + case_id="case", + passed=False, + score=0, + metrics=(MetricSnapshot(metric_name=metric_name, score=0, threshold=1, passed=False), ), + runs=(run, ), + error=error, + ) + return EvaluationSnapshot( + split=Split.TRAIN, + phase=Phase.BASELINE, + dataset_score=0, + pass_rate=0, + metric_scores={metric_name: 0}, + case_ids=("case", ), + cases=(case, ), + ) + + +_ATTRIBUTION_BENCHMARK = ( + (_failed_snapshot(error="backend failed"), AttributionConfig(), FailureCategory.EVALUATION_ERROR), + ( + _failed_snapshot(actual=_invocation("search"), expected=_invocation("calculate")), + AttributionConfig(), + FailureCategory.TOOL_CALL_ERROR, + ), + ( + _failed_snapshot( + actual=_invocation("search", {"q": "wrong"}), + expected=_invocation("search", {"q": "right"}), + ), + AttributionConfig(), + FailureCategory.TOOL_ARGUMENT_ERROR, + ), + ( + _failed_snapshot("llm_rubric_knowledge_recall"), + AttributionConfig(), + FailureCategory.KNOWLEDGE_RECALL_INSUFFICIENT, + ), + ( + _failed_snapshot("format_metric"), + AttributionConfig(metric_categories={"format_metric": FailureCategory.FORMAT_VIOLATION}), + FailureCategory.FORMAT_VIOLATION, + ), + ( + _failed_snapshot("llm_rubric_response"), + AttributionConfig(), + FailureCategory.LLM_RUBRIC_NOT_MET, + ), + ( + _failed_snapshot("final_response_avg_score"), + AttributionConfig(), + FailureCategory.FINAL_RESPONSE_MISMATCH, + ), + (_failed_snapshot(), AttributionConfig(), FailureCategory.UNKNOWN), +) + + +@pytest.mark.parametrize( + ("snapshot", "config", "expected"), + _ATTRIBUTION_BENCHMARK, +) +def test_attribution_taxonomy(snapshot, config, expected) -> None: + result = attribute_failures(snapshot, config, max_text_chars=200) + assert result.failures[0].primary == expected + assert result.failures[0].reasons + assert result.failures[0].evidence + assert result.statistics.total_failures == 1 + assert result.statistics.primary_category_counts == {expected: 1} + + +def test_attribution_statistics_cannot_diverge_from_failure_facts() -> None: + result = attribute_failures( + _failed_snapshot("final_response_avg_score"), + AttributionConfig(), + max_text_chars=200, + ) + with pytest.raises(ValidationError, match="statistics total"): + AttributionSnapshot( + split=result.split, + phase=result.phase, + failures=result.failures, + statistics=AttributionStatistics(), + ) + + +def test_select_attribution_recalculates_statistics() -> None: + result = attribute_failures( + _failed_snapshot("final_response_avg_score"), + AttributionConfig(), + max_text_chars=200, + ) + selected = select_attribution(result, set()) + assert selected.failures == () + assert selected.statistics == AttributionStatistics() + + +def test_attribution_policy_matrix_matches_every_expected_category() -> None: + assert all( + attribute_failures(snapshot, config, max_text_chars=200).failures[0].primary == expected + for snapshot, config, expected in _ATTRIBUTION_BENCHMARK) + + +def test_reference_free_failure_uses_metric_attribution_without_expected_trace() -> None: + result = attribute_failures( + _failed_snapshot("llm_rubric_response", expected=None), + AttributionConfig(), + max_text_chars=200, + ) + assert result.failures[0].primary == FailureCategory.LLM_RUBRIC_NOT_MET + + +@pytest.mark.parametrize( + ("reason", "expected"), + ( + ("response violates required JSON format schema", FailureCategory.FORMAT_VIOLATION), + ("knowledge retrieval recall is insufficient", FailureCategory.KNOWLEDGE_RECALL_INSUFFICIENT), + ("final answer mismatch", FailureCategory.FINAL_RESPONSE_MISMATCH), + ), +) +def test_reason_evidence_disambiguates_failures_from_the_same_metric(reason, expected) -> None: + result = attribute_failures( + _failed_snapshot("final_response_avg_score", reason=reason), + AttributionConfig(), + max_text_chars=200, + ) + assert result.failures[0].primary == expected + + +def test_tool_difference_evidence_is_recursively_sanitized() -> None: + result = attribute_failures( + _failed_snapshot( + actual=_invocation("search", {"apiKey": "actual-tool-secret"}), + expected=_invocation("search", {"apiKey": "expected-tool-secret"}), + ), + AttributionConfig(), + max_text_chars=1000, + ) + evidence = "\n".join(result.failures[0].evidence) + assert "actual-tool-secret" not in evidence + assert "expected-tool-secret" not in evidence + assert "[REDACTED]" in evidence + + +def test_structured_argument_difference_outranks_generic_tool_metric() -> None: + result = attribute_failures( + _failed_snapshot( + "tool_trajectory_avg_score", + actual=_invocation("search", {"q": "wrong"}), + expected=_invocation("search", {"q": "right"}), + ), + AttributionConfig(), + max_text_chars=1000, + ) + assert result.failures[0].primary == FailureCategory.TOOL_ARGUMENT_ERROR + + +def test_attribution_uses_only_failed_rubric_outcomes() -> None: + snapshot = _failed_snapshot( + "llm_rubric_response", + rubrics=( + RubricOutcome(id="passed_rule", score=1, passed=True), + RubricOutcome(id="failed_rule", score=0, passed=False), + ), + ) + config = AttributionConfig(rubric_categories={ + "passed_rule": FailureCategory.TOOL_CALL_ERROR, + "failed_rule": FailureCategory.FORMAT_VIOLATION, + }) + + result = attribute_failures(snapshot, config, max_text_chars=200) + + failure = result.failures[0] + assert failure.primary == FailureCategory.FORMAT_VIOLATION + assert failure.trigger_rubrics == ("failed_rule", ) + + +def _comparison( + split: Split, + *, + score_delta: float = 0.1, + pass_delta: float = 1.0, + critical: bool = False, + hard: bool = False, +) -> ComparisonSnapshot: + if pass_delta > 0: + baseline_passed, candidate_passed = False, True + transition = Transition.NEW_PASS + elif pass_delta < 0: + baseline_passed, candidate_passed = True, False + transition = Transition.NEW_FAIL + else: + baseline_passed = candidate_passed = True + transition = (Transition.IMPROVED + if score_delta > 0 else Transition.REGRESSED if score_delta < 0 else Transition.UNCHANGED) + return ComparisonSnapshot( + split=split, + score_delta=score_delta, + pass_rate_delta=pass_delta, + metric_deltas={"quality": score_delta}, + cases=(CaseComparison( + case_id="case", + baseline_passed=baseline_passed, + candidate_passed=candidate_passed, + baseline_score=0, + candidate_score=score_delta, + delta=score_delta, + metrics=(MetricDelta( + metric_name="quality", + baseline=0, + candidate=score_delta, + delta=score_delta, + ), ), + transition=transition, + critical=critical, + hard=hard, + new_hard_failure=hard and baseline_passed and not candidate_passed, + ), ), + ) + + +def _candidate() -> CandidateProposal: + return CandidateProposal( + algorithm="fake", + baseline_prompts={"system": "old"}, + prompts={"system": "new"}, + changed=True, + rounds=(CandidateRound( + round=1, + candidate_prompts={"system": "new"}, + accepted=True, + score=1, + ), ), + ) + + +def _gate(**overrides): + values = { + "train": _comparison(Split.TRAIN), + "validation": _comparison(Split.VALIDATION), + "candidate": _candidate(), + "cost": CostSummary( + sources=(CostSource(name="fake", cost_usd=0), ), + total_cost_usd=0, + ), + "duration_seconds": 1, + "baseline_prompt_hashes": { + "system": "old" + }, + "candidate_prompt_hashes": { + "system": "new" + }, + "config": GateConfig(metric_max_regression={"quality": 0}), + } + values.update(overrides) + return evaluate_gate(**values) + + +def test_gate_accepts_only_when_checks_pass_in_fixed_order() -> None: + result = _gate() + assert result.decision == Decision.ACCEPT + assert [check.code for check in result.checks] == [ + "REPORT_COMPLETE", + "CANDIDATE_CHANGED", + "VALIDATION_SCORE_DELTA", + "VALIDATION_PASS_RATE_DELTA", + "NO_NEW_HARD_FAIL", + "CRITICAL_CASE_NON_REGRESSION", + "METRIC_NON_REGRESSION", + "COST_BUDGET", + "DURATION_BUDGET", + "OVERFIT_GUARD", + ] + + +def test_gate_rejects_train_only_improvement_as_overfit() -> None: + validation = _comparison(Split.VALIDATION, score_delta=-0.1, pass_delta=0) + result = _gate(validation=validation) + assert result.decision == Decision.REJECT + assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in result.reasons + + +def test_overfit_guard_cannot_be_bypassed_by_permissive_thresholds() -> None: + validation = _comparison(Split.VALIDATION, score_delta=-0.1, pass_delta=0) + result = _gate( + validation=validation, + config=GateConfig( + min_validation_score_delta=-0.2, + min_validation_pass_rate_delta=-1, + metric_max_regression={"quality": 1}, + ), + ) + assert result.decision == Decision.REJECT + assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in result.reasons + + +def test_overfit_guard_includes_train_pass_rate_improvement() -> None: + train = _comparison(Split.TRAIN, score_delta=0, pass_delta=1) + validation = _comparison(Split.VALIDATION, score_delta=-0.1, pass_delta=0) + result = _gate( + train=train, + validation=validation, + config=GateConfig( + min_validation_score_delta=-0.2, + min_validation_pass_rate_delta=-1, + metric_max_regression={"quality": 1}, + ), + ) + assert result.decision == Decision.REJECT + assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" in result.reasons + + +def test_gate_fails_closed_when_enabled_cost_is_unknown() -> None: + cost = CostSummary( + sources=(CostSource(name="live-agent", cost_usd=None), ), + total_cost_usd=None, + ) + result = _gate(cost=cost, config=GateConfig(max_cost_usd=1, metric_max_regression={"quality": 0})) + assert result.decision == Decision.REJECT + assert "COST_UNAVAILABLE" in result.reasons + + +def test_gate_returns_error_for_missing_configured_metric() -> None: + result = _gate(config=GateConfig(metric_max_regression={"missing": 0})) + assert result.decision == Decision.ERROR + assert result.reasons == ("GATE_INPUT_INVALID", ) + + +def test_gate_policy_matrix_matches_every_expected_decision() -> None: + unchanged = CandidateProposal( + algorithm="fake", + baseline_prompts={"system": "old"}, + prompts={"system": "old"}, + changed=False, + ) + unknown_cost = CostSummary( + sources=(CostSource(name="live", cost_usd=None), ), + total_cost_usd=None, + ) + scenarios = ( + (_gate(), Decision.ACCEPT), + (_gate(report_complete=False), Decision.REJECT), + ( + _gate( + candidate=unchanged, + candidate_prompt_hashes={"system": "old"}, + ), + Decision.REJECT, + ), + ( + _gate(config=GateConfig(min_validation_score_delta=0.2)), + Decision.REJECT, + ), + ( + _gate(config=GateConfig(min_validation_pass_rate_delta=1.1)), + Decision.REJECT, + ), + ( + _gate(validation=_comparison( + Split.VALIDATION, + score_delta=-0.1, + pass_delta=-1, + hard=True, + )), + Decision.REJECT, + ), + ( + _gate(validation=_comparison( + Split.VALIDATION, + score_delta=-0.1, + pass_delta=-1, + critical=True, + )), + Decision.REJECT, + ), + ( + _gate( + validation=_comparison(Split.VALIDATION, score_delta=-0.1, pass_delta=0), + config=GateConfig( + min_validation_score_delta=-1, + min_validation_pass_rate_delta=-1, + metric_max_regression={"quality": 0}, + ), + ), + Decision.REJECT, + ), + ( + _gate( + cost=unknown_cost, + config=GateConfig(max_cost_usd=1, metric_max_regression={"quality": 0}), + ), + Decision.REJECT, + ), + ( + _gate( + duration_seconds=2, + config=GateConfig(max_duration_seconds=1, metric_max_regression={"quality": 0}), + ), + Decision.REJECT, + ), + ( + _gate(config=GateConfig(metric_max_regression={"missing": 0})), + Decision.ERROR, + ), + ) + assert len(scenarios) >= 6 + assert all(actual.decision == expected for actual, expected in scenarios) diff --git a/tests/examples/eval_optimize_loop/test_backends.py b/tests/examples/eval_optimize_loop/test_backends.py new file mode 100644 index 000000000..d42728741 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_backends.py @@ -0,0 +1,1214 @@ +"""Backend and candidate-generator substitution contract tests.""" + +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from trpc_agent_sdk.evaluation import ( + AgentEvaluator, + EvalConfig, + EvalSet, + EvalStatus, + TargetPrompt, +) + +from examples.optimization.eval_optimize_loop.pipeline import backends as backends_module +from examples.optimization.eval_optimize_loop.pipeline import optimizer_worker as optimizer_worker_module +from examples.optimization.eval_optimize_loop.pipeline.backends import ( + DeterministicCandidateGenerator, + FakeEvaluationBackend, + LiveCandidateGenerator, + LiveEvaluationBackend, + TraceEvaluationBackend, +) +from examples.optimization.eval_optimize_loop.pipeline.evaluation import normalize_result +from examples.optimization.eval_optimize_loop.pipeline.live_adapter import ( + LiveAdapterSpec, + load_verified_callback, +) +from examples.optimization.eval_optimize_loop.pipeline.offline_evaluation import prepare_offline_evaluation +from examples.optimization.eval_optimize_loop.pipeline.models import ( + AttributionSnapshot, + Phase, + Split, +) + + +def _eval_set() -> EvalSet: + return EvalSet.model_validate({ + "evalSetId": + "backend_contract", + "evalCases": [{ + "evalId": + "case", + "conversation": [{ + "invocationId": "expected", + "userContent": { + "role": "user", + "parts": [{ + "text": "BEHAVIOR:stable; ANSWER:OK" + }], + }, + "finalResponse": { + "role": "model", + "parts": [{ + "text": "OK" + }] + }, + }], + "sessionInput": { + "appName": "backend", + "userId": "test", + "state": {} + }, + }], + }) + + +def _config() -> EvalConfig: + return EvalConfig( + metrics=[{ + "metric_name": "final_response_avg_score", + "threshold": 1, + "criterion": { + "final_response": { + "text": { + "match": "exact" + } + } + }, + }], + num_runs=1, + ) + + +async def importable_call_agent(query: str) -> str: + return query + + +def _live_adapter() -> LiveAdapterSpec: + return LiveAdapterSpec.resolve(f"{__name__}:importable_call_agent", importable_call_agent) + + +def test_worker_callback_verification_rejects_source_drift(tmp_path) -> None: + adapter = _live_adapter() + assert load_verified_callback( + adapter.import_path, + expected_source_path=str(adapter.source_path), + expected_source_sha256=adapter.source_sha256, + expected_callable_sha256=adapter.callable_sha256, + ) is importable_call_agent + with pytest.raises(ValueError, match="source path differs"): + load_verified_callback( + adapter.import_path, + expected_source_path=str(tmp_path / "shadowed.py"), + expected_source_sha256=adapter.source_sha256, + expected_callable_sha256=adapter.callable_sha256, + ) + with pytest.raises(ValueError, match="source hash differs"): + load_verified_callback( + adapter.import_path, + expected_source_path=str(adapter.source_path), + expected_source_sha256="0" * 64, + expected_callable_sha256=adapter.callable_sha256, + ) + with pytest.raises(ValueError, match="callback code differs"): + load_verified_callback( + adapter.import_path, + expected_source_path=str(adapter.source_path), + expected_source_sha256=adapter.source_sha256, + expected_callable_sha256="0" * 64, + ) + + +def _install_worker_result(monkeypatch, result, captured=None) -> None: + class CompletedProcess: + returncode = 0 + + async def wait(self): + return self.returncode + + async def fake_subprocess(*args, **kwargs): + request_path = Path(args[-1]) + request = json.loads(request_path.read_text(encoding="utf-8")) + output_dir = Path(request["outputDir"]) + output_dir.mkdir(parents=True) + (output_dir / "result.json").write_text("{}", encoding="utf-8") + if captured is not None: + captured.update({"args": args, "kwargs": kwargs, "request": request}) + return CompletedProcess() + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_subprocess) + monkeypatch.setattr( + backends_module, + "OptimizeResult", + SimpleNamespace(from_file=lambda path: result), + ) + + +@pytest.mark.asyncio +async def test_fake_backend_uses_public_services_and_does_not_mutate_inputs(tmp_path, monkeypatch) -> None: + backend = FakeEvaluationBackend() + eval_set = _eval_set() + config = _config() + before_set = eval_set.model_dump(mode="json") + before_config = config.model_dump(mode="json") + calls = 0 + original = AgentEvaluator.evaluate_eval_set + + async def recording_evaluate_eval_set(*args, **kwargs): + nonlocal calls + calls += 1 + return await original(*args, **kwargs) + + monkeypatch.setattr(AgentEvaluator, "evaluate_eval_set", recording_evaluate_eval_set) + raw = await backend.evaluate( + eval_set=eval_set, + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + assert snapshot.pass_rate == 1 + assert calls == 1 + assert eval_set.model_dump(mode="json") == before_set + assert config.model_dump(mode="json") == before_config + + +@pytest.mark.asyncio +async def test_fake_llm_metric_uses_deterministic_offline_substitute(tmp_path, monkeypatch) -> None: + + def forbid_real_judge(*args, **kwargs): + raise AssertionError("fake mode constructed a real LLM judge") + + monkeypatch.setattr( + AgentEvaluator, + "evaluate_eval_set", + lambda *args, **kwargs: pytest.fail("custom registry must remain run-local"), + ) + monkeypatch.setattr( + "trpc_agent_sdk.evaluation._llm_judge.LLMJudge.__init__", + forbid_real_judge, + ) + config = EvalConfig( + metrics=[ + { + "metric_name": "llm_final_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + } + } + }, + }, + { + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "response_quality", + "content": { + "text": "The response is correct." + }, + "type": "OFFLINE_RESPONSE_EXACT_REFERENCE", + }], + } + }, + }, + ], + num_runs=1, + ) + eval_set = _eval_set() + raw = await FakeEvaluationBackend().evaluate( + eval_set=eval_set, + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + assert snapshot.metric_scores == { + "llm_final_response": 1, + "llm_rubric_response": 1, + } + run_metrics = {metric.metric_name: metric for metric in snapshot.cases[0].runs[0].metrics} + assert [rubric.id for rubric in run_metrics["llm_rubric_response"].rubrics] == ["response_quality"] + + +def test_offline_rubric_empty_invocations_are_a_hard_failure() -> None: + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "response_quality", + "type": "OFFLINE_RESPONSE_NON_EMPTY", + }], + } + }, + }], + num_runs=1, + ) + offline_config, registry = prepare_offline_evaluation(config) + assert registry is not None + evaluator = registry.get_evaluator(offline_config.get_eval_metrics()[0]) + + result = evaluator.evaluate_invocations([], None) + + assert result.overall_score == 0.0 + assert result.overall_eval_status == EvalStatus.FAILED + assert result.per_invocation_results == [] + + +@pytest.mark.asyncio +async def test_fake_knowledge_recall_fails_before_black_box_inference(tmp_path, monkeypatch) -> None: + + def fail_if_called(*args, **kwargs): + raise AssertionError("black-box inference ran before capability validation") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.backends.deterministic_fake_response", + fail_if_called, + ) + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_knowledge_recall", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "knowledge_evidence", + "type": "OFFLINE_KNOWLEDGE_NON_EMPTY", + }], + } + }, + }], + num_runs=1, + ) + with pytest.raises(ValueError, match="does not support metrics"): + await FakeEvaluationBackend().evaluate( + eval_set=_eval_set(), + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + + +@pytest.mark.asyncio +async def test_offline_rubric_rejects_unstructured_natural_language(tmp_path) -> None: + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "quality", + "content": { + "text": "Be correct." + } + }], + } + }, + }], + num_runs=1, + ) + with pytest.raises(ValueError, match="requires an explicit type"): + await FakeEvaluationBackend().evaluate( + eval_set=_eval_set(), + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + + +@pytest.mark.asyncio +async def test_offline_equals_requires_explicit_content(tmp_path) -> None: + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "missing_operand", + "type": "OFFLINE_RESPONSE_EQUALS", + }], + } + }, + }], + num_runs=1, + ) + with pytest.raises(ValueError, match="requires content.text"): + await FakeEvaluationBackend().evaluate( + eval_set=_eval_set(), + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + + +@pytest.mark.asyncio +async def test_offline_exact_reference_preserves_empty_response(tmp_path) -> None: + eval_set = _eval_set() + invocation = eval_set.eval_cases[0].conversation[0] + invocation.user_content.parts[0].text = "BEHAVIOR:stable; ANSWER:" + invocation.final_response.parts[0].text = "" + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "empty_reference", + "type": "OFFLINE_RESPONSE_EXACT_REFERENCE", + }], + } + }, + }], + num_runs=1, + ) + raw = await FakeEvaluationBackend().evaluate( + eval_set=eval_set, + eval_config=config, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + assert snapshot.metric_scores == {"llm_rubric_response": 1} + + +@pytest.mark.asyncio +async def test_trace_reference_free_rubric_uses_placeholder_expected(tmp_path) -> None: + source = _eval_set().model_dump(mode="json", by_alias=True, exclude_none=True) + case = source["evalCases"][0] + actual = case.pop("conversation") + case["evalMode"] = "trace" + case["actualConversation"] = actual + eval_set = EvalSet.model_validate(source) + fixture = { + "schemaVersion": "v1", + "datasetHashes": { + "train": "train-hash", + "validation": "validation-hash" + }, + "phases": { + phase: { + "train": { + "case": actual + }, + "validation": { + "case": actual + }, + } + for phase in ("baseline", "candidate") + }, + } + fixture_path = tmp_path / "trace-reference-free.json" + fixture_path.write_text(json.dumps(fixture), encoding="utf-8") + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_response", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [ + { + "id": "reference_free_quality", + "content": { + "text": "OK" + }, + "type": "OFFLINE_RESPONSE_CONTAINS", + }, + { + "id": "missing_quality", + "content": { + "text": "MISSING" + }, + "type": "OFFLINE_RESPONSE_CONTAINS", + }, + ], + } + }, + }], + num_runs=1, + ) + raw = await TraceEvaluationBackend( + str(fixture_path), + { + "train": "train-hash", + "validation": "validation-hash" + }, + ).evaluate( + eval_set=eval_set, + eval_config=config, + prompts={"system": "unused"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + metric = snapshot.cases[0].runs[0].metrics[0] + assert metric.score == 0.5 + assert [(rubric.id, rubric.passed) for rubric in metric.rubrics] == [ + ("reference_free_quality", True), + ("missing_quality", False), + ] + assert snapshot.cases[0].runs[0].trace[0]["expected"] is None + + +@pytest.mark.asyncio +async def test_trace_knowledge_recall_scores_recorded_tool_evidence(tmp_path) -> None: + source = _eval_set().model_dump(mode="json", by_alias=True, exclude_none=True) + case = source["evalCases"][0] + actual = case.pop("conversation") + actual[0]["intermediateData"] = { + "toolResponses": [{ + "name": "knowledge_search", + "response": { + "text": "pinned source fact" + }, + }] + } + case["evalMode"] = "trace" + case["actualConversation"] = actual + eval_set = EvalSet.model_validate(source) + fixture = { + "schemaVersion": "v1", + "datasetHashes": { + "train": "train-hash", + "validation": "validation-hash" + }, + "phases": { + phase: { + "train": { + "case": actual + }, + "validation": { + "case": actual + } + } + for phase in ("baseline", "candidate") + }, + } + fixture_path = tmp_path / "trace-knowledge.json" + fixture_path.write_text(json.dumps(fixture), encoding="utf-8") + config = EvalConfig( + metrics=[{ + "metric_name": "llm_rubric_knowledge_recall", + "threshold": 1, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "must-not-load" + }, + "rubrics": [{ + "id": "source_fact", + "content": { + "text": "pinned source fact" + }, + "type": "OFFLINE_KNOWLEDGE_CONTAINS", + }], + } + }, + }], + num_runs=1, + ) + raw = await TraceEvaluationBackend( + str(fixture_path), + { + "train": "train-hash", + "validation": "validation-hash" + }, + ).evaluate( + eval_set=eval_set, + eval_config=config, + prompts={"system": "unused"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + metric = snapshot.cases[0].runs[0].metrics[0] + assert metric.score == 1 + assert [(rubric.id, rubric.passed) for rubric in metric.rubrics] == [("source_fact", True)] + + +@pytest.mark.asyncio +async def test_trace_backend_replays_fixture_without_agent(tmp_path) -> None: + eval_set = _eval_set() + actual = eval_set.eval_cases[0].conversation[0].model_dump(mode="json", by_alias=True) + fixture = { + "schemaVersion": "v1", + "datasetHashes": { + "train": "train-hash", + "validation": "validation-hash" + }, + "phases": { + phase: { + "train": { + "case": [actual] + }, + "validation": { + "case": [actual] + }, + } + for phase in ("baseline", "candidate") + }, + } + fixture_path = tmp_path / "trace.json" + fixture_path.write_text(__import__("json").dumps(fixture), encoding="utf-8") + backend = TraceEvaluationBackend( + str(fixture_path), + { + "train": "train-hash", + "validation": "validation-hash" + }, + ) + fixture["phases"]["baseline"]["train"]["case"][0]["finalResponse"] = { + "role": "model", + "parts": [{ + "text": "CHANGED_AFTER_PREFLIGHT" + }], + } + fixture_path.write_text(json.dumps(fixture), encoding="utf-8") + raw = await backend.evaluate( + eval_set=eval_set, + eval_config=_config(), + prompts={"system": "unused"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + eval_set, + _config(), + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + assert snapshot.pass_rate == 1 + + +def test_live_backend_requires_async_callback() -> None: + with pytest.raises(TypeError, match="async"): + LiveEvaluationBackend(lambda query: query) # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_live_backend_uses_agent_evaluator_entrypoint(tmp_path, monkeypatch) -> None: + calls = 0 + original = AgentEvaluator.evaluate_eval_set + + async def recording_evaluate_eval_set(*args, **kwargs): + nonlocal calls + calls += 1 + return await original(*args, **kwargs) + + async def call_agent(query: str) -> str: + del query + return "OK" + + monkeypatch.setattr(AgentEvaluator, "evaluate_eval_set", recording_evaluate_eval_set) + raw = await LiveEvaluationBackend(call_agent).evaluate( + eval_set=_eval_set(), + eval_config=_config(), + prompts={"system": "baseline"}, + split=Split.VALIDATION, + phase=Phase.BASELINE, + audit_dir=str(tmp_path), + ) + snapshot = normalize_result( + raw, + _eval_set(), + _config(), + split=Split.VALIDATION, + phase=Phase.BASELINE, + ) + assert snapshot.pass_rate == 1 + assert calls == 1 + + +@pytest.mark.asyncio +async def test_deterministic_generator_only_uses_failure_facts(tmp_path) -> None: + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + target = TargetPrompt().add_path("system", str(prompt_path)) + attribution = AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()) + proposal = await DeterministicCandidateGenerator().generate( + target_prompt=target, + baseline_prompts={"system": "baseline"}, + train_attribution=attribution, + inner_train_path="inner-train", + inner_selection_path="inner-selection", + config_path="config", + output_dir="output", + ) + assert proposal.changed is False + + +@pytest.mark.asyncio +async def test_live_generator_forces_update_source_false(monkeypatch, tmp_path) -> None: + async def call_agent(query: str) -> str: + return query + + captured = {} + + async def fake_optimize(**kwargs): + captured.update(kwargs) + return SimpleNamespace(dump_to=lambda path: Path(path).write_text("{}", encoding="utf-8")) + + monkeypatch.setattr( + optimizer_worker_module, + "load_verified_callback", + lambda spec, **kwargs: call_agent, + ) + monkeypatch.setattr(optimizer_worker_module.AgentOptimizer, "optimize", fake_optimize) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + request_path = tmp_path / "request.json" + request_path.write_text( + json.dumps({ + "callbackSpec": _live_adapter().import_path, + "callbackSourcePath": str(_live_adapter().source_path), + "callbackSourceSha256": _live_adapter().source_sha256, + "callbackCallableSha256": _live_adapter().callable_sha256, + "promptPaths": { + "system": str(prompt_path) + }, + "configPath": "optimizer.json", + "trainPath": "inner-train.json", + "validationPath": "inner-selection.json", + "outputDir": str(tmp_path / "optimizer-output"), + "verbose": 0, + }), + encoding="utf-8", + ) + await optimizer_worker_module._run(request_path) + assert captured["update_source"] is False + assert captured["validation_dataset_path"] == "inner-selection.json" + + +@pytest.mark.parametrize("error_type", (KeyboardInterrupt, SystemExit)) +def test_optimizer_worker_rethrows_process_control(monkeypatch, tmp_path, error_type) -> None: + output_dir = tmp_path / "optimizer-output" + request_path = tmp_path / "request.json" + request_path.write_text( + json.dumps({ + "outputDir": str(output_dir) + }), + encoding="utf-8", + ) + control = error_type("stop worker") + + async def fail(_request_path): + raise control + + monkeypatch.setattr(optimizer_worker_module, "_run", fail) + monkeypatch.setattr(sys, "argv", ["optimizer_worker", str(request_path)]) + + with pytest.raises(error_type) as raised: + optimizer_worker_module.main() + + assert raised.value is control + assert not (output_dir / "worker_error.json").exists() + + +def test_optimizer_worker_error_artifact_is_bounded_and_redacted(monkeypatch, tmp_path) -> None: + output_dir = tmp_path / "optimizer-output" + request_path = tmp_path / "request.json" + request_path.write_text( + json.dumps({ + "outputDir": str(output_dir) + }), + encoding="utf-8", + ) + + async def fail(_request_path): + raise RuntimeError( + "Authorization: Bearer worker-token; api_key=worker-key; " + "context" * 1000) + + monkeypatch.setattr(optimizer_worker_module, "_run", fail) + monkeypatch.setattr(sys, "argv", ["optimizer_worker", str(request_path)]) + + assert optimizer_worker_module.main() == 1 + payload = json.loads((output_dir / "worker_error.json").read_text(encoding="utf-8")) + assert payload["errorType"] == "RuntimeError" + assert len(payload["message"]) <= 4000 + assert "worker-token" not in payload["message"] + assert "worker-key" not in payload["message"] + assert payload["message"].count("[REDACTED]") == 2 + + +@pytest.mark.asyncio +async def test_live_generator_accepts_sdk_skipped_round(monkeypatch, tmp_path) -> None: + async def fake_optimize(**kwargs): + return SimpleNamespace( + status="SUCCEEDED", + error_message="", + rounds=[ + SimpleNamespace( + round=1, + candidate_prompts={}, + accepted=False, + validation_pass_rate=0.0, + kind="reflective", + optimized_field_names=[], + metric_breakdown={}, + acceptance_reason="no candidate produced this round", + skip_reason="reflect-LM produced no usable new prompt", + error_message=None, + round_llm_cost=0.02, + round_token_usage={"total": 4}, + duration_seconds=0.3, + ) + ], + algorithm="gepa_reflective", + baseline_prompts={"system": "baseline"}, + best_prompts={"system": "baseline"}, + stop_reason="completed", + total_reflection_lm_calls=1, + total_llm_cost=0.02, + total_token_usage={"total": 4}, + duration_seconds=0.4, + ) + + _install_worker_result(monkeypatch, await fake_optimize()) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + proposal = await LiveCandidateGenerator(_live_adapter()).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + ) + assert proposal.changed is False + assert proposal.rounds[0].candidate_prompts == {} + assert proposal.rounds[0].skip_reason == "reflect-LM produced no usable new prompt" + assert proposal.rounds[0].score is None + assert proposal.rounds[0].cost_usd == 0.02 + + +@pytest.mark.asyncio +async def test_live_generator_preserves_failed_sdk_result_facts(monkeypatch, tmp_path) -> None: + result = SimpleNamespace( + status="FAILED", + error_message="optimizer request failed", + rounds=[SimpleNamespace( + round=1, + candidate_prompts={}, + accepted=False, + validation_pass_rate=0.0, + kind="reflective", + optimized_field_names=[], + metric_breakdown={}, + acceptance_reason="", + skip_reason=None, + error_message="reflection request failed", + round_llm_cost=0.04, + round_token_usage={"total": 6}, + duration_seconds=0.2, + )], + algorithm="gepa_reflective", + baseline_prompts={"system": "baseline"}, + best_prompts={"system": "baseline"}, + stop_reason=None, + total_reflection_lm_calls=1, + total_llm_cost=0.04, + total_token_usage={"total": 6}, + duration_seconds=0.3, + ) + _install_worker_result(monkeypatch, result) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + + proposal = await LiveCandidateGenerator(_live_adapter()).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + ) + + assert proposal.status == "FAILED" + assert proposal.error_message == "optimizer request failed" + assert proposal.rounds[0].error_message == "reflection request failed" + assert proposal.cost_sources[0].cost_usd == 0.04 + assert proposal.duration_seconds == 0.3 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("reported_baseline", "best_prompts", "expected_error"), + ( + ({"system": "stale"}, {"system": "candidate"}, "baseline prompts differ"), + ({"system": "baseline"}, {}, "best prompt keys differ"), + ), +) +async def test_live_generator_rejects_invalid_success_contract( + monkeypatch, + tmp_path, + reported_baseline, + best_prompts, + expected_error, +) -> None: + result = SimpleNamespace( + status="SUCCEEDED", + error_message="", + rounds=[SimpleNamespace( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=True, + validation_pass_rate=0.9, + kind="reflective", + optimized_field_names=["system"], + metric_breakdown={"quality": 0.9}, + acceptance_reason="improved", + skip_reason=None, + error_message=None, + round_llm_cost=0.02, + round_token_usage={"total": 4}, + duration_seconds=0.1, + )], + algorithm="gepa_reflective", + baseline_prompts=reported_baseline, + best_prompts=best_prompts, + stop_reason="completed", + total_reflection_lm_calls=1, + total_llm_cost=0.02, + total_token_usage={"total": 4}, + duration_seconds=0.2, + ) + _install_worker_result(monkeypatch, result) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + + proposal = await LiveCandidateGenerator(_live_adapter()).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + ) + + assert proposal.status == "SUCCEEDED" + assert expected_error in proposal.adapter_error + assert proposal.prompts == {"system": "baseline"} + assert proposal.changed is False + assert proposal.rounds[0].cost_usd == 0.02 + + +@pytest.mark.asyncio +async def test_live_worker_is_reaped_after_repeated_parent_cancellation(monkeypatch, tmp_path) -> None: + class StuckProcess: + + def __init__(self) -> None: + self.returncode = None + self.waiting = asyncio.Event() + self.cleanup_waiting = asyncio.Event() + self.finished = asyncio.Event() + self.wait_calls = 0 + self.terminated = False + self.killed = False + + async def wait(self): + self.wait_calls += 1 + self.waiting.set() + if self.wait_calls >= 2: + self.cleanup_waiting.set() + await self.finished.wait() + return self.returncode + + def terminate(self): + self.terminated = True + self.returncode = -15 + self.finished.set() + + def kill(self): + self.killed = True + self.returncode = -9 + self.finished.set() + + process = StuckProcess() + + async def fake_subprocess(*args, **kwargs): + return process + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_subprocess) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + output_dir = tmp_path / "optimizer-output" + task = asyncio.create_task( + LiveCandidateGenerator( + _live_adapter(), + shutdown_timeout_seconds=0.1, + ).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(output_dir), + )) + await process.waiting.wait() + task.cancel() + await process.cleanup_waiting.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError) as raised: + await task + assert (output_dir / "optimize.stop").read_text(encoding="utf-8") == "cancel requested\n" + assert process.terminated is True + assert process.returncode in {-15, -9} + assert any("additional cancellation" in note for note in raised.value.__notes__) + + +@pytest.mark.asyncio +async def test_live_worker_cleanup_escalates_after_wait_and_terminate_errors(monkeypatch, tmp_path) -> None: + class KillOnlyProcess: + + def __init__(self) -> None: + self.returncode = None + self.waiting = asyncio.Event() + self.finished = asyncio.Event() + self.wait_calls = 0 + self.killed = False + + async def wait(self): + self.wait_calls += 1 + self.waiting.set() + if self.wait_calls == 2: + raise OSError("Authorization: Bearer wait-secret") + await self.finished.wait() + return self.returncode + + def terminate(self): + raise PermissionError("api_key=terminate-secret") + + def kill(self): + self.killed = True + self.returncode = -9 + self.finished.set() + + process = KillOnlyProcess() + + async def fake_subprocess(*args, **kwargs): + return process + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_subprocess) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + task = asyncio.create_task( + LiveCandidateGenerator( + _live_adapter(), + shutdown_timeout_seconds=0.01, + ).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + )) + await process.waiting.wait() + task.cancel() + + with pytest.raises(asyncio.CancelledError) as raised: + await task + + diagnostics = "\n".join(raised.value.__notes__) + assert process.killed is True + assert process.returncode == -9 + assert "wait-secret" not in diagnostics + assert "terminate-secret" not in diagnostics + assert diagnostics.count("[REDACTED]") == 2 + + +@pytest.mark.asyncio +async def test_live_worker_uses_sanitized_stderr_when_error_artifact_is_missing(monkeypatch, tmp_path) -> None: + stderr = asyncio.StreamReader() + stderr.feed_data(b"".join(( + b"old-diagnostic\nAuthorization: Bearer ", + b"boundary-secret-" * 400, + b"\nImportError: failed before worker main; Authorization: Bearer startup-secret\n", + ))) + stderr.feed_eof() + + class FailedProcess: + returncode = 1 + + def __init__(self) -> None: + self.stderr = stderr + + async def wait(self): + return self.returncode + + captured = {} + + async def fake_subprocess(*args, **kwargs): + captured.update(kwargs) + return FailedProcess() + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_subprocess) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + + with pytest.raises(RuntimeError) as raised: + await LiveCandidateGenerator(_live_adapter()).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + ) + + message = str(raised.value) + assert captured["stderr"] is asyncio.subprocess.PIPE + assert "failed without an error artifact" in message + assert "ImportError: failed before worker main" in message + assert "startup-secret" not in message + assert "boundary-secret" not in message + assert "[REDACTED]" in message + assert "old-diagnostic" not in message + assert len(message) < 4200 + + +@pytest.mark.asyncio +async def test_live_worker_success_is_loaded_through_the_strict_candidate_adapter(monkeypatch, tmp_path) -> None: + round_ = SimpleNamespace( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=True, + validation_pass_rate=0.8, + kind="reflective", + optimized_field_names=["system"], + metric_breakdown={"quality": 0.8}, + acceptance_reason="improved selection score", + skip_reason=None, + error_message=None, + round_llm_cost=0.03, + round_token_usage={"total": 8}, + duration_seconds=0.2, + ) + result = SimpleNamespace( + status="SUCCEEDED", + error_message="", + rounds=[round_], + algorithm="gepa_reflective", + baseline_prompts={"system": "baseline"}, + best_prompts={"system": "candidate"}, + stop_reason="completed", + total_reflection_lm_calls=1, + total_llm_cost=0.03, + total_token_usage={"total": 8}, + duration_seconds=0.2, + ) + captured = {} + _install_worker_result(monkeypatch, result, captured) + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + proposal = await LiveCandidateGenerator(_live_adapter()).generate( + target_prompt=TargetPrompt().add_path("system", str(prompt_path)), + baseline_prompts={"system": "baseline"}, + train_attribution=AttributionSnapshot(split=Split.TRAIN, phase=Phase.BASELINE, failures=()), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + config_path="optimizer.json", + output_dir=str(tmp_path / "optimizer-output"), + ) + assert captured["args"][1:3] == ( + "-m", + "examples.optimization.eval_optimize_loop.pipeline.optimizer_worker", + ) + assert captured["request"]["callbackSpec"] == _live_adapter().import_path + assert captured["request"]["callbackSourcePath"] == str(_live_adapter().source_path) + assert captured["request"]["callbackSourceSha256"] == _live_adapter().source_sha256 + assert captured["request"]["callbackCallableSha256"] == _live_adapter().callable_sha256 + assert captured["request"]["promptPaths"] == {"system": str(prompt_path)} + assert proposal.prompts == {"system": "candidate"} + assert proposal.rounds[0].metric_scores == {"quality": 0.8} + assert proposal.cost_sources[0].cost_usd == 0.03 diff --git a/tests/examples/eval_optimize_loop/test_candidate_runtime.py b/tests/examples/eval_optimize_loop/test_candidate_runtime.py new file mode 100644 index 000000000..1bb4261f0 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_candidate_runtime.py @@ -0,0 +1,83 @@ +"""Candidate-generation lifecycle tests for primary-error and artifact boundaries.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline.candidate_runtime import generate_candidate + + +class _DumpConfig: + + def model_dump(self, **kwargs): + return {} + + +class _Workspace: + baseline = {"system": "baseline"} + + def create_candidate_target(self, path: str): + Path(path).mkdir(parents=True, exist_ok=False) + return object() + + +class _ImportFailingSink: + + def __init__(self, error: BaseException | None = None) -> None: + self.error = error or json.JSONDecodeError("half-written optimizer output", "{", 1) + + def phase_dir(self, name: str) -> None: + return None + + def import_tree(self, source: str | Path, destination: str) -> None: + raise self.error + + +@pytest.mark.asyncio +@pytest.mark.parametrize("primary", [asyncio.CancelledError("cancel"), RuntimeError("optimizer failed")]) +async def test_artifact_import_failure_does_not_replace_generator_failure(primary) -> None: + + class FailingGenerator: + + async def generate(self, **kwargs): + raise primary + + with pytest.raises(type(primary), match=str(primary)) as raised: + await generate_candidate( + generator=FailingGenerator(), + workspace=_Workspace(), + sink=_ImportFailingSink(), + eval_config=_DumpConfig(), + optimize_config=_DumpConfig(), + train_attribution=object(), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + ) + assert any("artifact import also failed" in note for note in raised.value.__notes__) + + +@pytest.mark.asyncio +async def test_base_exception_during_import_does_not_replace_generator_failure() -> None: + + class FailingGenerator: + + async def generate(self, **kwargs): + raise RuntimeError("optimizer failed first") + + sink = _ImportFailingSink(KeyboardInterrupt("import interrupted")) + with pytest.raises(RuntimeError, match="optimizer failed first") as raised: + await generate_candidate( + generator=FailingGenerator(), + workspace=_Workspace(), + sink=sink, + eval_config=_DumpConfig(), + optimize_config=_DumpConfig(), + train_attribution=object(), + inner_train_path="inner-train.json", + inner_selection_path="inner-selection.json", + ) + assert any("KeyboardInterrupt: import interrupted" in note for note in raised.value.__notes__) diff --git a/tests/examples/eval_optimize_loop/test_cli.py b/tests/examples/eval_optimize_loop/test_cli.py new file mode 100644 index 000000000..db9154e60 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_cli.py @@ -0,0 +1,18 @@ +"""Command-line boundary tests.""" + +from examples.optimization.eval_optimize_loop.run_pipeline import _format_cli_error +from examples.optimization.eval_optimize_loop.pipeline.schema import add_exception_note + + +def test_cli_error_output_is_bounded_and_redacted() -> None: + error = RuntimeError( + "Authorization: Bearer live-token; api_key=live-key; " + "context" * 1000) + add_exception_note(error, "password=note-secret") + + rendered = _format_cli_error(error) + + assert len(rendered) <= 4000 + assert "live-token" not in rendered + assert "live-key" not in rendered + assert "note-secret" not in rendered + assert rendered.count("[REDACTED]") == 3 diff --git a/tests/examples/eval_optimize_loop/test_deliverables.py b/tests/examples/eval_optimize_loop/test_deliverables.py new file mode 100644 index 000000000..36e36f13b --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_deliverables.py @@ -0,0 +1,88 @@ +"""Issue-level deliverable contract tests for the public example.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +from examples.optimization.eval_optimize_loop.pipeline.models import ( + Decision, + OptimizationReport, + Transition, +) +from examples.optimization.eval_optimize_loop.pipeline.reporting import render_markdown + +EXAMPLE_ROOT = (Path(__file__).resolve().parents[3] / "examples" / "optimization" / "eval_optimize_loop") + + +def _dataset(name: str) -> dict: + return json.loads((EXAMPLE_ROOT / name).read_text(encoding="utf-8")) + + +def test_public_datasets_cover_required_case_matrix() -> None: + train = _dataset("train.evalset.json")["evalCases"] + validation = _dataset("val.evalset.json")["evalCases"] + assert len(train) == 3 + assert len(validation) == 3 + assert {case["evalId"] for case in train}.isdisjoint(case["evalId"] for case in validation) + queries = [case["conversation"][0]["userContent"]["parts"][0]["text"] for case in (*train, *validation)] + assert all( + any(f"BEHAVIOR:{behavior}" in query for query in queries) for behavior in ("improve", "stable", "regress")) + + +def test_solution_is_300_to_500_chinese_characters_and_covers_required_topics() -> None: + solution = (EXAMPLE_ROOT / "SOLUTION.md").read_text(encoding="utf-8") + chinese_characters = re.findall(r"[\u4e00-\u9fff]", solution) + assert 300 <= len(chinese_characters) <= 500 + for topic in ("失败归因", "接受门禁", "过拟合", "审计"): + assert topic in solution + + +def test_committed_sample_report_is_complete_and_rendered_from_the_model() -> None: + sample_root = EXAMPLE_ROOT / "sample_output" + payload = json.loads((sample_root / "optimization_report.json").read_text(encoding="utf-8")) + assert { + "baseline", + "candidate", + "delta", + "gateDecision", + "failureAttribution", + } <= payload.keys() + + report = OptimizationReport.model_validate(payload) + assert report.status == Decision.REJECT + assert report.stage == "complete" + assert report.inputs["auditHashes"] == { + "train": report.inputs["hashes"]["train"], + "validation": report.inputs["hashes"]["validation"], + } + assert report.duration_seconds < 180 + assert report.reproducibility.reproducible is True + assert report.reproducibility.git_dirty is False + assert report.baseline and report.baseline.train and report.baseline.validation + assert report.candidate and report.candidate.train and report.candidate.validation + assert report.delta and report.delta.train and report.delta.validation + assert report.gate_decision and report.gate_decision.decision == Decision.REJECT + assert "NEW_HARD_FAIL_BUDGET_EXCEEDED" in report.gate_decision.reasons + transitions = {case.transition for case in report.delta.validation.cases} + assert transitions == { + Transition.NEW_PASS, + Transition.NEW_FAIL, + Transition.UNCHANGED, + } + assert report.failure_attribution + assert report.optimization and report.optimization["innerSplit"]["auditHashes"] == { + "train": report.optimization["innerSplit"]["trainHash"], + "selection": report.optimization["innerSplit"]["selectionHash"], + } + for snapshot in ( + report.failure_attribution.train, + report.failure_attribution.validation, + ): + assert snapshot + assert snapshot.statistics.total_failures == len(snapshot.failures) + assert all(failure.reasons and failure.evidence for failure in snapshot.failures) + + markdown = (sample_root / "optimization_report.md").read_text(encoding="utf-8") + assert markdown == render_markdown(report) diff --git a/tests/examples/eval_optimize_loop/test_evaluation.py b/tests/examples/eval_optimize_loop/test_evaluation.py new file mode 100644 index 000000000..c76342efd --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_evaluation.py @@ -0,0 +1,232 @@ +"""Dataset, normalization and comparison tests.""" + +from __future__ import annotations + +from copy import deepcopy + +import pytest + +from trpc_agent_sdk.evaluation import ( + EvalCaseResult, + EvalConfig, + EvalMetricResult, + EvalMetricResultPerInvocation, + EvalSet, + EvalSetAggregateResult, + EvalStatus, + EvaluateResult, +) + +from examples.optimization.eval_optimize_loop.pipeline.evaluation import ( + compare_snapshots, + normalize_result, + split_train_dataset, + validate_datasets, +) +from examples.optimization.eval_optimize_loop.pipeline.models import Phase, Split, Transition + + +def _eval_set(set_id: str, case_ids: list[str]) -> EvalSet: + return EvalSet.model_validate({ + "evalSetId": + set_id, + "evalCases": [{ + "evalId": + case_id, + "conversation": [{ + "invocationId": f"input-{case_id}", + "userContent": { + "role": "user", + "parts": [{ + "text": f"query {case_id}" + }] + }, + "finalResponse": { + "role": "model", + "parts": [{ + "text": "expected" + }] + }, + }], + "sessionInput": { + "appName": "loop-test", + "userId": "test", + "state": {} + }, + } for case_id in case_ids], + }) + + +def _raw(eval_set: EvalSet, scores: dict[str, list[float]], threshold: float = 0.5) -> EvaluateResult: + by_case: dict[str, list[EvalCaseResult]] = {} + for case in eval_set.eval_cases: + runs = [] + expected = case.conversation[0] + for run_id, score in enumerate(scores[case.eval_id], 1): + passed = score >= threshold + status = EvalStatus.PASSED if passed else EvalStatus.FAILED + actual = expected.model_copy( + update={ + "invocation_id": f"actual-{case.eval_id}-{run_id}", + "final_response": expected.final_response, + }, + deep=True, + ) + metric = EvalMetricResult( + metric_name="quality", + threshold=threshold, + score=score, + eval_status=status, + ) + runs.append( + EvalCaseResult( + eval_set_id=eval_set.eval_set_id, + eval_id=case.eval_id, + run_id=run_id, + final_eval_status=status, + overall_eval_metric_results=[metric], + eval_metric_result_per_invocation=[ + EvalMetricResultPerInvocation( + actual_invocation=actual, + expected_invocation=expected, + eval_metric_results=[metric], + ) + ], + session_id=f"session-{run_id}", + user_id="test", + )) + by_case[case.eval_id] = runs + return EvaluateResult( + results_by_eval_set_id={ + eval_set.eval_set_id: + EvalSetAggregateResult( + eval_results_by_eval_id=by_case, + num_runs=len(next(iter(scores.values()))), + ) + }) + + +def test_dataset_validation_rejects_content_leakage() -> None: + train = _eval_set("train", ["a", "b", "c"]) + validation = _eval_set("validation", ["v"]) + leaked = train.eval_cases[0].model_copy(update={"eval_id": "v"}, deep=True) + validation = validation.model_copy(update={"eval_cases": [leaked]}, deep=True) + with pytest.raises(ValueError, match="duplicate case content"): + validate_datasets( + train, + validation, + train_path="train.json", + validation_path="validation.json", + configured_metrics=["quality"], + ) + + +def test_dataset_validation_rejects_same_input_with_changed_reference() -> None: + train = _eval_set("train", ["a", "b", "c"]) + validation = _eval_set("validation", ["v"]) + leaked = train.eval_cases[0].model_copy(update={"eval_id": "v"}, deep=True) + leaked.conversation[0].final_response.parts[0].text = "different label" + validation = validation.model_copy(update={"eval_cases": [leaked]}, deep=True) + with pytest.raises(ValueError, match="duplicate model inputs"): + validate_datasets( + train, + validation, + train_path="train.json", + validation_path="validation.json", + configured_metrics=["quality"], + ) + + +@pytest.mark.parametrize("metrics", [[], ["quality", "quality"]]) +def test_dataset_validation_rejects_invalid_metric_schema(metrics) -> None: + with pytest.raises(ValueError, match="metric names"): + validate_datasets( + _eval_set("train", ["a", "b", "c"]), + _eval_set("validation", ["v"]), + train_path="train.json", + validation_path="validation.json", + configured_metrics=metrics, + ) + + +def test_normalization_aggregates_runs_and_inner_split_is_deterministic() -> None: + train = _eval_set("train", ["a", "b", "c"]) + config = EvalConfig(metrics=[{"metric_name": "quality", "threshold": 0.5}], num_runs=2) + snapshot = normalize_result( + _raw(train, { + "a": [1, 1], + "b": [0, 1], + "c": [0, 0] + }), + train, + config, + split=Split.TRAIN, + phase=Phase.BASELINE, + case_weights={"a": 2}, + ) + assert snapshot.case_ids == ("a", "b", "c") + assert snapshot.cases[1].score == 0.5 + assert snapshot.cases[1].passed is False + assert snapshot.dataset_score == pytest.approx(0.625) + first = split_train_dataset(train, snapshot, seed=42, selection_ratio=0.34) + second = split_train_dataset(train, snapshot, seed=42, selection_ratio=0.34) + assert [case.eval_id for case in first[0].eval_cases] == [case.eval_id for case in second[0].eval_cases] + assert len(first[0].eval_cases) == 2 + assert len(first[1].eval_cases) == 1 + + +def test_normalization_fails_on_threshold_drift() -> None: + eval_set = _eval_set("validation", ["v"]) + config = EvalConfig(metrics=[{"metric_name": "quality", "threshold": 0.5}], num_runs=1) + raw = _raw(eval_set, {"v": [1]}, threshold=0.4) + with pytest.raises(ValueError, match="threshold drift"): + normalize_result(raw, eval_set, config, split=Split.VALIDATION, phase=Phase.BASELINE) + + +def test_comparison_classifies_all_transition_types() -> None: + eval_set = _eval_set("validation", ["a", "b", "c", "d", "e"]) + config = EvalConfig(metrics=[{"metric_name": "quality", "threshold": 0.5}], num_runs=1) + baseline = normalize_result( + _raw(eval_set, { + "a": [0], + "b": [1], + "c": [0.6], + "d": [0.9], + "e": [1] + }), + eval_set, + config, + split=Split.VALIDATION, + phase=Phase.BASELINE, + ) + candidate = normalize_result( + _raw(eval_set, { + "a": [1], + "b": [0], + "c": [0.8], + "d": [0.7], + "e": [1] + }), + eval_set, + config, + split=Split.VALIDATION, + phase=Phase.CANDIDATE, + ) + comparison = compare_snapshots(baseline, candidate, epsilon=1e-6, hard_case_ids=["b"]) + assert [case.transition for case in comparison.cases] == [ + Transition.NEW_PASS, + Transition.NEW_FAIL, + Transition.IMPROVED, + Transition.REGRESSED, + Transition.UNCHANGED, + ] + assert comparison.cases[1].new_hard_failure is True + + +def test_normalizer_does_not_mutate_sdk_result() -> None: + eval_set = _eval_set("validation", ["v"]) + config = EvalConfig(metrics=[{"metric_name": "quality", "threshold": 0.5}], num_runs=1) + raw = _raw(eval_set, {"v": [1]}) + before = deepcopy(raw.model_dump(mode="json")) + normalize_result(raw, eval_set, config, split=Split.VALIDATION, phase=Phase.BASELINE) + assert raw.model_dump(mode="json") == before diff --git a/tests/examples/eval_optimize_loop/test_models_and_imports.py b/tests/examples/eval_optimize_loop/test_models_and_imports.py new file mode 100644 index 000000000..60a4496d6 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_models_and_imports.py @@ -0,0 +1,360 @@ +"""Strict model and architecture boundary tests for the loop example.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest +from pydantic import ValidationError +from trpc_agent_sdk.evaluation import EvalSet + +from examples.optimization.eval_optimize_loop.pipeline.artifacts import load_strict_json +from examples.optimization.eval_optimize_loop.pipeline.configuration import ( + GateConfig, + PipelineSettings, +) +from examples.optimization.eval_optimize_loop.pipeline.evaluation import dataset_fingerprint +from examples.optimization.eval_optimize_loop.pipeline.models import ( + CandidateProposal, + CandidateRound, + CaseComparison, + CostSource, + CostSummary, + MetricDelta, + MetricRun, + OptimizationReport, + RubricOutcome, + Transition, +) +from examples.optimization.eval_optimize_loop.pipeline.schema import ( + add_exception_note, + parse_strict_json, + sanitized_text, + validate_safe_component, +) + +PIPELINE_DIR = Path(__file__).resolve().parents[3] / "examples" / "optimization" / "eval_optimize_loop" / "pipeline" +EXAMPLE_DIR = PIPELINE_DIR.parent + + +def test_dataset_fingerprint_tracks_only_explicit_input_fields() -> None: + train = EvalSet.model_validate(load_strict_json(EXAMPLE_DIR / "train.evalset.json")) + validation = EvalSet.model_validate(load_strict_json(EXAMPLE_DIR / "val.evalset.json")) + + assert dataset_fingerprint(train) == "af410bcc6f46c041a373efb1d1dda9f0f0f20258d1ef2146eac8a583cde99b19" + assert dataset_fingerprint(validation) == "5011756743a6211c78c75f2b7cbc435e98cf4caf0e0c484bbac9c5f03951c7d9" + + +def test_models_forbid_unknown_and_non_finite_values() -> None: + with pytest.raises(ValidationError): + PipelineSettings.model_validate({"unexpected": True}) + with pytest.raises(ValidationError): + GateConfig(epsilon=float("nan")) + with pytest.raises(ValidationError): + PipelineSettings(metric_weights={"metric": 0}) + with pytest.raises(ValidationError, match="v2"): + OptimizationReport.model_validate({"schemaVersion": "v1"}) + + +def test_pipeline_policy_scalars_reject_string_coercion() -> None: + with pytest.raises(ValidationError): + PipelineSettings.model_validate({"applyCandidate": "yes"}) + with pytest.raises(ValidationError): + PipelineSettings.model_validate({"seed": "42"}) + with pytest.raises(ValidationError): + GateConfig.model_validate({"maxNewHardFailures": "2"}) + with pytest.raises(ValidationError): + GateConfig.model_validate({"overfitGuard": False}) + + +def test_strict_json_and_path_components_reject_ambiguous_inputs() -> None: + with pytest.raises(ValueError, match="duplicate"): + parse_strict_json('{"key": 1, "key": 2}') + with pytest.raises(ValueError, match="non-finite"): + parse_strict_json('{"value": NaN}') + with pytest.raises(ValueError, match="non-finite"): + parse_strict_json('{"value": 1e999}') + for value in ("../escape", "a..b", "CON", "name.", "C:drive"): + with pytest.raises(ValueError): + validate_safe_component(value) + + +def test_candidate_requires_contiguous_accepted_history() -> None: + with pytest.raises(ValidationError): + CandidateProposal( + algorithm="fake", + baseline_prompts={"system": "old"}, + prompts={"system": "new"}, + changed=True, + rounds=(CandidateRound( + round=2, + candidate_prompts={"system": "new"}, + accepted=True, + score=1, + ), ), + ) + + +def test_candidate_accounting_defaults_to_unknown() -> None: + proposal = CandidateProposal( + algorithm="custom", + baseline_prompts={"system": "same"}, + prompts={"system": "same"}, + changed=False, + ) + assert proposal.cost_sources == (CostSource(name="unreported", cost_usd=None), ) + with pytest.raises(ValidationError): + CandidateRound( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=True, + score=1, + cost_usd=-1, + ) + + +def test_candidate_round_distinguishes_skipped_rounds_and_rejects_invalid_accounting() -> None: + skipped = CandidateProposal( + algorithm="gepa_reflective", + baseline_prompts={"system": "same"}, + prompts={"system": "same"}, + changed=False, + rounds=(CandidateRound( + round=1, + candidate_prompts={}, + accepted=False, + skip_reason="reflect-LM produced no usable new prompt", + ), ), + ) + assert skipped.rounds[0].score is None + with pytest.raises(ValidationError, match="skip or error"): + CandidateRound(round=1, candidate_prompts={}, accepted=False) + with pytest.raises(ValidationError, match="round score"): + CandidateRound( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=False, + score=-0.1, + ) + with pytest.raises(ValidationError, match="token usage"): + CandidateRound( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=False, + score=0, + token_usage={"total": -1}, + ) + negative_metric = CandidateRound( + round=1, + candidate_prompts={"system": "candidate"}, + accepted=False, + score=0, + metric_scores={"log_likelihood": -2.5}, + ) + assert negative_metric.metric_scores == {"log_likelihood": -2.5} + + +def test_sanitized_text_redacts_structured_values_and_exception_messages() -> None: + assert "tool-secret" not in sanitized_text( + {"actual": { + "apiKey": "tool-secret" + }}, + max_text_chars=1000, + ) + assert "error-secret" not in sanitized_text( + RuntimeError('{"clientSecret":"error-secret"}'), + max_text_chars=1000, + ) + error = RuntimeError("primary failure") + add_exception_note(error, "Authorization: Bearer diagnostic-secret") + diagnostic = sanitized_text(error, max_text_chars=1000) + assert "diagnostic-secret" not in diagnostic + assert "[REDACTED]" in diagnostic + + +def test_exception_notes_work_without_python_311_add_note() -> None: + + class LegacyError(RuntimeError): + add_note = None + + error = LegacyError("primary") + add_exception_note(error, "secondary diagnostic") + assert error.__notes__ == ["secondary diagnostic"] + + +def test_rubric_and_cost_contracts_reject_inconsistent_aggregates() -> None: + scaled = MetricRun( + metric_name="quality_5_point", + score=4, + threshold=3, + passed=True, + rubrics=(RubricOutcome(id="quality", score=4, passed=True), ), + ) + assert scaled.rubrics[0].score == 4 + with pytest.raises(ValidationError, match="finite"): + RubricOutcome(id="quality", score=float("inf"), passed=True) + with pytest.raises(ValidationError, match="metric threshold"): + MetricRun( + metric_name="quality", + score=0, + threshold=0.5, + passed=False, + rubrics=(RubricOutcome(id="quality", score=1, passed=False), ), + ) + with pytest.raises(ValidationError, match="must be unknown"): + CostSummary( + sources=(CostSource(name="judge", cost_usd=None), ), + total_cost_usd=0, + ) + with pytest.raises(ValidationError, match="does not equal"): + CostSummary( + sources=(CostSource(name="judge", cost_usd=1), ), + total_cost_usd=2, + ) + + +def test_comparison_contract_rejects_contradictory_delta_and_failure_flags() -> None: + with pytest.raises(ValidationError, match="metric delta"): + MetricDelta(metric_name="quality", baseline=0, candidate=1, delta=0) + with pytest.raises(ValidationError, match="hard failure"): + CaseComparison( + case_id="critical", + baseline_passed=True, + candidate_passed=False, + baseline_score=1, + candidate_score=0, + delta=-1, + metrics=(MetricDelta(metric_name="quality", baseline=1, candidate=0, delta=-1), ), + transition=Transition.NEW_FAIL, + hard=True, + new_hard_failure=False, + ) + + +def _pipeline_imports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + imports: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.level == 1 and node.module: + imports.add(node.module.split(".")[0]) + return imports + + +def test_production_pipeline_has_no_optimizable_assertions() -> None: + paths = [*PIPELINE_DIR.glob("*.py"), EXAMPLE_DIR / "run_pipeline.py"] + violations = [] + for path in paths: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + violations.extend( + f"{path.name}:{node.lineno}" for node in ast.walk(tree) if isinstance(node, ast.Assert)) + assert violations == [] + + +def test_pipeline_import_boundaries() -> None: + allowed = { + "artifacts": {"models", "schema"}, + "attribution": {"configuration", "models", "schema"}, + "backends": {"contracts", "live_adapter", "models", "offline_evaluation", "schema", "trace_fixture"}, + "candidate_runtime": { + "artifacts", + "attribution", + "contracts", + "evaluation", + "models", + "prompt_workspace", + "schema", + }, + "configuration": {"live_adapter", "models", "schema"}, + "contracts": {"models"}, + "costing": {"models"}, + "evaluation": {"models", "schema"}, + "evaluation_runtime": { + "artifacts", + "configuration", + "contracts", + "costing", + "evaluation", + "models", + }, + "gate": {"configuration", "models"}, + "live_adapter": set(), + "models": {"schema"}, + "offline_evaluation": set(), + "optimizer_worker": {"live_adapter", "schema"}, + "orchestrator": { + "artifacts", + "attribution", + "backends", + "candidate_runtime", + "configuration", + "contracts", + "costing", + "evaluation", + "evaluation_runtime", + "gate", + "models", + "preflight", + "prompt_workspace", + "reporting", + "schema", + }, + "preflight": { + "artifacts", + "configuration", + "evaluation", + "live_adapter", + "prompt_workspace", + "schema", + "trace_fixture", + }, + "prompt_workspace": {"schema"}, + "reporting": {"artifacts", "configuration", "models"}, + "schema": set(), + "trace_fixture": {"models", "schema"}, + } + graph: dict[str, set[str]] = {} + for path in PIPELINE_DIR.glob("*.py"): + if path.name == "__init__.py": + continue + module = path.stem + graph[module] = _pipeline_imports(path) + assert set(graph) == set(allowed) + for module, dependencies in graph.items(): + unexpected = dependencies - allowed[module] + assert not unexpected, f"{module} imports disallowed pipeline modules: {sorted(unexpected)}" + + visiting: set[str] = set() + visited: set[str] = set() + + def visit(module: str) -> None: + if module in visiting: + raise AssertionError(f"pipeline import cycle involving {module}") + if module in visited: + return + visiting.add(module) + for dependency in graph.get(module, set()): + visit(dependency) + visiting.remove(module) + visited.add(module) + + for module in graph: + visit(module) + + +def test_only_normalizer_unfolds_raw_evaluate_result() -> None: + raw_result_attributes = { + "results_by_eval_set_id", + "eval_results_by_eval_id", + "overall_eval_metric_results", + "eval_metric_result_per_invocation", + } + for path in PIPELINE_DIR.glob("*.py"): + if path.stem in {"__init__", "evaluation"}: + continue + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + used = { + node.attr + for node in ast.walk(tree) if isinstance(node, ast.Attribute) and node.attr in raw_result_attributes + } + assert not used, f"{path.name} unfolds raw EvaluateResult fields: {sorted(used)}" diff --git a/tests/examples/eval_optimize_loop/test_orchestrator.py b/tests/examples/eval_optimize_loop/test_orchestrator.py new file mode 100644 index 000000000..6dd3ebde4 --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_orchestrator.py @@ -0,0 +1,906 @@ +"""End-to-end lifecycle tests for the fake orchestration path.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import shlex +import shutil +import subprocess +import time +from pathlib import Path + +import pytest +from trpc_agent_sdk.evaluation import EvalSet + +from examples.optimization.eval_optimize_loop.pipeline.models import ( + CandidateProposal, + CandidateRound, + CostSource, + Decision, + OptimizationReport, + Transition, +) +from examples.optimization.eval_optimize_loop.pipeline.backends import ( + DeterministicCandidateGenerator, + FakeEvaluationBackend, +) +from examples.optimization.eval_optimize_loop.pipeline.artifacts import ( + AuditPersistenceError, + AuditSink, +) +from examples.optimization.eval_optimize_loop.pipeline.costing import CostLedger +from examples.optimization.eval_optimize_loop.pipeline.evaluation import ( + dataset_fingerprint, + dataset_fingerprint_payload, +) +from examples.optimization.eval_optimize_loop.pipeline.evaluation_runtime import create_evaluation_runtime +from examples.optimization.eval_optimize_loop.pipeline.models import Phase, Split +from examples.optimization.eval_optimize_loop.pipeline.orchestrator import run_pipeline +from examples.optimization.eval_optimize_loop.pipeline.preflight import preflight_run +from examples.optimization.eval_optimize_loop.pipeline.prompt_workspace import ( + PromptRestoreError, +) +from examples.optimization.eval_optimize_loop.pipeline import orchestrator as orchestrator_module +from examples.optimization.eval_optimize_loop.pipeline import reporting as reporting_module + +SOURCE = Path(__file__).resolve().parents[3] / "examples" / "optimization" / "eval_optimize_loop" + + +async def importable_call_agent(query: str) -> str: + return query + + +async def alternate_importable_call_agent(query: str) -> str: + return query + + +def _example(tmp_path: Path, *, accept: bool = False, apply: bool = False) -> Path: + root = tmp_path / "example" + (root / "prompts").mkdir(parents=True) + (root / "traces").mkdir() + for name in ("optimizer.json", "train.evalset.json", "val.evalset.json"): + shutil.copyfile(SOURCE / name, root / name) + shutil.copyfile(SOURCE / "prompts" / "system.md", root / "prompts" / "system.md") + shutil.copyfile(SOURCE / "traces" / "trace_cases.json", root / "traces" / "trace_cases.json") + config = json.loads((root / "optimizer.json").read_text(encoding="utf-8")) + if accept: + config["pipeline"]["gate"]["minValidationScoreDelta"] = 0 + config["pipeline"]["gate"]["maxNewHardFailures"] = 1 + config["pipeline"]["applyCandidate"] = apply + (root / "optimizer.json").write_text(json.dumps(config), encoding="utf-8") + return root + + +@pytest.mark.asyncio +async def test_fake_reject_with_hard_regression_is_a_completed_audit_run(tmp_path) -> None: + root = _example(tmp_path) + report = await run_pipeline(str(root), run_id="reject-run") + assert report.status == Decision.REJECT + assert [case.transition for case in report.delta.validation.cases] == [ + Transition.NEW_PASS, + Transition.NEW_FAIL, + Transition.UNCHANGED, + ] + assert "NEW_HARD_FAIL_BUDGET_EXCEEDED" in report.gate_decision.reasons + assert "OVERFIT_TRAIN_UP_VALIDATION_DOWN" not in report.gate_decision.reasons + assert report.source_application.applied is False + actual_prompt = (root / "prompts" / "system.md").read_text(encoding="utf-8") + expected_prompt = (SOURCE / "prompts" / "system.md").read_text(encoding="utf-8") + assert actual_prompt == expected_prompt + assert report.duration_seconds < 180 + artifact_paths = {record.path for record in report.artifacts} + assert "optimization_report.json" not in artifact_paths + assert "optimization_report.md" not in artifact_paths + manifest = json.loads((root / "artifacts" / "reject-run" / "manifest.json").read_text(encoding="utf-8")) + records = {record["path"]: record for record in manifest["files"]} + for name in ("optimization_report.json", "optimization_report.md"): + content = (root / "artifacts" / "reject-run" / name).read_bytes() + assert records[name]["sha256"] == hashlib.sha256(content).hexdigest() + + +@pytest.mark.asyncio +async def test_audit_dataset_copies_round_trip_to_reported_fingerprints(tmp_path) -> None: + root = _example(tmp_path) + report = await run_pipeline(str(root), run_id="dataset-contract") + run_dir = root / "artifacts" / "dataset-contract" + + for source_name, audit_name in ( + ("train.evalset.json", "train.evalset.json"), + ("val.evalset.json", "val.evalset.json"), + ): + source = EvalSet.model_validate(json.loads((root / source_name).read_text(encoding="utf-8"))) + audited = EvalSet.model_validate(json.loads((run_dir / audit_name).read_text(encoding="utf-8"))) + assert dataset_fingerprint(audited) == dataset_fingerprint(source) + + assert report.inputs["auditHashes"] == { + split: report.inputs["hashes"][split] for split in ("train", "validation") + } + assert report.optimization is not None + inner = report.optimization["innerSplit"] + for split_name, hash_name, path_name in ( + ("train", "trainHash", "trainPath"), + ("selection", "selectionHash", "selectionPath"), + ): + audited_payload = json.loads((run_dir / inner[path_name]).read_text(encoding="utf-8")) + audited = EvalSet.model_validate(audited_payload) + assert inner[hash_name] == dataset_fingerprint(audited) + assert inner["auditHashes"][split_name] == dataset_fingerprint_payload(audited_payload) + + +@pytest.mark.asyncio +async def test_secret_bearing_dataset_has_distinct_source_and_audit_hashes(tmp_path) -> None: + root = _example(tmp_path) + train_path = root / "train.evalset.json" + train_payload = json.loads(train_path.read_text(encoding="utf-8")) + train_payload["evalCases"][0]["sessionInput"]["state"]["apiKey"] = "fixture-secret" + train_path.write_text(json.dumps(train_payload), encoding="utf-8") + + source = EvalSet.model_validate(train_payload) + report = await run_pipeline(str(root), run_id="redacted-dataset-contract") + audited_payload = json.loads( + (root / "artifacts" / "redacted-dataset-contract" / "train.evalset.json").read_text(encoding="utf-8")) + + assert audited_payload["evalCases"][0]["sessionInput"]["state"]["apiKey"] == "[REDACTED]" + assert report.inputs["hashes"]["train"] == dataset_fingerprint(source) + assert report.inputs["auditHashes"]["train"] == dataset_fingerprint_payload(audited_payload) + assert report.inputs["hashes"]["train"] != report.inputs["auditHashes"]["train"] + + +@pytest.mark.asyncio +async def test_accept_without_apply_leaves_source_unchanged(tmp_path) -> None: + root = _example(tmp_path, accept=True) + baseline = (root / "prompts" / "system.md").read_text(encoding="utf-8") + report = await run_pipeline(str(root), run_id="accept-no-apply") + assert report.status == Decision.ACCEPT + assert report.source_application.requested is False + assert report.source_application.applied is False + assert (root / "prompts" / "system.md").read_text(encoding="utf-8") == baseline + + +@pytest.mark.asyncio +async def test_accept_apply_writes_and_verifies_candidate(tmp_path) -> None: + root = _example(tmp_path, accept=True, apply=True) + report = await run_pipeline(str(root), run_id="accept-apply") + prompt = (root / "prompts" / "system.md").read_text(encoding="utf-8") + assert report.status == Decision.ACCEPT + assert report.source_application.applied is True + assert "BEHAVIOR_PROFILE=precision-v2" in prompt + assert report.source_application.final_hashes["system"] == hashlib.sha256(prompt.encode("utf-8")).hexdigest() + + +@pytest.mark.asyncio +async def test_final_report_failure_after_apply_rolls_back(tmp_path) -> None: + root = _example(tmp_path, accept=True, apply=True) + baseline = (root / "prompts" / "system.md").read_text(encoding="utf-8") + + def fail_final_report(stage: str) -> None: + if stage == "final_report": + raise OSError("injected final report failure") + + report = await run_pipeline( + str(root), + run_id="failed-final-report", + fault_injector=fail_final_report, + ) + assert report.status == Decision.ERROR + assert report.stage == "final_report" + assert report.source_application.applied is False + assert (root / "prompts" / "system.md").read_text(encoding="utf-8") == baseline + + +@pytest.mark.asyncio +async def test_apply_and_restore_double_failure_is_preserved_in_terminal_audit(tmp_path, monkeypatch) -> None: + root = _example(tmp_path, accept=True, apply=True) + original_write = orchestrator_module.PromptWorkspace._write_verified + candidate_writes = 0 + + async def fail_apply_and_restore(workspace, prompts): + nonlocal candidate_writes + if prompts != workspace.baseline: + candidate_writes += 1 + if candidate_writes > 1: + raise OSError("candidate write unavailable") + elif candidate_writes > 1: + raise OSError("restore unavailable: " + "r" * 5000) + return await original_write(workspace, prompts) + + monkeypatch.setattr( + orchestrator_module.PromptWorkspace, + "_write_verified", + fail_apply_and_restore, + ) + with pytest.raises(PromptRestoreError): + await run_pipeline(str(root), run_id="apply-restore-double-failure") + + report_path = root / "artifacts" / "apply-restore-double-failure" / "optimization_report.json" + payload = json.loads(report_path.read_text(encoding="utf-8")) + message = payload["errors"][0]["message"] + assert "candidate write unavailable" in message + assert "restore unavailable" in message + assert len(message) <= 4000 + + +@pytest.mark.asyncio +async def test_error_notes_have_bounded_count_and_total_size(tmp_path) -> None: + root = _example(tmp_path) + + class FailingGenerator: + + async def generate(self, **kwargs): + error = RuntimeError("primary failure") + for index in range(100): + error.add_note(f"note-{index}: " + "x" * 10_000) + raise error + + report = await run_pipeline(str(root), run_id="bounded-error-notes", candidate_generator=FailingGenerator()) + assert report.status == Decision.ERROR + message = report.errors[0].message + assert "primary failure" in message + assert "diagnostics omitted" in message + assert len(message) <= 4000 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failed_stage", + ( + "audit_config", + "baseline_train", + "baseline_validation", + "inner_split", + "candidate_generation", + "candidate_train", + "candidate_validation", + "comparison", + "gate", + "pre_apply_report", + "apply", + "final_report", + ), +) +async def test_every_orchestration_stage_fails_to_an_error_audit_and_restores_prompt( + tmp_path, + failed_stage, +) -> None: + root = _example(tmp_path, accept=True, apply=True) + baseline = (root / "prompts" / "system.md").read_text(encoding="utf-8") + + def inject(stage: str) -> None: + if stage == failed_stage: + raise RuntimeError(f"injected {stage} failure") + + report = await run_pipeline( + str(root), + run_id=f"stage-{failed_stage}", + fault_injector=inject, + ) + + assert report.status == Decision.ERROR + assert report.stage == failed_stage + assert report.errors[0].stage == failed_stage + assert report.source_application.applied is False + assert (root / "prompts" / "system.md").read_text(encoding="utf-8") == baseline + persisted = OptimizationReport.model_validate_json( + (root / "artifacts" / f"stage-{failed_stage}" / "optimization_report.json").read_text(encoding="utf-8")) + assert persisted.status == Decision.ERROR + + +@pytest.mark.asyncio +async def test_run_directory_is_never_reused(tmp_path) -> None: + root = _example(tmp_path) + await run_pipeline(str(root), run_id="immutable") + report_path = root / "artifacts" / "immutable" / "optimization_report.json" + before = report_path.read_bytes() + with pytest.raises(FileExistsError): + await run_pipeline(str(root), run_id="immutable") + assert report_path.read_bytes() == before + + +@pytest.mark.asyncio +async def test_trace_drift_fails_before_artifact_creation(tmp_path) -> None: + root = _example(tmp_path) + fixture_path = root / "traces" / "trace_cases.json" + fixture = json.loads(fixture_path.read_text(encoding="utf-8")) + fixture["datasetHashes"]["train"] = "0" * 64 + fixture_path.write_text(json.dumps(fixture), encoding="utf-8") + with pytest.raises(ValueError, match="hashes"): + await run_pipeline(str(root), mode="trace", run_id="trace-drift") + assert not (root / "artifacts").exists() + + +@pytest.mark.asyncio +async def test_source_trace_fixture_completes_offline_pipeline(tmp_path) -> None: + root = _example(tmp_path) + + report = await run_pipeline( + str(root), + mode="trace", + run_id="source-trace-complete", + ) + + assert report.status == Decision.REJECT + assert report.stage == "complete" + assert report.duration_seconds < 180 + + +@pytest.mark.asyncio +async def test_cancellation_restores_prompt_and_propagates(tmp_path) -> None: + root = _example(tmp_path) + baseline = (root / "prompts" / "system.md").read_text(encoding="utf-8") + + def cancel_candidate_validation(stage: str) -> None: + if stage == "candidate_validation": + raise asyncio.CancelledError() + + with pytest.raises(asyncio.CancelledError): + await run_pipeline( + str(root), + run_id="cancelled", + fault_injector=cancel_candidate_validation, + ) + assert (root / "prompts" / "system.md").read_text(encoding="utf-8") == baseline + report = json.loads((root / "artifacts" / "cancelled" / "optimization_report.json").read_text(encoding="utf-8")) + assert report["status"] == "ERROR" + assert report["stage"] == "candidate_validation" + assert report["candidate"]["train"] is not None + assert report["candidate"]["validation"] is None + + +@pytest.mark.asyncio +async def test_error_report_preserves_completed_baseline_train(tmp_path) -> None: + root = _example(tmp_path) + + def fail_baseline_validation(stage: str) -> None: + if stage == "baseline_validation": + raise RuntimeError("validation unavailable") + + report = await run_pipeline( + str(root), + run_id="partial-baseline", + fault_injector=fail_baseline_validation, + ) + assert report.status == Decision.ERROR + assert report.baseline is not None + assert report.baseline.train is not None + assert report.baseline.validation is None + assert report.failure_attribution is not None + assert report.failure_attribution.train is not None + assert report.failure_attribution.validation is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure_call", "failed_stage", "candidate_completed"), + ((2, "baseline_validation", False), (4, "candidate_validation", True)), +) +async def test_backend_failure_preserves_partial_facts_and_unknown_failed_call_cost( + tmp_path, + failure_call, + failed_stage, + candidate_completed, +) -> None: + root = _example(tmp_path) + delegate = FakeEvaluationBackend() + + class FailingBackend: + + def __init__(self) -> None: + self.calls = 0 + + async def evaluate(self, **kwargs): + self.calls += 1 + if self.calls == failure_call: + raise RuntimeError("backend unavailable") + return await delegate.evaluate(**kwargs) + + report = await run_pipeline( + str(root), + run_id=f"backend-failure-{failure_call}", + backend=FailingBackend(), + ) + + assert report.status == Decision.ERROR + assert report.stage == failed_stage + failed_source = next(source for source in report.cost.sources if source.name == failed_stage) + assert failed_source.cost_usd is None + assert failed_source.model_calls is None + assert failed_source.metric_calls is None + if candidate_completed: + assert report.candidate.train is not None + assert report.candidate.validation is None + assert report.candidate_failure_attribution.train is not None + assert report.candidate_failure_attribution.validation is None + else: + assert report.baseline.train is not None + assert report.baseline.validation is None + assert report.failure_attribution.train is not None + assert report.failure_attribution.validation is None + + +@pytest.mark.asyncio +async def test_generator_failure_is_recorded_as_unknown_cost_and_redacted(tmp_path) -> None: + root = _example(tmp_path) + + class FailingGenerator: + + async def generate(self, **kwargs): + raise RuntimeError('optimizer failed with {"apiKey":"candidate-secret"}') + + report = await run_pipeline( + str(root), + run_id="generator-failure-cost", + candidate_generator=FailingGenerator(), + ) + + assert report.status == Decision.ERROR + assert report.stage == "candidate_generation" + source = next(item for item in report.cost.sources if item.name == "candidate_generation.unreported_failure") + assert source.cost_usd is None + assert source.model_calls is None + assert source.metric_calls is None + assert "candidate-secret" not in report.errors[0].message + assert "[REDACTED]" in report.errors[0].message + + +@pytest.mark.asyncio +async def test_failed_optimizer_result_preserves_rounds_cost_and_duration(tmp_path) -> None: + root = _example(tmp_path) + + class FailedGenerator: + async def generate(self, **kwargs): + baseline = dict(kwargs["baseline_prompts"]) + return CandidateProposal( + status="FAILED", + error_message="optimizer backend failed", + algorithm="gepa_reflective", + baseline_prompts=baseline, + prompts=baseline, + changed=False, + stop_reason="error", + rounds=(CandidateRound( + round=1, + candidate_prompts={}, + accepted=False, + error_message="reflection request failed", + cost_usd=0.12, + duration_seconds=0.4, + ), ), + cost_sources=(CostSource( + name="optimizer_reported", + cost_usd=0.12, + model_calls=1, + ), ), + duration_seconds=0.4, + ) + + report = await run_pipeline( + str(root), + run_id="failed-optimizer-facts", + candidate_generator=FailedGenerator(), + ) + + assert report.status == Decision.ERROR + assert report.optimization["proposal"]["status"] == "FAILED" + assert report.optimization["proposal"]["rounds"][0]["costUsd"] == 0.12 + assert report.optimization["proposal"]["durationSeconds"] == 0.4 + source = next(item for item in report.cost.sources if item.name.endswith("optimizer_reported")) + assert source.cost_usd == 0.12 + assert all(item.name != "candidate_generation.unreported_failure" for item in report.cost.sources) + + +@pytest.mark.asyncio +async def test_adapter_rejected_success_is_audited_without_evaluation_or_apply(tmp_path) -> None: + root = _example(tmp_path, accept=True, apply=True) + prompt_path = root / "prompts" / "system.md" + baseline_prompt = prompt_path.read_text(encoding="utf-8") + + class RejectedSuccessGenerator: + async def generate(self, **kwargs): + baseline = dict(kwargs["baseline_prompts"]) + return CandidateProposal( + status="SUCCEEDED", + adapter_error="optimizer baseline prompts differ from the workspace baseline", + algorithm="gepa_reflective", + baseline_prompts=baseline, + prompts=baseline, + changed=False, + cost_sources=(CostSource( + name="optimizer_reported", + cost_usd=0.03, + model_calls=1, + ), ), + duration_seconds=0.2, + ) + + report = await run_pipeline( + str(root), + run_id="adapter-rejected-success", + candidate_generator=RejectedSuccessGenerator(), + ) + + assert report.status == Decision.ERROR + assert report.stage == "candidate_generation" + assert report.optimization["proposal"]["status"] == "SUCCEEDED" + assert report.optimization["proposal"]["adapterError"] + assert report.candidate is None + assert prompt_path.read_text(encoding="utf-8") == baseline_prompt + + +@pytest.mark.asyncio +async def test_terminal_audit_failure_is_raised_instead_of_silently_returned(tmp_path, monkeypatch) -> None: + root = _example(tmp_path) + + class FailingGenerator: + + async def generate(self, **kwargs): + raise RuntimeError("optimizer unavailable") + + def fail_persistence(*args, **kwargs): + raise OSError("audit volume unavailable") + + monkeypatch.setattr(orchestrator_module, "persist_terminal_report", fail_persistence) + with pytest.raises(AuditPersistenceError, match="audit volume unavailable") as captured: + await run_pipeline( + str(root), + run_id="terminal-persistence-failure", + candidate_generator=FailingGenerator(), + ) + assert any("original pipeline error: RuntimeError" in note for note in captured.value.__notes__) + + +@pytest.mark.asyncio +async def test_live_evaluation_cost_bounds_are_accounted_conservatively(tmp_path) -> None: + root = _example(tmp_path) + config_path = root / "optimizer.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config["pipeline"].update({ + "mode": "live", + "liveAgentCallMaxCostUsd": 0.01, + "liveMetricCallMaxCostUsd": 0.02, + }) + config_path.write_text(json.dumps(config), encoding="utf-8") + + validated = preflight_run( + str(root), + config_path=None, + train_path=None, + validation_path=None, + mode=None, + run_id="live-cost-bounds", + apply_candidate=False, + call_agent=importable_call_agent, + callback_spec=f"{__name__}:importable_call_agent", + backend=None, + candidate_generator=None, + ) + sink = AuditSink(validated.artifact_root, validated.run_id) + sink.create() + ledger = CostLedger() + runtime = create_evaluation_runtime( + validated=validated, + backend=FakeEvaluationBackend(), + custom_backend=False, + sink=sink, + ledger=ledger, + ) + await runtime.evaluate( + eval_set=validated.train, + prompts={"system": "baseline"}, + split=Split.TRAIN, + phase=Phase.BASELINE, + ) + source = ledger.summary().sources[0] + invocations = sum(len(case.conversation or case.actual_conversation or []) for case in validated.train.eval_cases) + expected = invocations * validated.config.evaluate.num_runs * 0.03 + assert source.cost_usd == pytest.approx(expected) + assert source.upper_bound is True + assert source.model_calls == invocations * validated.config.evaluate.num_runs + + +@pytest.mark.asyncio +async def test_generator_receives_only_inner_train_attribution(tmp_path) -> None: + root = _example(tmp_path) + + class CapturingGenerator: + + def __init__(self) -> None: + self.checked = False + + async def generate(self, **kwargs): + inner = json.loads(Path(kwargs["inner_train_path"]).read_text(encoding="utf-8")) + inner_ids = {case["evalId"] for case in inner["evalCases"]} + failure_ids = {failure.case_id for failure in kwargs["train_attribution"].failures} + assert failure_ids <= inner_ids + output = Path(kwargs["output_dir"]) + output.mkdir(parents=True) + (output / "config.snapshot.json").write_text( + json.dumps({"apiKey": "optimizer-secret"}), + encoding="utf-8", + ) + self.checked = True + return await DeterministicCandidateGenerator().generate(**kwargs) + + generator = CapturingGenerator() + report = await run_pipeline( + str(root), + run_id="inner-attribution", + candidate_generator=generator, + ) + assert report.status != Decision.ERROR + assert generator.checked is True + audit_text = "\n".join( + path.read_text(encoding="utf-8") for path in (root / "artifacts" / "inner-attribution").rglob("*") + if path.is_file()) + assert "optimizer-secret" not in audit_text + assert "[REDACTED]" in audit_text + assert not list((root / "artifacts" / "inner-attribution").rglob("prompt_sandbox")) + + +@pytest.mark.asyncio +async def test_custom_backend_is_isolated_and_has_unknown_accounting(tmp_path) -> None: + root = _example(tmp_path) + delegate = FakeEvaluationBackend() + + class MutatingBackend: + + async def evaluate(self, **kwargs): + raw = await delegate.evaluate(**kwargs) + kwargs["eval_set"].eval_cases.clear() + kwargs["eval_config"].num_runs = 9 + kwargs["prompts"].clear() + return raw + + report = await run_pipeline( + str(root), + run_id="isolated-custom-backend", + backend=MutatingBackend(), + ) + assert report.status != Decision.ERROR + evaluation_sources = [ + source for source in report.cost.sources if not source.name.startswith("candidate_generation.") + ] + assert len(evaluation_sources) == 4 + assert all(source.cost_usd is None for source in evaluation_sources) + assert all(source.model_calls is None for source in evaluation_sources) + assert report.baseline.train.case_ids == ( + "train_improve_alpha", + "train_improve_beta", + "train_stable", + ) + + +@pytest.mark.asyncio +async def test_custom_generator_unknown_cost_fails_closed_when_budget_enabled(tmp_path) -> None: + root = _example(tmp_path) + config_path = root / "optimizer.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config["pipeline"]["gate"]["maxCostUsd"] = 1 + config_path.write_text(json.dumps(config), encoding="utf-8") + + class UnmeteredGenerator: + + async def generate(self, **kwargs): + known = await DeterministicCandidateGenerator().generate(**kwargs) + payload = known.model_dump(mode="python") + payload.pop("cost_sources") + return CandidateProposal.model_validate(payload) + + report = await run_pipeline( + str(root), + run_id="unknown-candidate-cost", + candidate_generator=UnmeteredGenerator(), + ) + source = next(item for item in report.cost.sources if item.name == "candidate_generation.unreported") + assert source.cost_usd is None + assert source.model_calls is None + assert report.status == Decision.REJECT + assert "COST_UNAVAILABLE" in report.gate_decision.reasons + + +@pytest.mark.parametrize("object_id_length", (40, 64)) +def test_reproducibility_command_contains_all_effective_cli_inputs( + tmp_path, + monkeypatch, + object_id_length, +) -> None: + repo = tmp_path / "repo with space" + root = repo / "examples" / "optimization" / "eval_optimize_loop" + root.mkdir(parents=True) + + def fake_run(args, **kwargs): + if args[-1] == "HEAD": + stdout = "a" * object_id_length + "\n" + elif args[-1] == "--porcelain": + stdout = "" + elif args[-1] == "--show-toplevel": + stdout = str(repo) + "\n" + elif args[1] == "ls-files": + stdout = args[-1] + "\n" + else: + raise AssertionError(args) + return subprocess.CompletedProcess(args, 0, stdout=stdout, stderr="") + + monkeypatch.setattr(reporting_module.subprocess, "run", fake_run) + result = reporting_module.build_reproducibility( + str(root), + mode="live", + config_path=str(root / "custom config.json"), + train_path=str(root / "custom train.json"), + validation_path=str(root / "custom validation.json"), + run_id="exact-replay", + apply_candidate=True, + callback_spec="package.agent:call_agent", + input_paths=( + str(root / "custom config.json"), + str(root / "custom train.json"), + str(root / "custom validation.json"), + ), + ) + args = shlex.split(result.command) + assert result.reproducible is True + assert result.git_commit == "a" * object_id_length + assert args[1] == "examples/optimization/eval_optimize_loop/run_pipeline.py" + assert args[args.index("--config") + 1].endswith("custom config.json") + assert args[args.index("--train") + 1].endswith("custom train.json") + assert args[args.index("--validation") + 1].endswith("custom validation.json") + assert args[args.index("--run-id") + 1] == "exact-replay-replay" + assert "--apply-candidate" in args + assert args[args.index("--call-agent") + 1] == "package.agent:call_agent" + + +@pytest.mark.parametrize( + ("input_path", "tracked", "expected_reason"), + ( + ("outside/input.json", True, "input_outside_git"), + ("repo/untracked.json", False, "input_not_tracked"), + ), +) +def test_reproducibility_requires_all_inputs_in_the_pinned_commit( + tmp_path, + monkeypatch, + input_path, + tracked, + expected_reason, +) -> None: + repo = tmp_path / "repo" + root = repo / "example" + root.mkdir(parents=True) + resolved_input = tmp_path / input_path + + def fake_run(args, **kwargs): + if args[-1] == "HEAD": + return subprocess.CompletedProcess(args, 0, stdout="a" * 40 + "\n", stderr="") + if args[-1] == "--porcelain": + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + if args[-1] == "--show-toplevel": + return subprocess.CompletedProcess(args, 0, stdout=str(repo) + "\n", stderr="") + if args[1] == "ls-files": + return subprocess.CompletedProcess(args, 0 if tracked else 1, stdout="", stderr="") + raise AssertionError(args) + + monkeypatch.setattr(reporting_module.subprocess, "run", fake_run) + result = reporting_module.build_reproducibility( + str(root), + mode="fake", + config_path=str(root / "optimizer.json"), + train_path=str(root / "train.json"), + validation_path=str(root / "validation.json"), + run_id="replay", + apply_candidate=False, + input_paths=(str(resolved_input), ), + ) + assert result.reproducible is False + assert result.reason == expected_reason + assert result.command is None + + +@pytest.mark.asyncio +async def test_generator_baseline_drift_is_rejected_at_stage_boundary(tmp_path) -> None: + root = _example(tmp_path) + + class DriftingGenerator: + + async def generate(self, **kwargs): + candidate = {"system": "candidate"} + return CandidateProposal( + algorithm="drifting-test-generator", + baseline_prompts={"system": "stale baseline"}, + prompts=candidate, + changed=True, + rounds=(CandidateRound( + round=1, + candidate_prompts=candidate, + accepted=True, + score=1, + ), ), + ) + + report = await run_pipeline( + str(root), + run_id="candidate-baseline-drift", + candidate_generator=DriftingGenerator(), + ) + assert report.status == Decision.ERROR + assert report.stage == "candidate_generation" + assert "baseline prompts differ" in report.errors[0].message + + +def test_default_run_id_includes_effective_apply_setting(tmp_path) -> None: + root = _example(tmp_path) + common = { + "config_path": None, + "train_path": None, + "validation_path": None, + "mode": None, + "run_id": None, + "call_agent": None, + "callback_spec": None, + "backend": None, + "candidate_generator": None, + } + dry_run = preflight_run(str(root), apply_candidate=False, **common) + applying_run = preflight_run(str(root), apply_candidate=True, **common) + assert dry_run.run_id != applying_run.run_id + + first_callback = preflight_run( + str(root), + apply_candidate=False, + **{ + **common, + "mode": "live", + "call_agent": importable_call_agent, + "callback_spec": f"{__name__}:importable_call_agent", + }, + ) + second_callback = preflight_run( + str(root), + apply_candidate=False, + **{ + **common, + "mode": "live", + "call_agent": alternate_importable_call_agent, + "callback_spec": f"{__name__}:alternate_importable_call_agent", + }, + ) + assert first_callback.run_id != second_callback.run_id + + +def test_preflight_rejects_mismatched_live_callback_identity(tmp_path) -> None: + root = _example(tmp_path) + with pytest.raises(ValueError, match="must resolve to the same function"): + preflight_run( + str(root), + config_path=None, + train_path=None, + validation_path=None, + mode="live", + run_id="mismatched-live-callback", + apply_candidate=False, + call_agent=importable_call_agent, + callback_spec=f"{__name__}:alternate_importable_call_agent", + backend=None, + candidate_generator=None, + ) + + +@pytest.mark.asyncio +async def test_terminal_report_is_committed_once_with_gate_consistent_duration(tmp_path, monkeypatch) -> None: + root = _example(tmp_path) + original = reporting_module.persist_report + complete_writes = 0 + + def persist_with_delay(sink, report): + nonlocal complete_writes + if report.stage == "complete": + complete_writes += 1 + time.sleep(0.05) + return original(sink, report) + + monkeypatch.setattr(reporting_module, "persist_report", persist_with_delay) + report = await run_pipeline(str(root), run_id="duration-includes-report") + duration_check = next(check for check in report.gate_decision.checks if check.code == "DURATION_BUDGET") + assert complete_writes == 1 + assert report.duration_seconds == pytest.approx(duration_check.observed, abs=1e-12) + assert report.inputs["environment"]["sdk"] diff --git a/tests/examples/eval_optimize_loop/test_workspace_artifacts.py b/tests/examples/eval_optimize_loop/test_workspace_artifacts.py new file mode 100644 index 000000000..78c120aba --- /dev/null +++ b/tests/examples/eval_optimize_loop/test_workspace_artifacts.py @@ -0,0 +1,436 @@ +"""Prompt lifecycle, audit persistence, sanitization and cost tests.""" + +from __future__ import annotations + +import hashlib +import json +import multiprocessing +import os +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +from trpc_agent_sdk.evaluation import TargetPrompt + +from examples.optimization.eval_optimize_loop.pipeline.artifacts import AuditSink +from examples.optimization.eval_optimize_loop.pipeline import artifacts as artifacts_module +from examples.optimization.eval_optimize_loop.pipeline.costing import CostLedger +from examples.optimization.eval_optimize_loop.pipeline.prompt_workspace import ( + PromptRestoreError, + PromptRunLock, + PromptWorkspace, +) +from examples.optimization.eval_optimize_loop.pipeline.schema import sanitized_text + + +def _directory_link_or_emulation(link: Path, target: Path, monkeypatch) -> None: + try: + link.symlink_to(target, target_is_directory=True) + except OSError: + link.mkdir() + original = getattr(Path, "is_junction", None) + monkeypatch.setattr( + Path, + "is_junction", + lambda path: path == link or (bool(original(path)) if original else False), + raising=False, + ) + + +def _contend_for_prompt_lock(prompt_path: str, lock_root: str, start, release, outcomes) -> None: + lock = PromptRunLock((prompt_path, ), lock_root=lock_root) + start.wait(timeout=10) + try: + lock.acquire() + except RuntimeError: + outcomes.put("blocked") + return + try: + outcomes.put("acquired") + release.wait(timeout=10) + finally: + lock.release() + + +@pytest.mark.asyncio +async def test_temporary_candidate_always_restores_baseline(tmp_path) -> None: + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + workspace = PromptWorkspace(TargetPrompt().add_path("system", str(prompt_path))) + await workspace.initialize() + with pytest.raises(RuntimeError, match="candidate failed"): + async with workspace.temporary({"system": "candidate"}): + assert prompt_path.read_text(encoding="utf-8") == "candidate" + raise RuntimeError("candidate failed") + assert prompt_path.read_text(encoding="utf-8") == "baseline" + assert await workspace.current_hashes() == workspace.baseline_hashes + + +@pytest.mark.asyncio +async def test_apply_is_verified_and_can_be_rolled_back(tmp_path) -> None: + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + workspace = PromptWorkspace(TargetPrompt().add_path("system", str(prompt_path))) + await workspace.initialize() + hashes = await workspace.apply({"system": "candidate"}) + assert hashes["system"] == hashlib.sha256(b"candidate").hexdigest() + await workspace.restore() + assert prompt_path.read_text(encoding="utf-8") == "baseline" + + +def test_prompt_run_lock_rejects_concurrent_owner_and_releases(tmp_path) -> None: + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + lock_root = str(tmp_path / "locks") + first = PromptRunLock((str(prompt_path), ), lock_root=lock_root) + second = PromptRunLock((str(prompt_path), ), lock_root=lock_root) + with first: + with pytest.raises(RuntimeError, match="already owned"): + second.acquire() + with second: + assert second.path == first.path + + +def test_prompt_run_lock_first_acquire_has_one_sentinel_byte(tmp_path) -> None: + prompt_path = tmp_path / "system.md" + prompt_path.write_text("baseline", encoding="utf-8") + lock_root = str(tmp_path / "locks") + context = multiprocessing.get_context("spawn") + start = context.Event() + release = context.Event() + outcomes = context.Queue() + processes = [ + context.Process( + target=_contend_for_prompt_lock, + args=(str(prompt_path), lock_root, start, release, outcomes), + ) for _ in range(2) + ] + for process in processes: + process.start() + start.set() + try: + results = [outcomes.get(timeout=15) for _ in processes] + finally: + release.set() + for process in processes: + process.join(timeout=15) + + assert sorted(results) == ["acquired", "blocked"] + assert all(process.exitcode == 0 for process in processes) + lock_path = PromptRunLock((str(prompt_path), ), lock_root=lock_root).path + assert lock_path.read_bytes() == b"\0" + + +@pytest.mark.asyncio +async def test_restoration_failure_preserves_the_primary_operation_diagnostic() -> None: + state = {"value": "baseline", "fail_restore": False} + + async def read() -> str: + return state["value"] + + async def write(value: str) -> None: + if value == "baseline" and state["fail_restore"]: + raise OSError("restore unavailable") + state["value"] = value + + workspace = PromptWorkspace(TargetPrompt().add_callback("system", read=read, write=write)) + await workspace.initialize() + with pytest.raises(PromptRestoreError) as raised: + async with workspace.temporary({"system": "candidate"}): + state["fail_restore"] = True + raise RuntimeError("candidate evaluation failed") + message = sanitized_text(raised.value, max_text_chars=4000) + assert "candidate evaluation failed" in message + assert "restore unavailable" in message + + +@pytest.mark.asyncio +async def test_apply_double_failure_preserves_write_and_restore_diagnostics() -> None: + async def read() -> str: + return "baseline" + + async def write(value: str) -> None: + if value == "candidate": + raise OSError("candidate write unavailable") + raise OSError("restore unavailable") + + workspace = PromptWorkspace(TargetPrompt().add_callback("system", read=read, write=write)) + await workspace.initialize() + with pytest.raises(PromptRestoreError) as raised: + await workspace.apply({"system": "candidate"}) + message = sanitized_text(raised.value, max_text_chars=4000) + assert "candidate write unavailable" in message + assert "restore unavailable" in message + + +def test_audit_sink_is_immutable_safe_sanitized_and_manifested(tmp_path) -> None: + sink = AuditSink(tmp_path / "artifacts", "run-1") + sink.create() + sink.write_json( + "config.json", + { + "api_key": "secret-value", + "nested": { + "message": "Authorization: Bearer abc.def" + }, + "tokenUsage": { + "total": 12 + }, + }, + ) + sink.write_text("notes.txt", "token=abc123") + with pytest.raises(ValueError, match="unsafe|path"): + sink.write_text("../escape.txt", "bad") + records = sink.write_manifest() + payload = json.loads((sink.run_dir / "config.json").read_text(encoding="utf-8")) + assert payload["api_key"] == "[REDACTED]" + assert "abc.def" not in payload["nested"]["message"] + assert payload["tokenUsage"] == {"total": 12} + assert {record.path for record in records} == {"config.json", "notes.txt"} + for record in records: + content = (sink.run_dir / record.path).read_bytes() + assert record.sha256 == hashlib.sha256(content).hexdigest() + assert record.byte_size == len(content) + with pytest.raises(FileExistsError): + AuditSink(tmp_path / "artifacts", "run-1").create() + + +def test_audit_sink_preserves_complete_text_and_fails_explicitly_on_size_limit(tmp_path) -> None: + content = "complete-audit-content:" + "x" * 256 + sink = AuditSink(tmp_path / "artifacts", "complete", max_file_bytes=1024) + sink.create() + sink.write_text("prompt.md", content) + sink.write_json("prompt.json", {"prompt": content}) + assert (sink.run_dir / "prompt.md").read_text(encoding="utf-8") == content + assert json.loads((sink.run_dir / "prompt.json").read_text(encoding="utf-8"))["prompt"] == content + + limited = AuditSink(tmp_path / "artifacts", "limited", max_file_bytes=64) + limited.create() + with pytest.raises(ValueError, match="audit file exceeds byte limit"): + limited.write_text("prompt.md", content) + assert not (limited.run_dir / "prompt.md").exists() + + +def test_optimizer_tree_is_sanitized_before_audit_import(tmp_path) -> None: + raw = tmp_path / "raw-optimizer" + raw.mkdir() + (raw / "config.snapshot.json").write_text( + json.dumps({"optimize": { + "apiKey": "live-secret" + }}), + encoding="utf-8", + ) + (raw / "run.log").write_text( + 'Authorization: Bearer live-token\n{"apiKey":"quoted-log-secret"}', + encoding="utf-8", + ) + (raw / "summary.md").write_text( + "result: {'clientSecret': 'markdown-secret'}", + encoding="utf-8", + ) + sink = AuditSink(tmp_path / "artifacts", "run-2") + sink.create() + sink.import_tree(raw, "candidate_generation/optimizer") + imported = "\n".join(path.read_text(encoding="utf-8") for path in sink.run_dir.rglob("*") if path.is_file()) + assert "live-secret" not in imported + assert "live-token" not in imported + assert "quoted-log-secret" not in imported + assert "markdown-secret" not in imported + assert "[REDACTED]" in imported + + +def test_optimizer_tree_rejects_source_directory_symlink(tmp_path, monkeypatch) -> None: + raw = tmp_path / "raw-optimizer" + raw.mkdir() + (raw / "secret.log").write_text("secret", encoding="utf-8") + alias = tmp_path / "optimizer-alias" + _directory_link_or_emulation(alias, raw, monkeypatch) + + sink = AuditSink(tmp_path / "artifacts", "run-root-link") + sink.create() + with pytest.raises(ValueError, match="symlinks"): + sink.import_tree(alias, "candidate_generation/optimizer") + + +def test_optimizer_tree_rejects_nested_directory_symlink(tmp_path, monkeypatch) -> None: + raw = tmp_path / "raw-optimizer" + external = tmp_path / "external" + raw.mkdir() + external.mkdir() + (external / "secret.log").write_text("secret", encoding="utf-8") + _directory_link_or_emulation(raw / "linked", external, monkeypatch) + + sink = AuditSink(tmp_path / "artifacts", "run-nested-link") + sink.create() + with pytest.raises(ValueError, match="symlinks"): + sink.import_tree(raw, "candidate_generation/optimizer") + + +def test_optimizer_tree_rejects_nested_directory_junction(tmp_path, monkeypatch) -> None: + raw = tmp_path / "raw-optimizer" + linked = raw / "linked" + linked.mkdir(parents=True) + (linked / "outside.log").write_text("outside", encoding="utf-8") + original = getattr(Path, "is_junction", None) + + def is_junction(path: Path) -> bool: + return path == linked or (bool(original(path)) if original else False) + + monkeypatch.setattr(Path, "is_junction", is_junction, raising=False) + sink = AuditSink(tmp_path / "artifacts", "run-nested-junction") + sink.create() + + with pytest.raises(ValueError, match="reparse points"): + sink.import_tree(raw, "candidate_generation/optimizer") + + +def test_optimizer_tree_rejects_broken_source_junction(tmp_path, monkeypatch) -> None: + junction = tmp_path / "broken-junction" + original = getattr(Path, "is_junction", None) + monkeypatch.setattr( + Path, + "is_junction", + lambda path: path == junction or (bool(original(path)) if original else False), + raising=False, + ) + sink = AuditSink(tmp_path / "artifacts", "run-broken-junction") + sink.create() + + with pytest.raises(ValueError, match="reparse points"): + sink.import_tree(junction, "candidate_generation/optimizer") + + +def test_optimizer_tree_rejects_hard_link_to_external_file(tmp_path) -> None: + raw = tmp_path / "raw-optimizer" + raw.mkdir() + external = tmp_path / "external.log" + external.write_text("outside-content", encoding="utf-8") + os.link(external, raw / "outside.log") + + sink = AuditSink(tmp_path / "artifacts", "run-hard-link") + sink.create() + with pytest.raises(ValueError, match="hard links"): + sink.import_tree(raw, "candidate_generation/optimizer") + + +def test_optimizer_tree_validates_all_files_before_publishing(tmp_path) -> None: + raw = tmp_path / "raw-optimizer" + raw.mkdir() + (raw / "valid.log").write_text("valid", encoding="utf-8") + (raw / "invalid.bin").write_bytes(b"invalid") + + sink = AuditSink(tmp_path / "artifacts", "run-batch-validation") + sink.create() + with pytest.raises(ValueError, match="unsupported"): + sink.import_tree(raw, "candidate_generation/optimizer") + assert not (sink.run_dir / "candidate_generation").exists() + + +def test_optimizer_tree_rejects_file_identity_change_before_read(tmp_path, monkeypatch) -> None: + raw = tmp_path / "raw-optimizer" + raw.mkdir() + changed = raw / "changed.log" + changed.write_text("before", encoding="utf-8") + original_open = artifacts_module.os.open + + def replace_before_open(path, flags, *args): + if Path(path) == changed: + changed.write_text("after", encoding="utf-8") + return original_open(path, flags, *args) + + monkeypatch.setattr(artifacts_module.os, "open", replace_before_open) + sink = AuditSink(tmp_path / "artifacts", "run-identity-change") + sink.create() + with pytest.raises(ValueError, match="changed during validation"): + sink.import_tree(raw, "candidate_generation/optimizer") + + +@pytest.mark.parametrize( + ("sink_kwargs", "files", "message"), + [ + ({ + "max_import_files": 1 + }, { + "a.log": "a", + "b.log": "b" + }, "count"), + ({ + "max_import_file_bytes": 3 + }, { + "large.log": "1234" + }, "per-file"), + ( + { + "max_import_file_bytes": 4, + "max_import_total_bytes": 5 + }, + { + "a.log": "123", + "b.log": "456" + }, + "total", + ), + ], +) +def test_optimizer_tree_enforces_resource_budgets(tmp_path, sink_kwargs, files, message) -> None: + raw = tmp_path / "raw-budget" + raw.mkdir() + for name, content in files.items(): + (raw / name).write_text(content, encoding="utf-8") + sink = AuditSink( + tmp_path / "artifacts", + "run-budget", + **sink_kwargs, + ) + sink.create() + with pytest.raises(ValueError, match=message): + sink.import_tree(raw, "candidate_generation/optimizer") + + +def test_concurrent_latest_publication_uses_collision_free_atomic_files(tmp_path) -> None: + publication_root = tmp_path / "publication" + sinks = [] + for index in range(2): + sink = AuditSink( + tmp_path / "artifacts", + f"run-{index}", + publication_root=publication_root, + ) + sink.create() + sink.write_text("report.md", f"run-{index}\n") + sinks.append(sink) + + with ThreadPoolExecutor(max_workers=8) as executor: + futures = [ + executor.submit( + sinks[index % 2].publish_latest_snapshot, + "report.md", + "optimization_report.md", + ) for index in range(40) + ] + for future in futures: + future.result() + + assert (publication_root / "optimization_report.md").read_text(encoding="utf-8") in { + "run-0\n", + "run-1\n", + } + assert not list(publication_root.glob("*.tmp")) + + +def test_cost_ledger_preserves_unknown_sources() -> None: + ledger = CostLedger() + ledger.record("fake", cost_usd=0, metric_calls=3) + ledger.record("live-judge", cost_usd=None, model_calls=1) + summary = ledger.summary() + assert summary.total_cost_usd is None + assert [source.name for source in summary.sources] == ["fake", "live-judge"] + + +def test_cost_ledger_preserves_upper_bound_basis() -> None: + ledger = CostLedger() + ledger.record("live-evaluation", cost_usd=0.5, upper_bound=True, model_calls=2) + summary = ledger.summary() + assert summary.total_cost_usd == 0.5 + assert summary.sources[0].upper_bound is True