From cc3bd03bdf951fd8d1526485fb31b272bae5339c Mon Sep 17 00:00:00 2001 From: ygrowly Date: Wed, 29 Jul 2026 17:52:52 +0800 Subject: [PATCH 1/6] feat: add evaluation optimization loop example --- .claude/settings.local.json | 27 + Evaluation + Optimization.md | 32 + eval_optimize_analysis.md | 671 ++++++++++++++++++ .../eval_optimize_loop/.gitignore | 9 + .../optimization/eval_optimize_loop/DESIGN.md | 191 +++++ .../optimization/eval_optimize_loop/README.md | 78 ++ .../eval_optimize_loop/attribution.py | 293 ++++++++ .../data/attribution_holdout.json | 73 ++ .../eval_optimize_loop/data/test_config.json | 17 + .../data/train.evalset.json | 88 +++ .../eval_optimize_loop/data/val.evalset.json | 88 +++ .../eval_optimize_loop/fake_agent.py | 221 ++++++ .../optimization/eval_optimize_loop/gates.py | 180 +++++ .../eval_optimize_loop/live_agent.py | 102 +++ .../optimization_report.example.json | 345 +++++++++ .../eval_optimize_loop/optimizer.json | 46 ++ .../eval_optimize_loop/pipeline.py | 548 ++++++++++++++ .../eval_optimize_loop/prompts/system.md | 16 + .../optimization/eval_optimize_loop/report.py | 291 ++++++++ .../optimization/eval_optimize_loop/run.json | 15 + .../optimization/eval_optimize_loop/runner.py | 612 ++++++++++++++++ .../eval_optimize_loop/tests/conftest.py | 61 ++ .../eval_optimize_loop/tests/test_apply.py | 123 ++++ .../tests/test_attribution.py | 114 +++ .../eval_optimize_loop/tests/test_dataset.py | 64 ++ .../tests/test_evaluator_evidence.py | 260 +++++++ .../tests/test_fake_agent.py | 264 +++++++ .../eval_optimize_loop/tests/test_gates.py | 205 ++++++ .../tests/test_native_optimize.py | 231 ++++++ .../tests/test_report_fields.py | 148 ++++ .../tests/test_sha256_normalize.py | 50 ++ .../tests/test_truth_table.py | 140 ++++ 32 files changed, 5603 insertions(+) create mode 100644 .claude/settings.local.json create mode 100644 Evaluation + Optimization.md create mode 100644 eval_optimize_analysis.md create mode 100644 examples/optimization/eval_optimize_loop/.gitignore create mode 100644 examples/optimization/eval_optimize_loop/DESIGN.md create mode 100644 examples/optimization/eval_optimize_loop/README.md create mode 100644 examples/optimization/eval_optimize_loop/attribution.py create mode 100644 examples/optimization/eval_optimize_loop/data/attribution_holdout.json create mode 100644 examples/optimization/eval_optimize_loop/data/test_config.json create mode 100644 examples/optimization/eval_optimize_loop/data/train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/val.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/fake_agent.py create mode 100644 examples/optimization/eval_optimize_loop/gates.py create mode 100644 examples/optimization/eval_optimize_loop/live_agent.py create mode 100644 examples/optimization/eval_optimize_loop/optimization_report.example.json create mode 100644 examples/optimization/eval_optimize_loop/optimizer.json create mode 100644 examples/optimization/eval_optimize_loop/pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/prompts/system.md create mode 100644 examples/optimization/eval_optimize_loop/report.py create mode 100644 examples/optimization/eval_optimize_loop/run.json create mode 100644 examples/optimization/eval_optimize_loop/runner.py create mode 100644 examples/optimization/eval_optimize_loop/tests/conftest.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_apply.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_attribution.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_dataset.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_evaluator_evidence.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_fake_agent.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_gates.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_native_optimize.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_report_fields.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_sha256_normalize.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_truth_table.py diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 000000000..43bc8d046 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,27 @@ +{ + "permissions": { + "allow": [ + "Bash(python -m pip install:*)", + "Bash(where:*)", + "Bash(python:*)", + "Bash(\"E:/code/study/trpc-agent-python/.venv/Scripts/python.exe\" -m pip --version:*)", + "Bash(\"E:/code/study/trpc-agent-python/.venv/Scripts/python.exe\" -m pip install -r E:/code/study/trpc-agent-python/requirements-test.txt)", + "Bash(\"E:/code/study/trpc-agent-python/.venv/Scripts/python.exe\" -c:*)", + "WebSearch", + "Bash(\"E:/code/study/trpc-agent-python/.venv/Scripts/python.exe\" -m pytest tests -q --tb=no -p no:cacheprovider)", + "Bash(tee:*)", + "Bash(powershell:*)", + "Bash(\"E:/code/study/trpc-agent-python/.venv/Scripts/python.exe\" -m pytest:*)", + "Bash(Tee-Object:*)", + "Bash(pytest_eval.log)", + "Bash(\"E:\\code\\study\\trpc-agent-python\\.venv\\Scripts\\python.exe\" -m pytest \"E:\\code\\study\\trpc-agent-python\\examples\\optimization\\eval_optimize_loop\\tests\" -q --tb=line)", + "Bash(E:codestudytrpc-agent-pythonpytest_out.txt)", + "Bash(Get-Content E:codestudytrpc-agent-pythonpytest_out.txt -Tail 100)", + "Bash(\"E:\\code\\study\\trpc-agent-python\\pytest_out.txt\")", + "Bash(\"E:\\code\\study\\trpc-agent-python\\.venv\\Scripts\\python.exe\" -m pytest:*)", + "Bash(\"E:\\code\\study\\trpc-agent-python\\.venv\\Scripts\\python.exe\" -c \"import sys; sys.path.insert(0, r''E:\\code\\study\\trpc-agent-python\\examples\\optimization\\eval_optimize_loop''); from runner import sha256_text, sha256_file; print(sha256_text(''a\\nb\\n''))\")" + ], + "deny": [], + "ask": [] + } +} diff --git a/Evaluation + Optimization.md b/Evaluation + Optimization.md new file mode 100644 index 000000000..546ab211c --- /dev/null +++ b/Evaluation + Optimization.md @@ -0,0 +1,32 @@ +背景和价值 +tRPC-Agent 已提供 AgentEvaluator 和 AgentOptimizer 两类关键能力：前者负责把 Agent 行为固化成可回归的评测信号，后者基于评测结果自动搜索更优 prompt、skill 描述或 sub-agent 指令。真实业务中，评测和优化不能割裂：如果评测集质量差，优化器会过拟合；如果优化过程不可审计，改出来的 prompt 即使分数变高也很难进入生产。 +该题要求构建一个“评测 - 失败归因 - prompt 优化 - 回归验证 - 产物审计”的自动闭环。它不是简单跑一次 AgentOptimizer，而是要判断优化是否真的提升、是否牺牲其他指标、是否出现过拟合、是否值得回写源 prompt。 +任务描述 +设计并实现一个可复现的 Evaluation + Optimization pipeline。输入 baseline prompt、训练评测集、验证评测集和优化配置，系统自动运行 baseline 评测、定位失败 case、执行若干轮优化、对候选 prompt 做验证集回归，并输出结构化优化报告和是否接受候选的决策。 +具体要求 +pipeline 至少需要包含以下阶段： +1.Baseline 评测：使用 AgentEvaluator 对训练集和验证集分别打分，记录每条 case 的 metric 分、pass/fail、失败原因和关键轨迹。 +2.失败归因：按失败类型聚类，例如最终回复不匹配、工具调用错误、参数错误、LLM rubric 不达标、知识召回不足、格式不符合要求。 +3.优化执行：使用 AgentOptimizer 或等价扩展机制优化至少一个 TargetPrompt，支持 system prompt、skill prompt、router prompt 中的一种或多种。 +4.候选验证：优化后必须重新跑验证集，并和 baseline 做逐 case 对比，区分新增通过、新增失败、分数提升、分数下降。 +5.接受策略：实现可配置 gate，例如验证集总分提升 ≥ 指定阈值、不能新增 hard fail、关键 case 不能退化、成本不能超过预算。 +6.审计落盘：保存每轮候选 prompt、评测结果、接受/拒绝理由、运行成本、耗时、随机种子或复现实验配置。 +输入输出要求： +●输入包含 train.evalset.json、val.evalset.json、optimizer.json 和 prompt 源文件。 +●输出 optimization_report.json，包含 baseline、candidate、delta、gate decision、失败归因统计。 +●输出 optimization_report.md，用人能读懂的方式说明优化是否值得接受。 +●支持 fake judge / fake model / trace mode，保证没有真实 API Key 时也能跑通核心流程。 +交付物 +●新增示例目录，例如 examples/optimization/eval_optimize_loop/。 +●pipeline 入口脚本、样例 evalset、样例 prompt、优化配置和 README。 +●至少 6 条评测 case：3 条训练、3 条验证，其中需要包含可优化成功、优化无效、优化后退化三类情况。 +●optimization_report.json 示例输出。 +●一份 300 – 500 字方案设计说明，解释失败归因方法、接受策略、防过拟合策略和产物审计方式。 +验收标准 +1.公开提供的 6 条样例 case 必须全部可运行，并生成完整优化报告。 +2.在隐藏样本上，优化接受/拒绝决策准确率 ≥ 80%。 +3.对“验证集退化但训练集提升”的过拟合场景，必须能拒绝候选 prompt。 +4.失败归因分类准确率 ≥ 75%，且每个失败 case 至少能给出一个可解释原因。 +5.fake model / trace mode 下完整 pipeline 耗时 ≤ 3 分钟。 +6.报告必须包含 baseline 分数、candidate 分数、逐 case delta、gate 决策、拒绝或接受理由。 + diff --git a/eval_optimize_analysis.md b/eval_optimize_analysis.md new file mode 100644 index 000000000..3d02b9423 --- /dev/null +++ b/eval_optimize_analysis.md @@ -0,0 +1,671 @@ +# Evaluation + Optimization 分析报告 + +> 本报告基于 `trpc_agent_sdk/evaluation/`、`tests/evaluation/`、`examples/optimization/` 的源码与 `pytest_full.log` 的实测数据，回答四个问题： +> 1. AgentEvaluator 与 AgentOptimizer 的相关文件路径及职责 +> 2. 一条从 evalset 输入到评测结果、再到优化器输出的真实调用链 +> 3. evaluation 测试失败的完整 traceback +> 4. #91 新增示例目录应放在哪里 +> +> 本报告**不修改代码、不修复 Windows 平台问题**，仅做架构梳理与决策建议。 + +--- + +## 0. 任务规格回顾（来自 `Evaluation + Optimization.md`） + +构建"评测 → 失败归因 → prompt 优化 → 回归验证 → 产物审计"的自动闭环。输入 `train.evalset.json` / `val.evalset.json` / `optimizer.json` / 源 prompt，输出 `optimization_report.json` / `.md`，要求： + +- 6 条 case：3 训练 + 3 验证，覆盖**可优化成功 / 优化无效 / 优化后退化**三类场景 +- fake judge / fake model / trace mode 下 ≤3 分钟跑通 +- 隐藏样本上接受/拒绝决策准确率 ≥80% +- 对"验证集退化但训练集提升"的过拟合场景必须拒绝 +- 失败归因分类准确率 ≥75% +- 报告必须包含 baseline 分数、candidate 分数、逐 case delta、gate 决策、拒绝/接受理由 + +--- + +## 1. 相关文件路径及职责 + +### 1.1 核心模块（`trpc_agent_sdk/evaluation/`） + +| 文件 | 职责 | +|---|---| +| `__init__.py` | 对外总入口，导出 `AgentEvaluator` / `AgentOptimizer` / `TargetPrompt` / `EvalSet` / `EvalCase` / `EvaluateResult` / `OptimizeResult` 等 ~150 个符号 | +| `_agent_evaluator.py` | **评测入口**。`AgentEvaluator.evaluate()` / `get_executer()` 静态方法；`_EvalExecuter._run()`（L140-231）发现 `.evalset.json` → 逐文件加载 → 调 `evaluate_eval_set()` → 汇总打印 → 若有失败 `raise _EvaluationCasesFailed`（L78-93，`AssertionError` 子类，兼容 CI `except AssertionError`）| +| `_agent_optimizer.py` | **优化器门面**。`AgentOptimizer.optimize()` 类方法（L132-296）：预检算法名 → 加载 `OptimizeConfigFile` → 校验输入（`_validate_inputs` L544-614）→ 从 `OPTIMIZER_REGISTRY` 实例化算法 → 调 `optimizer.run(reporter=...)` → 写 `result.json` / `summary.txt` / `rounds/` / `best_prompts/` / `baseline_prompts/` / `config.snapshot.json` / `run.log`。`_mask_sigint`（L72-109）保证 Ctrl+C 期间产物落盘 | +| `_optimize_evaluator_call.py` | **桥接层**。`run_evaluator()`（L83-136）= 优化器调评测器的标准入口：`AgentEvaluator.get_executer()` → `await executer.evaluate()` → 捕获 `_EvaluationCasesFailed`（业务信号，吞掉）→ 其它异常透传 → `summarize_outcome(result)` 输出 `EvaluationOutcome`（`pass_rate` / `tiebreaker` / `metric_breakdown` / `failed_case_ids`）| +| `_base_optimizer.py` | 抽象基类 `BaseOptimizer`。定义 `__init__` 接收 config / call_agent / target_prompt / train / val 路径；声明抽象 `run(reporter=)`；提供 `resolve_required_thresholds`（L80-106）与 `metrics_meet_thresholds`（L108-123）给 framework-level stop policy 复用 | +| `_optimize_gepa_reflective.py` | **GEPA 反思算法实现**（当前 `OPTIMIZER_REGISTRY` 唯一注册项）。`GepaReflectiveOptimizer.run()`（L431-）→ 读 baseline prompts → 加载 train/val evalset → 构造 `_AgentGEPAAdapter` + `_OptimizeModelCallable` → 在 worker thread 调 `gepa.optimize(...)`（L425-429，`asyncio.to_thread`）；`_build_stop_callbacks`（L294-381）把 `max_metric_calls` / `no_improvement` / `timeout` / `score_threshold` / `max_candidate_proposals` / `max_tracked_candidates` / `optimize.stop` 文件 / framework `required_metrics` 都翻译成 gepa `StopperProtocol` | +| `_optimize_gepa_adapter.py` | **GEPA 协议适配器**。`_AgentGEPAAdapter` 实现 `gepa.core.adapter.GEPAAdapter`：`evaluate(candidate, subsample)` 调 `run_evaluator`；`make_reflective_dataset()`（L13-）把失败 case 渲染成 turn-sliced markdown 喂给 reflection LM（含 PASS/FAIL 行、rubric 子项分、synthesized failure reason） | +| `_optimize_gepa_callback.py` | `_AgentGEPACallback`：gepa 事件 → `RoundRecord` 实时缓冲，是 `_build_optimize_result` 的 round 数据首选来源 | +| `_optimize_model_callable.py` | `_OptimizeModelCallable`：把 `OptimizeModelOptions` 包装成 gepa 可调用的 reflection LM | +| `_optimize_config.py` | 配置 schema：`OptimizeConfigFile`（顶层）/ `EvalConfig`（评测）/ `FrameworkStopConfig`（`stop.required_metrics`）/ `GepaReflectiveAlgo`（算法超参，含 `max_metric_calls` / `max_iterations_without_improvement` / `reflection_minibatch_size` 等）| +| `_optimize_registry.py` | `OPTIMIZER_REGISTRY = OptimizerRegistry()`；`_optimize_registrations.py` 触发 gepa_reflective 注册 | +| `_optimize_reporter.py` | `OptimizeReporter` Rich 面板 + ASCII fallback；`RoundView` / `RunHeader` 数据类 | +| `_optimize_result.py` | 结果 schema：`OptimizeResult`（顶层）+ `RoundRecord`（每轮）+ `StopReason` / `RunStatus` / `FinishReason` Literal 类型 | +| `_target_prompt.py` | **多字段 prompt 注册表**。`TargetPrompt.add_path(name, path)` / `.add_callback(name, read=, write=)`；`read_all()` / `write_all()` 原子（path-backed 用 tmp+`os.replace`，部分失败时回滚）| +| `_eval_set.py` | `EvalSet` Pydantic 模型：`eval_set_id` / `app_name?` / `name?` / `description?` / `eval_cases: list[EvalCase]` | +| `_eval_case.py` | `EvalCase` / `Invocation` / `ConversationScenario` / `StaticConversation`；`EvalModeTrace = "trace"`（不跑 agent，直接评测预录对话）| +| `_eval_config.py` | `EvalConfig`：`criteria` (旧式) 或 `metrics` (新式) + `num_runs` | +| `_eval_metrics.py` | `EvalMetric` / `EvalStatus(PASSED\|FAILED)` / `PrebuiltMetrics` 枚举（`final_response_avg_score` 等）| +| `_eval_result.py` | `EvalCaseResult` / `EvalSetAggregateResult` / `EvaluateResult` / `EvalMetricResult`（含 score/threshold/eval_status/details）| +| `_eval_service_base.py` | `BaseEvalService` / `InferenceRequest` / `EvaluateRequest` / `InferenceConfig` / `EvaluateConfig` | +| `_local_eval_service.py` | `LocalEvalService`：本地驱动 agent 推理 + 评测 | +| `_remote_eval_service.py` | `RemoteEvalService` + `CallAgent = Callable[[str], Awaitable[str]]`：黑盒模式，业务回调驱动 agent | +| `_final_response_evaluator.py` | `FinalResponseEvaluator`：精确 / 包含 / 正则 / JSON 匹配 | +| `_trajectory_evaluator.py` | `TrajectoryEvaluator`：工具调用轨迹评分（需 session traces）| +| `_rouge_evaluator.py` | `RougeEvaluator`：ROUGE 分 | +| `_llm_criterion.py` / `_llm_evaluator.py` / `_llm_judge.py` | LLM-as-judge 三件套：rubric 打分 / 知识召回 / 最终回复质量，支持 multi-model judges / weighted aggregators | +| `_eval_callbacks.py` | `Callbacks`（生命周期钩子）+ `CallbacksRunner` | +| `_local_eval_sets_manager.py` / `_in_memory_eval_sets_manager.py` / `_local_eval_set_results_manager.py` | evalset 持久化与历史结果管理 | + +### 1.2 测试目录（`tests/evaluation/`，53 个文件） + +按职责分组： + +- **AgentEvaluator 主流程**：`test_agent_evaluator.py`、`test_agent_evaluator_call_agent.py` +- **AgentOptimizer 门面**：`test_agent_optimizer.py` +- **桥接层**：`test_optimize_evaluator_call.py` +- **GEPA 四件套**：`test_optimize_gepa_adapter.py`、`test_optimize_gepa_callback.py`、`test_optimize_gepa_e2e.py`、`test_optimize_gepa_reflective.py` +- **端到端**：`test_optimize_quickstart_example.py`（以 `examples/optimization/quickstart/` 为素材） +- **组件级**：`test_optimize_config.py`、`test_optimize_result.py`、`test_optimize_reporter.py`、`test_optimize_registry.py`、`test_optimize_model_callable.py`、`test_optimize_model_options.py`、`test_optimize_metric_info.py` +- **TargetPrompt**：`test_target_prompt.py` +- **数据模型**：`test_eval_case.py`、`test_eval_set.py`、`test_eval_config.py`、`test_eval_metrics.py`、`test_eval_pass.py`、`test_eval_result.py` +- **回调**：`test_eval_callbacks.py`、`test_eval_callbacks_ext.py` +- **基础设施**：`test_eval_session_service.py`、`test_eval_service_base.py`、`test_local_eval_sets_manager.py`、`test_in_memory_eval_sets_manager.py`、`test_remote_eval_service.py` +- **Evaluator 实现**：`test_final_response_evaluator.py`、`test_trajectory_evaluator.py`、`test_rouge_evaluator.py`、`test_llm_judge*.py`、`test_llm_criterion.py`、`test_llm_evaluator_registry.py` + +### 1.3 现有 examples 目录（`examples/optimization/`） + +``` +examples/optimization/ +├── quickstart/ # 入门示例（小学算术题，2 个 prompt 文件） +├── advanced_strategies/ # 进阶策略（pareto / current_best / merge） +├── blackbox_cli/ # CLI 黑盒 agent 接入 +├── ci_integration/ # pytest + AgentOptimizer 闭环 +├── http_service/ # HTTP 服务形态 +├── multi_agent_pipeline/ # 多 agent 编排 +├── multi_metric_with_judges/ # 多 metric + 独立 judge model +├── remote_prompt_store/ # 远端 prompt 源（Redis / HTTP） +└── slo_runtime_control/ # SLO / budget / kill switch +``` + +每个子目录都有 `README.md` + `optimizer.json` + 可独立运行的入口脚本 + `train.evalset.json` + `val.evalset.json`。 + +--- + +## 2. 真实调用链：evalset → 评测结果 → 优化器输出 + +以 `examples/optimization/quickstart/run_optimization.py` 为例，从 `asyncio.run(main())` 到产物落盘的完整链路： + +### 阶段 A：业务组装（`run_optimization.py`） + +```python +TargetPrompt().add_path("system_prompt", SYSTEM_PROMPT_PATH) + .add_path("skill", SKILL_PATH) +await AgentOptimizer.optimize( + config_path="optimizer.json", + call_agent=call_agent, # 业务回调 + target_prompt=target, + train_dataset_path="train.evalset.json", + validation_dataset_path="val.evalset.json", + output_dir="runs/", + update_source=False, +) +``` + +### 阶段 B：`AgentOptimizer.optimize` 门面（`_agent_optimizer.py:132-296`） + +1. `_precheck_algorithm_name(config_path)` L200 → 读 raw JSON，查 `OPTIMIZER_REGISTRY`，未注册则 fail-fast +2. `load_optimize_config(config_path)` L201 → pydantic 校验，输出 `OptimizeConfigFile` +3. `_validate_inputs(...)` L202-209 → async check / train≠val / disallowed metrics in call_agent mode / use_merge≥2 fields +4. `os.makedirs(output_dir)` L210 +5. `OPTIMIZER_REGISTRY.get("gepa_reflective")` L213 → `GepaReflectiveOptimizer` +6. 实例化 optimizer L214-224，注入 config / call_agent / target_prompt / train / val / output_dir +7. `create_reporter(verbose=1, stream=sys.stdout)` L226 +8. `baseline_snapshot = await target_prompt.read_all()` L227（**关键：快照基线 prompt，供 finally 回滚**） +9. `_build_run_header(...)` L228-235 → 读 train/val evalset 数 case 个数 +10. `reporter.run_started(header)` L236 +11. `try: result = await optimizer.run(reporter=reporter)` L251 +12. `if update_source and result.status == "SUCCEEDED": await target_prompt.write_all(result.best_prompts)` L256-264 +13. `finally:` L272 — 若 cleanup 未完成，回滚到 `baseline_snapshot`；调 `_persist_artifacts` 写产物；`reporter.run_finished/failed` + +### 阶段 C：`GepaReflectiveOptimizer.run`（`_optimize_gepa_reflective.py:431-`） + +1. `baseline_prompts = await self.target_prompt.read_all()` L441 +2. `_load_evalset_cases(train_path)` + `_load_evalset_cases(val_path)` L445-446 — `Path.read_text` + `EvalSet.model_validate_json` → `list[EvalCase]` +3. 构造 `_AgentGEPAAdapter(target_prompt, eval_config, call_agent, callbacks, num_runs, case_parallelism, reflection_history_top_k)` L457-465 +4. 构造 `_OptimizeModelCallable(algo.reflection_lm)` L466 — 包装 reflection LM +5. 调 `_run_with_adapter(...)` L469-481，`finally: adapter.close()` + +### 阶段 D：`_run_with_adapter` → `gepa.optimize(...)` + +- `_build_stop_callbacks(algo, stop_config, metric_thresholds, output_dir)` L294-381 把配置翻译成 gepa stopper 列表 +- `await self._call_gepa_optimize(**kwargs)` L425-429 → `asyncio.to_thread(gepa_optimize, **kwargs)` 把同步 GEPA 主循环扔到 worker thread，不阻塞 surrounding event loop +- GEPA 主循环每轮： + 1. `module_selector` 选要改写的 prompt 字段（`round_robin` 在 system_prompt ↔ skill 间交替） + 2. 从 trainset 抽 `reflection_minibatch_size` 条 case + 3. `adapter.evaluate(parent, subsample)` → **下到阶段 E** + 4. `adapter.make_reflective_dataset(failed_cases)` → 渲染 markdown 反思素材 + 5. reflection LM 生成新候选 prompt → `target_prompt.write_all(candidate)` 写回磁盘 + 6. `adapter.evaluate(candidate, full_valset)` → **下到阶段 E** + 7. Pareto 前沿比较，决定接受 / 拒绝 + 8. `_AgentGEPACallback` 实时缓冲 `RoundRecord` + 9. stop callbacks 投票，命中则退出 + +### 阶段 E：`_AgentGEPAAdapter.evaluate` → `run_evaluator`（`_optimize_evaluator_call.py:83-136`） + +```python +executer = AgentEvaluator.get_executer( + eval_dataset_path, # 临时文件 or val.evalset.json + call_agent=call_agent, # 业务回调 + callbacks=callbacks, + num_runs=num_runs, + print_detailed_results=False, # 优化器静默评测 + print_summary_report=False, + eval_metrics_file_path_or_dir=eval_metrics_path, + case_parallelism=case_parallelism, +) +try: + await executer.evaluate() +except _EvaluationCasesFailed: + pass # 业务信号，吞掉 +result = executer.get_result() +return summarize_outcome(result) # → EvaluationOutcome +``` + +### 阶段 F：`_EvalExecuter._run`（`_agent_evaluator.py:140-231`） + +1. `_resolve_shared_config(eval_metrics_file_path_or_dir)` L156 +2. 遍历 `.evalset.json` 文件 L158-165 +3. 对每个文件：`_load_eval_set_from_file` L187（支持 `file.json:case_id` 选择子集）→ `EvalSet.model_validate_json` +4. `evaluate_eval_set(eval_set, call_agent=call_agent, eval_config=eval_config, ...)` L188-202 +5. 收集 failures，构造 `EvalSetAggregateResult` +6. `_RESULT_HANDLER.print_evaluation_report` 打印 +7. 若 `all_failures`：`raise _EvaluationCasesFailed(json.dumps(...))` L231 — **此时 `self._result` 已被赋值**，优化器能拿到 + +### 阶段 G：`AgentEvaluator.evaluate_eval_set`（`_agent_evaluator.py:470-586`） + +1. 校验：`call_agent` / `agent_module` / `runner` 互斥；trace_only 模式可省 agent +2. `_get_eval_results_by_eval_id(...)` L540-552 — **下到阶段 H** +3. 后处理：`_get_eval_metric_results_with_invocation` L561 翻转 grouping +4. `_RESULT_HANDLER.process_metrics_and_get_failures` L563-570 提取失败 case +5. `build_summary` + `build_evaluation_result_lines` 输出表格 +6. 返回 `(failed_summary, details_lines, result_lines, eval_results_by_eval_id)` + +### 阶段 H：`_get_eval_results_by_eval_id`（`_agent_evaluator.py:815-915`） + +1. `InMemoryEvalSetsManager` 创建并填充 evalset +2. 选择 `RemoteEvalService`（call_agent 模式）或 `LocalEvalService` +3. `InferenceConfig(parallelism=case_parallelism)` 构造 N 份 `InferenceRequest`（N=num_runs） +4. `async for inference_result in eval_service.perform_inference(...)` L893 — 调 `call_agent(query)` 拿 agent 回复 +5. `EvaluateConfig(eval_metrics=eval_metrics)` + `EvaluateRequest(inference_results=inference_results)` L901-906 +6. `async for eval_result in eval_service.evaluate(evaluate_request)` L908 — 跑每个 metric 的 evaluator（`FinalResponseEvaluator` / `LLMRubricResponseEvaluator` 等） +7. 按 eval_id 聚合，返回 `dict[str, list[EvalCaseResult]]` + +### 阶段 I：结果回传与产物落盘 + +- 阶段 H 结果 → 阶段 G `eval_results_by_eval_id` → 阶段 F `EvaluateResult(results_by_eval_set_id)` → 阶段 E `summarize_outcome(result)` 输出 `EvaluationOutcome(pass_rate, tiebreaker, metric_breakdown, failed_case_ids)` +- GEPA 用 `EvaluationOutcome.pass_rate` 比较 Pareto 前沿 +- 主循环结束后，阶段 C `_build_optimize_result`（`_optimize_gepa_reflective.py:119-258`）从 `gepa_result` + `callback_rounds` 组装 `OptimizeResult` +- 阶段 B `_persist_artifacts`（L374-437）写： + - `output_dir/result.json` — 完整 `OptimizeResult.model_dump_json` + - `output_dir/summary.txt` — 人类可读摘要 + - `output_dir/rounds/round_NNN.json` — 每轮 `RoundRecord` + - `output_dir/baseline_prompts/.md` + - `output_dir/best_prompts/.md` + - `output_dir/config.snapshot.json` + - `output_dir/run.log` + +### 调用链时序图（一图速览） + +``` +run_optimization.py + │ + ▼ +AgentOptimizer.optimize(config_path, call_agent, target_prompt, train_path, val_path, output_dir) + │ + ├─ load_optimize_config ──► OptimizeConfigFile + ├─ _validate_inputs + ├─ target_prompt.read_all ──► baseline_snapshot + │ + ├─ GepaReflectiveOptimizer(config, call_agent, target_prompt, ...).run(reporter) + │ │ + │ ├─ _load_evalset_cases(train_path) + _load_evalset_cases(val_path) + │ ├─ _AgentGEPAAdapter(target_prompt, eval_config, call_agent, ...) + │ ├─ _OptimizeModelCallable(algo.reflection_lm) + │ │ + │ └─ asyncio.to_thread(gepa.optimize, ...) ◄── GEPA 主循环（每轮） + │ │ + │ ├─ module_selector 选字段 + │ ├─ adapter.evaluate(parent, train_subsample) + │ │ │ + │ │ └─ run_evaluator ──► AgentEvaluator.get_executer + │ │ │ + │ │ └─ _EvalExecuter._run + │ │ │ + │ │ ├─ _load_eval_set_from_file ──► EvalSet + │ │ ├─ evaluate_eval_set + │ │ │ │ + │ │ │ └─ _get_eval_results_by_eval_id + │ │ │ ├─ perform_inference (call_agent) + │ │ │ └─ eval_service.evaluate (metric evaluators) + │ │ │ └─► EvalCaseResult[] + │ │ │ + │ │ └─ (failure) raise _EvaluationCasesFailed + │ │ + │ ├─ make_reflective_dataset(failed_cases) ──► markdown + │ ├─ reflection_lm(markdown) ──► candidate prompt + │ ├─ target_prompt.write_all(candidate) ◄── 落盘新候选 + │ ├─ adapter.evaluate(candidate, full_valset) + │ ├─ Pareto 比较 → accept / reject + │ └─ stop_callbacks 投票 + │ + ├─ (optional) target_prompt.write_all(result.best_prompts) ◄── update_source=True + │ + └─ _persist_artifacts + ├─ result.json / summary.txt + ├─ rounds/round_NNN.json + ├─ baseline_prompts/.md + ├─ best_prompts/.md + ├─ config.snapshot.json + └─ run.log +``` + +--- + +## 3. evaluation 测试失败的完整 traceback + +`tests/evaluation` 共 **2 条**测试失败（来自 `pytest_full.log` L728-744 与 L1047+；完整 suite 9290 通过 / 118 失败，evaluation 模块占 2 条）。 + +### 失败 1：`test_eval_executer_raises_evaluation_cases_failed_on_case_failure` + +``` +tests\evaluation\test_optimize_evaluator_call.py:524: in test_eval_executer_raises_evaluation_cases_failed_on_case_failure + await executer.evaluate() +trpc_agent_sdk\evaluation\_agent_evaluator.py:244: in evaluate + await self._ensure_run() +trpc_agent_sdk\evaluation\_agent_evaluator.py:236: in _ensure_run + await self._task +trpc_agent_sdk\evaluation\_agent_evaluator.py:187: in _run + eval_set = AgentEvaluator._load_eval_set_from_file(test_file, eval_config) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +trpc_agent_sdk\evaluation\_agent_evaluator.py:673: in _load_eval_set_from_file + raise FileNotFoundError(f"Eval set file not found: {actual_file_path}") +E FileNotFoundError: Eval set file not found: C +``` + +**根因**：Windows 路径解析 bug。测试用 `tmp_path / "tiny.evalset.json"` 生成形如 `C:\Users\...\tiny.evalset.json` 的绝对路径。`_load_eval_set_from_file` 在 `_agent_evaluator.py:667` 检查 `if ":" in eval_set_file:` 时假定 `:` 是 ADK 风格的 case 选择分隔符（`file.json:case_id`），用 `split(":", 1)` 取 parts[0] 作为文件路径。Windows 盘符 `C:` 被错误切分，`actual_file_path` 变成 `"C"`，文件查找失败。 + +**修复方向**（不在本次任务范围）：用 `os.path.splitdrive()` 先剥离盘符再查 `:`，或限制分隔符只匹配 `.json:` 之后的字符。 + +### 失败 2：`test_quickstart_real_gepa_loop_reuses_single_event_loop_across_rounds` + +``` +tests\evaluation\test_optimize_quickstart_example.py:475: in test_quickstart_real_gepa_loop_reuses_single_event_loop_across_rounds + assert len(seen_loop_ids) >= 2, ( +E AssertionError: Expected real gepa main loop to call call_agent more than once; saw 0 call(s). +E assert 0 >= 2 +E + where 0 = len([]) +``` + +**根因**：这个测试 monkeypatch `_OptimizeModelCallable.__call__` 用 fake reflection LM，设了 `max_metric_calls=6` / `max_iterations_without_improvement=1`，期待 gepa 主循环至少跑 2 轮（baseline + round 1）。但 `seen_loop_ids` 为空 → call_agent 一次都没被调到。可能原因： + +- 该测试依赖 `examples/optimization/quickstart/optimizer.json` 中的 `${TRPC_AGENT_MODEL_NAME}` 等环境变量，未设置时 gepa 在 baseline 评估阶段就抛异常退出，没有进入第一轮反思 +- 也可能是 `asyncio.to_thread(gepa_optimize, ...)` 在 worker thread 内调用 `call_agent`（async），而 monkeypatch 的 `stub_call_agent` 没有跨线程传播；或 gepa 的 asyncio event loop 配置在 Windows ProactorEventLoop 下行为不同 + +### 测试运行说明 + +本次报告未能成功跑出 `pytest tests/evaluation -q --tb=short` 的新日志：用 PowerShell `Tee-Object` 时不实时刷新；改用 bash `> pytest_eval.log 2>&1` 重定向后，进程运行 16+ 分钟仍未结束（远超完整 suite 的 3.5 分钟），疑似 Windows 下 gepa / asyncio 阻塞或缺 API key 卡住。所有失败数据已从既有的 `pytest_full.log`（完整 suite 运行结果）提取。 + +--- + +## 4. #91 新增示例目录建议 + +任务规格明确要求 `examples/optimization/eval_optimize_loop/`。基于现有 9 个 example 的命名约定与目录结构，**强烈建议直接按规格放在**： + +``` +examples/optimization/eval_optimize_loop/ +├── README.md # 300–500 字方案设计说明 + 使用指引 +├── pipeline.py # 入口脚本（baseline 评测 → 失败归因 → 优化 → 回归 → 决策 → 落盘） +├── optimizer.json # GEPA + metric 配置（fake model + trace mode 默认开） +├── train.evalset.json # 3 条训练 case +├── val.evalset.json # 3 条验证 case（覆盖可优化成功 / 优化无效 / 优化退化的三类场景） +├── agent/ +│ ├── __init__.py +│ ├── agent.py # create_agent() 工厂，重读 prompt +│ ├── config.py # 环境变量读取，支持 FAKE_MODEL=1 短路 +│ └── prompts/ +│ ├── system.md # 被优化的 system prompt +│ └── skill.md # 被优化的 skill prompt +├── attribution/ # 失败归因模块（规格第 2 阶段） +│ └── cluster.py # 按失败类型聚类（reply_mismatch / tool_call_error / param_error / rubric_fail / knowledge_fail / format_fail） +├── gate/ # 接受策略（规格第 5 阶段） +│ └── accept_policy.py # 可配置 gate：val_improvement ≥ threshold / no_new_hard_fail / critical_case_no_regression / cost ≤ budget +├── audit/ # 审计落盘（规格第 6 阶段） +│ └── report.py # 生成 optimization_report.json + .md +├── fake/ # fake judge / fake model / trace mode（规格要求） +│ ├── fake_model.py +│ ├── fake_judge.py +│ └── traces/ # 预录对话，trace mode 用 +├── runs/ # 输出根目录（每次运行写到时间戳子目录） +└── optimization_report.json.example # 示例输出 +``` + +### 选此位置的理由 + +1. **目录命名与规格 1:1 对齐**：任务规格交付物明确写 `examples/optimization/eval_optimize_loop/`，与现有 9 个 example 同层级，便于索引 +2. **现有目录覆盖场景互补**：现有 9 个 example 都只展示"优化器本身怎么用"，没有任何一个演示"评测 → 归因 → 优化 → 回归 → 决策 → 审计"端到端闭环。`ci_integration/` 最接近但只覆盖 PR 守门 + 夜间优化两段，缺失败归因和 gate decision +3. **依赖现有模块最小**：`pipeline.py` 直接复用 `AgentEvaluator.evaluate` 跑 baseline + 回归，`AgentOptimizer.optimize` 跑优化轮，新增的 `attribution/` `gate/` `audit/` 是薄包装层 +4. **fake mode 可对照 quickstart 测试**：trace mode + fake judge 在 `optimizer.json` 里通过 `"judge_model": {"model_name": "fake-judge"}` 触发，`fake/fake_model.py` 提供 `async def fake_call_agent(query)` 直接返回 `val.evalset.json` 中预录的 `final_response`，满足"无 API Key 跑通 ≤3min"的验收标准 + +### 与现有模块的集成点 + +- `pipeline.py` 调 `AgentEvaluator.get_executer(val_path, call_agent=fake_call_agent, eval_metrics_path="optimizer.json", print_summary_report=False)` 跑 baseline，拿 `EvaluateResult` +- `attribution/cluster.py` 遍历 `EvaluateResult.results_by_eval_set_id[...].eval_results_by_eval_id` 的每个 `EvalCaseResult`，按 `eval_metric_results[*].metric_name` 与 `details.reason` 分类： + - `final_response_avg_score` fail → `"reply_mismatch"` 或 `"format_fail"` + - `tool_trajectory_avg_score` fail → `"tool_call_error"` 或 `"param_error"` + - `llm_rubric_response` fail → `"rubric_fail"` + - `llm_rubric_knowledge_recall` fail → `"knowledge_fail"` +- 调 `AgentOptimizer.optimize(config_path="optimizer.json", call_agent=fake_call_agent, target_prompt=target, train_path, val_path, output_dir)` 跑优化 +- 拿到 `OptimizeResult` 后，`audit/report.py` 调 `AgentEvaluator.get_executer(val_path, call_agent=...)` 用 `result.best_prompts`（先 `target_prompt.write_all(result.best_prompts)`）再跑一次 val 评测做候选验证 +- `gate/accept_policy.py` 比较 baseline 与 candidate 的 `EvaluationOutcome`，输出 `{"accepted": bool, "reason": str}`，写入 `optimization_report.json` + +### 6 条 case 的设计要点（满足规格三类场景） + +| case_id | 集合 | 场景类型 | 设计方式 | +|---|---|---|---| +| `t_opt_success` | train | 可优化成功 | baseline prompt 漏掉"输出答案前必须给出推理步骤"，导致 final_response_avg_score fail；reflection LM 补上即可通过 | +| `t_opt_noop` | train | 优化无效 | case 本身模糊（缺少关键数字），任何 prompt 都无法让 agent 给出匹配答案；reflection LM 反复改写无法提升 | +| `t_opt_regress` | train | 优化后退化的诱因 | 训练集包含一个特殊格式偏好（例如要求 "答：" 而非 "答案："），改写 prompt 满足此 case 会牺牲验证集 | +| `v_opt_success` | val | 验证可优化成功 | 与 `t_opt_success` 同类型，验证 baseline→candidate 提升 | +| `v_opt_noop` | val | 验证优化无效 | 同 `t_opt_noop`，确认 train 与 val 一致地无提升 | +| `v_opt_regress` | val | 验证退化 | 与 `t_opt_regress` 的格式偏好冲突，candidate 通过 train 但在 val 退化；gate policy 必须拒绝 | + +接受策略 gate 至少配置： +- `val_total_score_improvement >= 0.02`（防止噪声波动被误判为提升） +- `no_new_hard_fail = True`（candidate 不能让任何 baseline PASS 的 case 变 FAIL） +- `critical_case_ids = ["v_opt_regress"]` 不能退化（直接命中规格验收标准 3） +- `cost_budget_usd = 5.0`（防止 fake mode 失灵时 LLM 失控） + +--- + +## 附录：关键文件行号速查表 + +| 关注点 | 文件:行号 | +|---|---| +| AgentEvaluator 主入口 | `trpc_agent_sdk/evaluation/_agent_evaluator.py:255` | +| AgentEvaluator.evaluate 静态方法 | `_agent_evaluator.py:308` | +| AgentEvaluator.get_executer | `_agent_evaluator.py:363` | +| `_EvalExecuter._run` | `_agent_evaluator.py:140` | +| `_EvaluationCasesFailed` 抛出点 | `_agent_evaluator.py:231` | +| Windows 盘符 bug 现场 | `_agent_evaluator.py:667`（`if ":" in eval_set_file`）| +| `evaluate_eval_set` | `_agent_evaluator.py:470` | +| `_get_eval_results_by_eval_id` | `_agent_evaluator.py:815` | +| AgentOptimizer 主入口 | `trpc_agent_sdk/evaluation/_agent_optimizer.py:112` | +| `AgentOptimizer.optimize` | `_agent_optimizer.py:132` | +| `_validate_inputs` | `_agent_optimizer.py:544` | +| `_persist_artifacts` | `_agent_optimizer.py:374` | +| `_mask_sigint` 上下文管理器 | `_agent_optimizer.py:72` | +| `run_evaluator` 桥接函数 | `trpc_agent_sdk/evaluation/_optimize_evaluator_call.py:83` | +| `summarize_outcome` | `_optimize_evaluator_call.py:44` | +| `EvaluationOutcome` 数据类 | `_optimize_evaluator_call.py:23` | +| `GepaReflectiveOptimizer.run` | `trpc_agent_sdk/evaluation/_optimize_gepa_reflective.py:431` | +| `_build_stop_callbacks` | `_optimize_gepa_reflective.py:294` | +| `_build_optimize_result` | `_optimize_gepa_reflective.py:119` | +| `_AgentGEPAAdapter` | `trpc_agent_sdk/evaluation/_optimize_gepa_adapter.py` | +| `OptimizeResult` schema | `trpc_agent_sdk/evaluation/_optimize_result.py:167` | +| `RoundRecord` schema | `_optimize_result.py:43` | +| `TargetPrompt` | `trpc_agent_sdk/evaluation/_target_prompt.py:63` | +| `EvalSet` schema | `trpc_agent_sdk/evaluation/_eval_set.py:33` | +| quickstart 入口 | `examples/optimization/quickstart/run_optimization.py` | +| quickstart agent 工厂 | `examples/optimization/quickstart/agent/agent.py` | +| quickstart optimizer.json | `examples/optimization/quickstart/optimizer.json` | +| quickstart train.evalset.json | `examples/optimization/quickstart/train.evalset.json`（5 条小学算术题）| +| quickstart val.evalset.json | `examples/optimization/quickstart/val.evalset.json`（3 条小学算术题）| + +--- + +## 5. 现有能力盘点（Issue #91 视角） + +> 本节回答："SDK 已经提供了哪些能力可以直接复用，不必重造？"——是对第 1、2 节的能力视角重组，按 Issue #91 要求的"评测 / 归因 / 优化 / 验证 / 决策 / 审计"六个阶段归类。 + +### 5.1 评测阶段（AgentEvaluator） + +| 能力 | 关键文件 / 行号 | 是否已就绪 | +|---|---|---| +| 评测入口（有返回值版本） | `_agent_evaluator.py:363` `AgentEvaluator.get_executer(...)` → `_EvalExecuter` → `await executer.evaluate()` → `executer.get_result(): EvaluateResult` | ✅ | +| 评测入口（断言版本，CI 友好） | `_agent_evaluator.py:308` `AgentEvaluator.evaluate(...)`，无返回值，全失败抛 `_EvaluationCasesFailed`（L78-93，`AssertionError` 子类） | ✅ | +| EvalSet JSON schema | `_eval_set.py:33` `EvalSet`；`_eval_case.py:170-239` `EvalCase` / `Invocation` / `IntermediateData` | ✅ | +| trace mode（跳过 agent 直接打分预录对话） | `_eval_case.py:152` `EvalModeTrace = "trace"`；`_agent_evaluator.py:300-530` `_is_trace_only` 分支 | ✅ | +| 7 个内置 metric | `_eval_metrics.py:45-66` `PrebuiltMetrics`：`final_response_avg_score` / `tool_trajectory_avg_score` / `response_match_score` / `response_evaluation_score` / `llm_final_response` / `llm_rubric_response` / `llm_rubric_knowledge_recall` | ✅ | +| Pass/Fail 判定 | `_eval_metrics.py:38-42` `EvalStatus`；每个 evaluator 内部 `_get_eval_status(score) = PASSED if score >= threshold else FAILED` | ✅ | +| Case 级失败原因字段 | `_eval_result.py:185` `EvalCaseResult.error_message`；`:108-116` `EvalMetricResultDetails.reason`；`:77-81` `PerInvocationResult.reason/rubric_scores`；`:53` `NamedScoreResult.reason` | ✅ | +| 工具调用轨迹字段 | `_eval_case.py:121` `Invocation.intermediate_data.tool_uses/tool_responses`；`:242-289` `get_all_tool_calls` / `get_all_tool_responses` | ✅ | + +### 5.2 归因阶段（部分已就绪） + +| 能力 | 现状 | 缺口 | +|---|---|---| +| 字段级归因（GEPA 已做） | `RoundRecord.optimized_field_names`（`_optimize_result.py:88-91`）；`RoundRecord.per_field_diagnosis`（`:96-99`）来自反思 LM | — | +| 反思 LM 自动看失败 case | GEPA 算法内置（`_optimize_gepa_adapter.py:657-721` `make_reflective_dataset`），`reflection_minibatch_size` 控制批大小 | — | +| **case 级失败类型聚类（6 类）** | 字段都在，但**没有任何示例代码**做"按失败类型聚类"（回复不匹配 / 工具调用错 / 参数错 / rubric 不达标 / 召回不足 / 格式不符） | ❌ 需在 `eval_optimize_loop/` 新增 | + +### 5.3 优化阶段（AgentOptimizer） + +| 能力 | 关键文件 / 行号 | 是否已就绪 | +|---|---|---| +| 优化入口 | `_agent_optimizer.py:132-296` `AgentOptimizer.optimize(*, config_path, call_agent, target_prompt, train_dataset_path, validation_dataset_path, output_dir, update_source=False, ...)` | ✅ | +| GEPA 反思算法（当前唯一注册） | `_optimize_gepa_reflective.py:408` `GepaReflectiveOptimizer`；adapter 在 `_optimize_gepa_adapter.py:505`；callback 在 `_optimize_gepa_callback.py:81` | ✅ | +| 配置 schema | `_optimize_config.py:226-243` `OptimizeConfigFile`；`GepaReflectiveAlgo` 含 `max_metric_calls` / `max_iterations_without_improvement` / `timeout_seconds` / `score_threshold` / `max_candidate_proposals` / `max_tracked_candidates` 等 6 类停止条件 | ✅ | +| 候选选择策略 | `_optimize_config.py` `candidate_selection_strategy ∈ {pareto, current_best, epsilon_greedy, top_k_pareto}`；`frontier_type ∈ {instance, objective, hybrid, cartesian}` | ✅ | +| 多字段 prompt 优化 | `module_selector ∈ {round_robin, all, random}`；`use_merge=true` 触发字段合并（要求注册 ≥2 字段，`_agent_optimizer.py:602-614`） | ✅ | +| 框架层接受/早停 | `_optimize_gepa_reflective.py:89-116, 294-381` `_RequiredMetricsAboveThresholdStopper` + 6 类 stopper；`optimize.stop.required_metrics` ∈ `"all" / list / []` | ✅ | + +### 5.4 验证阶段（部分已就绪） + +| 能力 | 现状 | 缺口 | +|---|---|---| +| train / val 物理隔离校验 | `_agent_optimizer.py:544-614` `_validate_inputs` 强制 train≠val | ✅ | +| 每轮全量 val 评估 | GEPA 算法内置（`_optimize_gepa_adapter.py:594-655`） | ✅ | +| 独立 val 评测（脱离 optimizer） | `AgentEvaluator.get_executer(val_path, ...)` 即可，参考 `examples/optimization/ci_integration/tests/test_agent_quality.py:48-62` | ✅ | +| **逐 case 跨 run 回归 diff** | `examples/optimization/advanced_strategies/compare.py:52-96` 只对比汇总指标，**不做 case-level diff** | ❌ 需新增 | + +### 5.5 接受决策阶段（部分已就绪） + +| 能力 | 现状 | 缺口 | +|---|---|---| +| Pareto 前沿 / 早停 / required_metrics 门禁 | 见 5.3 | ✅ | +| **业务级可配置 gate**（"总分提升≥N"、"hard fail 数不增"、"关键 case 不退化"、"成本≤预算"） | **完全不存在**，SDK 的 stop policy 只控制"早停"，不等价于业务"是否回写" | ❌ 需新增 | + +### 5.6 审计阶段（部分已就绪） + +| 能力 | 现状 | 缺口 | +|---|---|---| +| `result.json` / `summary.txt` / `rounds/round_NNN.json` | `_agent_optimizer.py:374-491` `_persist_artifacts` | ✅ | +| `baseline_prompts/` + `best_prompts/` 双快照 | 同上 | ✅ | +| 时间戳子目录隔离 | 所有 example 入口都做了，例如 `quickstart/run_optimization.py:147-148` | ✅ | +| `config.snapshot.json` 复现配置 | 同上 | ✅ | +| **`optimization_report.{json,md}`（含 baseline / candidate / 逐 case delta / gate decision / 失败归因 / 成本 / 耗时 / 随机种子）** | **完全不存在** | ❌ 需新增 | +| **操作员级审计日志（谁/何时/改了哪个 prompt）** | **完全不存在**；`update_source=True` 直接覆盖源文件 | ❌ 需新增 | + +### 5.7 fake judge / fake model / trace mode + +> Issue 验收标准 #5 要求"fake model / trace mode 下完整 pipeline 耗时 ≤ 3 分钟"。 + +| 路径 | 现状 | 实现策略 | +|---|---|---| +| **trace mode（原生）** | ✅ `_eval_case.py:152` + `_agent_evaluator.py:300-530`；示例 `examples/evaluation/trace_mode/` | **首选**：在 `evalset` 中给 `eval_mode="trace"` + `actual_conversation`，评测阶段完全跳过 agent 推理，Windows 上稳定 ≤3 min | +| **fake call_agent（黑盒）** | ✅ `_agent_optimizer.py:547-551`、`_optimize_evaluator_call.py:83-136`，`CallAgent = async (query: str) -> str` 协议，业务自定义 | 次选：在 `fake/fake_model.py` 实现 async 函数，按规则 / 字典返回字符串 | +| **fake judge（自定义 scorer）** | ✅ `_llm_evaluator.py:103-178` `LLM_EVALUATOR_REGISTRY` 支持注册自定义 `response_scorer` / `models_aggregator`，scorer 内可不调 LLM | 在 `fake/fake_judge.py` 注册假 scorer | +| **fake reflection LM** | ✅ `_optimize_model_callable.py:210-309` 可被 monkeypatch；`OptimizeModelOptions.base_url`（`_optimize_model_options.py:24`）可指向本地 fake server；`ModelRegistry` 也支持自注册 provider | 在 `fake/fake_reflection.py` 包一个符合 gepa `LanguageModel` 协议的同步 callable | +| **框架内置 fake 实现** | ❌ 不存在 | **必须新增** `fake/` 子目录，落地上述四个 fake 路径 | + +--- + +## 6. 需要新增能力清单（仅在 `examples/optimization/eval_optimize_loop/`） + +下表把第 5 节标 ❌ 的缺口集中起来，给出建议落点。 + +| # | 新增能力 | 落点（相对 `eval_optimize_loop/`） | 输入 | 输出 | +|---|---|---|---|---| +| 1 | 失败归因分类器 | `attribution/cluster.py` | `EvalCaseResult[]` + `EvalCase.conversation[*].intermediate_data` | `{category: str, case_ids: list[str], sample_reason: str}[]`，category ∈ {reply_mismatch, tool_call_error, param_error, rubric_fail, knowledge_fail, format_fail} | +| 2 | 逐 case 跨 run regression diff | `regression/diff.py` | baseline `EvaluateResult` + candidate `EvaluateResult` | `{newly_passed, newly_failed, score_up, score_down: list[{eval_id, baseline_score, candidate_score, delta}]}` | +| 3 | 可配置接受 gate | `gate/accept_policy.py` | gate config（JSON）+ regression diff + cost + critical_case_ids | `{accepted: bool, reasons: list[str], violated_rules: list[str]}` | +| 4 | 双格式优化报告 | `audit/report_builder.py` | baseline / candidate 评测结果 + diff + 归因 + gate 决策 + cost + duration + seed | `optimization_report.json` + `optimization_report.md` | +| 5 | 审计落盘增强 | 同 #4，加 `audit` 节 | seed / total_cost / duration / accepted / reasons / repro_config_hash | 写入 `optimization_report.json` 顶层 | +| 6 | fake 实现四件套 | `fake/fake_model.py` / `fake_judge.py` / `fake_reflection.py` / `traces/*.json` | — | 满足"无 API Key 跑通" | +| 7 | 6 条三类样例 case | `data/train.evalset.json`（3 条）+ `data/val.evalset.json`（3 条） | — | 覆盖"可优化成功 / 优化无效 / 优化后退化"三类各 1/3 | +| 8 | pipeline 入口 + README | `run_pipeline.py` + `README.md` | `optimizer.json` + `data/*.json` + `prompts/*.md` | 端到端产物 + 300–500 字设计说明 | + +--- + +## 7. 建议集成点（不修改 SDK 的前提下） + +> 这一节回答："新增的 8 个模块怎么和 SDK 现有 API 对接？" + +### 7.1 跑 baseline 与 candidate 评测 + +```python +# 拿 EvaluateResult 对象做 diff，必须用 get_executer，不要用 evaluate（无返回值） +executer = AgentEvaluator.get_executer( + eval_dataset_path=str(VAL_PATH), + call_agent=fake_call_agent, # 或 trace mode 下传 None + eval_metrics_file_path_or_dir="optimizer.json", # 复用同一份 metric 配置 + num_runs=1, + print_detailed_results=False, + print_summary_report=False, +) +await executer.evaluate() +baseline_result = executer.get_result() # → EvaluateResult +``` + +关键位置：`_agent_evaluator.py:363`（`get_executer`）、`_agent_evaluator.py:140`（`_EvalExecuter._run`）、`_eval_result.py:310`（`EvaluateResult`）。 + +### 7.2 跑优化（保留自动回滚） + +```python +result: OptimizeResult = await AgentOptimizer.optimize( + config_path="optimizer.json", + call_agent=fake_call_agent, + target_prompt=target, # TargetPrompt().add_path("system_prompt", ...).add_path("skill", ...) + train_dataset_path="data/train.evalset.json", + validation_dataset_path="data/val.evalset.json", + output_dir="runs//optimizer", + update_source=False, # ★ 让 SDK 自动回滚 baseline，避免污染下一次评测 + verbose=1, +) +best_prompts: dict[str, str] = result.best_prompts # 最优候选 prompt +``` + +关键位置：`_agent_optimizer.py:132-296`、`_optimize_result.py:167`（`OptimizeResult`）、`_target_prompt.py:63`（`TargetPrompt`，**注意：仓库中没有 `TargetSkill` 类，skill 是 `TargetPrompt` 的字段名之一**）。 + +### 7.3 写回 candidate 重跑验证集 + +```python +baseline_snapshot = await target.read_all() +try: + await target.write_all(best_prompts) # 临时把 candidate 写到源文件 + candidate_executer = AgentEvaluator.get_executer( + eval_dataset_path=str(VAL_PATH), + call_agent=fake_call_agent, + eval_metrics_file_path_or_dir="optimizer.json", + num_runs=1, + print_detailed_results=False, + print_summary_report=False, + ) + await candidate_executer.evaluate() + candidate_result = candidate_executer.get_result() +finally: + await target.write_all(baseline_snapshot) # ★ 必须复位，否则污染后续 run +``` + +关键位置：`_target_prompt.py:135-171` `write_all`（path 字段用 tmp+`os.replace` 原子写，部分失败自动回滚）、`_target_prompt.py:128-133` `read_all`。 + +### 7.4 失败归因输入来源 + +从 baseline 的 `EvaluateResult` 取： + +- `result.results_by_eval_set_id[set_id].eval_results_by_eval_id[eval_id].error_message`（`_eval_result.py:185`） +- `result....eval_metric_results[*].details.reason`（`_eval_result.py:108-116`） +- 对应的 `EvalCase.conversation[*].intermediate_data.tool_uses` / `tool_responses`（`_eval_case.py:121`）—— 用于区分"工具调用错"vs"参数错" + +trace mode 下这些字段都已就绪；非 trace 模式需要让 fake call_agent 在 `IntermediateData` 里回填假 tool_uses。 + +### 7.5 fake 路径选择优先级 + +1. **首选 trace mode**：评测阶段完全跳过 agent 推理，Windows 上稳定 ≤3 分钟。需要让 train/val evalset 都带 `eval_mode="trace"` + `actual_conversation`。 +2. **次选 fake call_agent**：当 case 不易预录对话时，在 `fake/fake_model.py` 实现 async 函数（按规则 / 字典返回字符串），由 pipeline 传入 `AgentEvaluator` / `AgentOptimizer`。 +3. **fake judge**：在 `fake/fake_judge.py` 用 `LLM_EVALUATOR_REGISTRY.register_response_scorer(...)`（`_llm_evaluator.py:103-178`）注册假 scorer，让 LLM metric 不调真实 LLM。 +4. **fake reflection LM**：在 `fake/fake_reflection.py` 包一个符合 gepa `LanguageModel` 协议的同步 callable，通过 `optimizer.json` 的 `reflection_lm.model_name = "fake-reflection"` 触发，或在 `run_pipeline.py` 里 monkeypatch `_OptimizeModelCallable.__call__`（`_optimize_model_callable.py:245-248`）。 + +### 7.6 接受 gate 必须独立于 SDK 决策 + +SDK 的 `stop.required_metrics` 只控制"早停"，不等于业务"是否接受 candidate"。新 pipeline 的 gate 必须放在 `AgentOptimizer.optimize` 返回**之后**再判一次： + +```python +result: OptimizeResult = await AgentOptimizer.optimize(...) +# 即使 result.status == "SUCCEEDED"，也要再判业务 gate +gate_decision = accept_policy.evaluate( + baseline_result=baseline_val_result, + candidate_result=candidate_val_result, + regression_diff=diff, + cost_usd=result.total_cost, + config=gate_config, +) +if not gate_decision.accepted: + # 即使 SDK 把 best_prompts 落到 best_prompts/ 也不回写源文件 + pass +``` + +这是"不能新增 hard fail"、"关键 case 不能退化"、"成本 ≤ 预算"等业务规则的**唯一干净落点**——SDK 内部的 Pareto / threshold 早停都不覆盖这些。 + +### 7.7 复现配置与随机种子 + +- `optimizer.json` 中的 `algorithm.seed`（`_optimize_config.py` `GepaReflectiveAlgo.seed`）是 GEPA 随机种子，写进 `audit.seed` +- `config.snapshot.json` 已由 SDK 自动落（`_agent_optimizer.py:425-432`），做 SHA-256 写进 `audit.repro_config_hash` +- 环境变量差异（`TRPC_AGENT_MODEL_NAME` 等）脱敏后写进 `audit.env_diff_redacted` + +### 7.8 集成点时序图（一图速览） + +``` +run_pipeline.py + │ + ├─ ① baseline 评测 + │ └─ AgentEvaluator.get_executer(val_path, call_agent=fake, ...) → EvaluateResult + │ + ├─ ② 失败归因 + │ └─ attribution/cluster.py(EvaluateResult) → 归因类别列表 + │ + ├─ ③ 优化 + │ └─ AgentOptimizer.optimize(config_path, call_agent=fake, target_prompt, train, val, output_dir) + │ │ + │ └─ 内部: GEPA 主循环（每轮：反思 LM → 写候选 → 跑 train/val → Pareto 比较） + │ └─ 返回 OptimizeResult.best_prompts + │ + ├─ ④ candidate 评测（写回 + 跑 val + 复位） + │ ├─ target_prompt.write_all(best_prompts) + │ ├─ AgentEvaluator.get_executer(val_path, call_agent=fake, ...) → EvaluateResult + │ └─ finally: target_prompt.write_all(baseline_snapshot) ◄── 必须复位 + │ + ├─ ⑤ 逐 case regression diff + │ └─ regression/diff.py(baseline_result, candidate_result) → diff + │ + ├─ ⑥ 接受 gate（独立于 SDK 决策） + │ └─ gate/accept_policy.py(diff, cost, config) → {accepted, reasons[]} + │ + └─ ⑦ 双格式报告 + 审计 + └─ audit/report_builder.py(全部上述产物) → optimization_report.{json,md} +``` + +--- + +## 8. 关键决策与陷阱速查 + +1. **没有 `TargetSkill` 类**——全仓库只有 `TargetPrompt`（`_target_prompt.py:63`），skill 是 `TargetPrompt.add_path("skill", ...)` 的字段名。 +2. **框架不内置 fake 实现**——所有 fake 路径都是扩展点（call_agent 协议 / `LLM_EVALUATOR_REGISTRY` / `ModelRegistry` / monkeypatch），必须由 `eval_optimize_loop/fake/` 落地。 +3. **trace mode 是 Windows ≤3 min 的最优解**——评测阶段完全跳过 agent，避免 `asyncio.to_thread` + Windows ProactorEventLoop 的潜在阻塞（参考第 3 节失败 #2）。 +4. **`AgentEvaluator.evaluate` 无返回值，`get_executer` 才有返回值**——做 regression diff 必须用后者。 +5. **`update_source=False` 是安全默认**——SDK 会自动把 baseline 回滚到源文件；pipeline 内不要再额外写回。 +6. **接受 gate 必须独立于 SDK**——SDK 的 `stop.required_metrics` / Pareto 早停只控制迭代终止，不等于业务接受。 +7. **Windows 盘符 bug 待避**（`_agent_evaluator.py:667`，参考第 3 节失败 #1）——evalset 路径优先用相对路径或 `Path.as_posix()`，绕开 `:` 分隔符误判。 +8. **6 条 case 的"退化"场景构造**——让 train 含特殊格式偏好（例如要求 "答：" 而非 "答案："），val 的 expected 与该偏好冲突；candidate 为通过 train 会牺牲 val，gate 必须拒绝。这是覆盖 Issue 验收标准 #3 的关键设计。 diff --git a/examples/optimization/eval_optimize_loop/.gitignore b/examples/optimization/eval_optimize_loop/.gitignore new file mode 100644 index 000000000..36e3337a6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/.gitignore @@ -0,0 +1,9 @@ +# Runtime artifact output +runs/ +optimization_report.json +optimization_report.md + +# Python +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 000000000..eb5f33fbc --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,191 @@ +# 方案设计 + +管线把当前源提示词定义为 Champion，把 `AgentOptimizer.optimize(update_source=False)` 产生的最佳提示词或显式候选定义为 Challenger。优化器只使用训练集发现问题；独立验证集只参加回归和门禁，避免把验证失败细节反馈给优化器。fake 模式由提示词内公开控制项生成可评测轨迹，仅验证流程，不作为隐藏样本能力证明。 + +每次运行以 UTC 微秒时间和随机后缀生成唯一目录，冻结提示词、数据集、评测配置、优化配置、Gate、模型和随机种子的哈希或标识。Champion 与 Challenger 均由 `AgentEvaluator` 评分；逐 case 保存实际/预期回复、指标原因、工具调用与响应、参数差异及 trace 引用。归因依据这些证据输出回复、格式、工具、参数、rubric、知识、基础设施或证据不足类别，不依赖场景标签。 + +Gate 同时检查验证提升、训练涨验证跌的过拟合、新增高风险失败、protected case 与 slice 退化、成本证据和微小波动。成本未知时保存空值并由 G6 拒绝自动应用。评测中的临时替换、异常恢复和最终写回都经 `TargetPrompt`；只有显式 `--apply` 且全部 Gate 通过才更新 Champion。JSON 与 Markdown 报告记录决策、每轮候选、耗时、成本、产物路径和复现命令，优化失败也会落盘可审计的 REJECT 报告。 + +你在可信的本地仓库 trpc-agent-python 中工作，目标是完成 GitHub Issue #91： + +“构建一个可复现的 Evaluation + Optimization pipeline： +评测 → 失败归因 → Prompt 优化 → 回归验证 → 产物审计。” + +请连续完成以下工作；只有在确实无法从仓库源码判断时才提问。完成后不要 git commit、不要 git push、不要修改或删除当前任务范围之外的文件。 + +# 总体边界 + +1. 只新增或修改： + examples/optimization/eval_optimize_loop/ +2. 不修改 trpc_agent_sdk/、tests/ 现有 SDK 测试、项目依赖、CI、git 配置。 +3. 不做 Web UI、数据库、远端 Prompt Store、多 Agent 编排、MCP 服务。 +4. 不复制其他 PR 的代码；可以复用当前仓库公开 API 和已有 example 的编码风格。 +5. 不运行全仓 pytest；Windows 存在与本任务无关的 POSIX 平台失败。 +6. 使用相对 evalset 路径，避免 Windows 盘符解析问题。 +7. 所有写 Prompt 的操作必须通过 TargetPrompt 的公开 API，并正确 await read_all/write_all。 +8. 默认绝不改 prompts/system.md。只有 gate=ACCEPT 且 CLI 显式传入 --apply 时才写回。 +9. 不使用 monkeypatch SDK 私有成员，不新增 jsonschema 或第三方依赖。 + +# 要保留的设计主线 + +当前 Prompt 是 Champion；新 Prompt 是 Challenger。 +候选不能因为训练集分高就自动成为 Champion，必须经过独立验证与门禁。 +任何证据缺失都不能自动接受候选。 + +训练集 = 用于优化器发现问题； +验证集 = 独立裁判，不能把其详细失败内容喂回优化器。 + +# 必须交付的目录内容 + +在 examples/optimization/eval_optimize_loop/ 内实现： + +- README.md：中文优先，80 行左右，说明目标、运行命令、三种场景、产物。 +- DESIGN.md：简洁说明 Champion/Challenger、数据隔离、gate、fake/live 两种模式。 +- pipeline.py：唯一 CLI 入口。 +- runner.py：运行 Champion 与 Challenger 的评测、冻结输入、确保恢复 Prompt。 +- attribution.py：把失败结果归类为： + reply_mismatch / format_fail / tool_call_error / param_error / + rubric_fail / knowledge_fail / none +- gates.py：纯规则、可独立单测。 +- report.py：输出 optimization_report.json 和 optimization_report.md。 +- data/train.evalset.json：3 条。 +- data/val.evalset.json：3 条。 +- prompts/system.md。 +- 必要的 fake 实现与 tests/。 +- runs/ 输出目录应 gitignore。 + +核心 Python 文件保持精简；不要为了形式拆成大量 package。 + +# 两阶段实现 + +## Milestone A：确定性 fake 模式，必须完整可跑 + +实现 --mode fake，完全不需要 API Key。 + +fake agent 的行为必须由当前 Prompt 中公开、可读的标记决定，例如： +[[FORMAT_JSON]] +[[USE_CORRECT_TOOL]] +[[MEMORIZE_TRAIN]] + +禁止根据 prompt hash 偷偷切换预录答案。 +也禁止伪造总分；必须返回可评测的回答或工具轨迹，再由评测/规则计算结果。 + +必须内置三个可复现情景： + +1. success + Candidate 修复格式或工具问题，train 和 val 都提升，最终 ACCEPT。 + +2. no_effect + Candidate 没有产生有效改善，最终 REJECT。 + +3. overfit + Candidate 只记住 train 样本，train 提升而 val 退化， + 或新增高风险失败，最终 REJECT；理由必须明确提到“过拟合”。 + +## Milestone B：原生 AgentOptimizer 接入 + +实现 --mode optimize： + +- 构造 TargetPrompt； +- 调用： + await AgentOptimizer.optimize( + config_path=..., + call_agent=..., + target_prompt=..., + train_dataset_path=..., + validation_dataset_path=..., + output_dir=..., + update_source=False, + ) +- 从结果中获取 best_prompts 作为 Challenger； +- 使用与 fake 模式相同的 runner / gates / report； +- 即使本地没有真实模型 API Key，也要做到配置缺失时报错清楚； +- fake 模式必须始终能完整跑通。 + +# Gate 规则 + +Gate 必须同时看 train 与 val，输出 ACCEPT 或 REJECT 以及逐条理由。 + +必须包含： + +G1. validation 最小有效提升，例如 min_val_lift=0.02。 +G2. train 提升而 validation 下降，视为 overfit，必拒绝。 +G3. 不新增 hard fail。 +G4. protected case 不退化。 +G5. 关键 slice 平均分不超过 tolerance 地下降。 +G6. 成本证据完整： + - trace/fake 模式可写 cost_status="measured", cost=0， + 因为确实没有模型调用； + - 无法采集真实模型成本时写 cost_status="unavailable"， + 并拒绝自动 ACCEPT，不能把未知成本写成 0。 +G7. 明显微小的分数变化不算有效提升。 + +不要写互相重复的规则。 +每条规则都必须有独立单元测试。 + +# 数据与报告 + +只使用 train / val 两个 evalset，各 3 条，eval_id 不能重叠。 + +报告 optimization_report.json 和 optimization_report.md 必须含： + +- frozen：prompt、dataset、optimizer config、运行模式等哈希/版本信息； +- champion 与 challenger 的来源和 sha256； +- train / val 的 baseline、candidate、delta； +- 每个 case 的状态、分数、delta、slice、risk_level、失败分类； +- decision、违反的 gate、自然语言理由； +- audit：duration_seconds、cost_status、cost、applied、artifact 路径、可复现命令； +- candidate_source：candidate_file 或 agent_optimizer。 + +Markdown 和 JSON 表达同一套核心信息。 + +# Prompt 安全写回 + +候选评测时可临时写入 Challenger，但必须： + +- 先保存 Champion snapshot； +- 用 try/finally 恢复； +- 测试 dry-run 后 prompts/system.md 的 sha256 不变； +- --apply + ACCEPT 才真正写回； +- --apply + REJECT 必须报出原因且源文件不变； +- 备份和写回也使用 TargetPrompt 的机制，不要 Path.write_text 另起一套逻辑。 + +# 测试要求 + +新增 tests 至少覆盖： + +1. train / val eval_id 不重叠； +2. 六类失败归因； +3. G1-G7； +4. success / no_effect / overfit 三个端到端场景； +5. dry-run 后 Champion 不变； +6. --apply 成功与拒绝； +7. 无 API Key 的 fake mode； +8. 报告字段完整； +9. 目标目录测试总时长小于 180 秒。 + +先写测试，再实现对应最小代码。 + +# 验证命令 + +只运行： + +python -m pytest examples/optimization/eval_optimize_loop/tests -q +python examples/optimization/eval_optimize_loop/pipeline.py --mode fake --scenario success +python examples/optimization/eval_optimize_loop/pipeline.py --mode fake --scenario no_effect +python examples/optimization/eval_optimize_loop/pipeline.py --mode fake --scenario overfit +python -m compileall -q examples/optimization/eval_optimize_loop +git diff --check +git status -sb + +# 最终回复格式 + +完成后按以下格式汇报： + +1. 已修改/新增文件； +2. 每个阶段如何对应 Issue #91； +3. 三个 fake 场景的实际输出与 decision； +4. pytest 结果、耗时、其他验证结果； +5. 未验证的 live optimize 风险； +6. git diff 摘要； +7. 不要提交或推送。 \ No newline at end of file diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 000000000..a3c08ea84 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,78 @@ +# Evaluation + Optimization 闭环示例 + +本示例把 `AgentEvaluator` 与 `AgentOptimizer` 串成可复现的 +Champion–Challenger 流程：冻结输入、评测基线、归因失败、生成候选、 +重新评测、执行 Gate，并保存 JSON/Markdown 报告。默认 dry-run；只有 +显式传入 `--apply` 且全部 Gate 通过时才更新源 Prompt。 + +## 无 API Key 的管线回归 + +```powershell +python .\examples\optimization\eval_optimize_loop\pipeline.py --mode fake --scenario success +python .\examples\optimization\eval_optimize_loop\pipeline.py --mode fake --scenario no_effect +python .\examples\optimization\eval_optimize_loop\pipeline.py --mode fake --scenario overfit +``` + +| 场景 | 明文 `FAKE_CONTROLS` | 预期结果 | +|---|---|---| +| `success` | `ADD_STEPS=true` | train/val 均提升，ACCEPT | +| `no_effect` | 两项均为 `false` | 无有效提升，REJECT G1 | +| `overfit` | `MEMORIZE_TRAIN=true` | train 提升、val 退化，REJECT G2 | + +Fake 模式根据 Prompt 内的公开控制项生成 trace，再交给真实 +`AgentEvaluator` 评分；它只用于确定性管线回归，不证明隐藏样本泛化能力， +也不替代原生优化器。 + +## 原生 optimize + +```powershell +$env:TRPC_AGENT_API_KEY="..." +$env:TRPC_AGENT_BASE_URL="https://..." +$env:TRPC_AGENT_MODEL_NAME="..." +python .\examples\optimization\eval_optimize_loop\pipeline.py --mode optimize --optimizer-config optimizer.json +``` + +该路径使用真实模型回调并实际调用 +`AgentOptimizer.optimize(..., update_source=False)`。Optimizer 仅看到训练集； +返回的 `best_prompts["system"]` 作为 Challenger，再用同一 train/val 数据和 +Gate 做回归。缺少配置、优化异常或未产出 Candidate 时仍会生成 +`OPTIMIZER_FAILURE + G6` 的 REJECT 报告。模型端未返回可信 token/费用时， +报告写 `cost_status="unavailable"`、数值为 `null`，并禁止自动写回。 + +## Gate 与写回 + +G1 验证集最小提升；G2 训练涨而验证跌；G3 不新增高风险失败；G4 protected +case 不退化；G5 slice 退化不超阈值；G6 成本证据完整且不超预算；G7 +过滤 epsilon 内的微小变化。阈值见 `run.json`。 + +```powershell +# 只有 ACCEPT 才会通过 TargetPrompt 原子写回 +python .\examples\optimization\eval_optimize_loop\pipeline.py --mode fake --scenario success --apply +``` + +REJECT 加 `--apply` 返回退出码 2，Champion 保持不变。评测临时切换和测试恢复 +也统一使用 `TargetPrompt.read_all/write_all`。 + +## 数据、证据与产物 + +- `data/train.evalset.json`、`data/val.evalset.json`：各 3 条，eval_id 互斥。 +- `data/attribution_holdout.json`：公开标注证据集，不是官方隐藏集。 +- `optimization_report.json/.md`：最新决策及逐 case 证据。 +- `optimization_report.example.json`：已脱敏的报告结构示例。 +- `runs/-<随机后缀>/`：冻结清单、Prompt 快照、完整 evaluator + 结果、优化轮次、调用审计与报告；该目录被 Git 忽略。 + +逐 case 报告保存 actual/expected response、metric reason、tool use/response、 +参数差异和 trace 引用，并区分 agent-quality、infrastructure failure 与 +`insufficient_evidence`。官方隐藏集未提供，因此不声称满足隐藏样本准确率； +公开归因 holdout 的实际统计由测试计算。 + +## 验证 + +```powershell +python -m pytest .\examples\optimization\eval_optimize_loop\tests -q +python -m compileall -q .\examples\optimization\eval_optimize_loop +``` + +测试包含三种 fake 场景、mock native optimizer 完整闭环、真实 evaluator +工具/参数证据、成本不可用拒绝 apply、Prompt 恢复、Gate 和报告 schema。 diff --git a/examples/optimization/eval_optimize_loop/attribution.py b/examples/optimization/eval_optimize_loop/attribution.py new file mode 100644 index 000000000..88764ddc8 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/attribution.py @@ -0,0 +1,293 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Evidence-based failure attribution for evaluation cases.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Optional + +CATEGORIES = ( + "reply_mismatch", + "format_fail", + "tool_call_error", + "param_error", + "rubric_fail", + "knowledge_fail", + "infrastructure_failure", + "insufficient_evidence", + "none", +) + + +@dataclass(frozen=True) +class AttributionInput: + status: str + metric_name_failed: Optional[str] = None + metric_names_failed: tuple[str, ...] = () + metric_reasons: tuple[str, ...] = () + expected_text: Optional[str] = None + actual_text: Optional[str] = None + expected_tool_uses: tuple[dict[str, Any], ...] = () + actual_tool_uses: tuple[dict[str, Any], ...] = () + expected_tool_responses: tuple[dict[str, Any], ...] = () + actual_tool_responses: tuple[dict[str, Any], ...] = () + error_message: Optional[str] = None + # Backward-compatible fields used by older callers/tests. + scenario_tag: Optional[str] = None + score: float = 1.0 + has_tool_calls_actual: bool = False + has_tool_calls_expected: bool = False + has_tool_param_mismatch: bool = False + has_knowledge_response: bool = True + + +@dataclass(frozen=True) +class AttributionResult: + category: str + reason: str + evidence: dict[str, Any] = field(default_factory=dict) + + +def _tool_name(tool: dict[str, Any]) -> str: + return str(tool.get("name") or "") + + +def _tool_args(tool: dict[str, Any]) -> dict[str, Any]: + args = tool.get("args") + return args if isinstance(args, dict) else {} + + +def _tool_mismatch(inp: AttributionInput) -> tuple[bool, bool]: + expected = {_tool_name(tool): tool for tool in inp.expected_tool_uses if _tool_name(tool)} + actual = {_tool_name(tool): tool for tool in inp.actual_tool_uses if _tool_name(tool)} + if not expected: + return False, False + missing_or_wrong = bool(set(expected) - set(actual)) + parameter_error = any( + _tool_args(expected[name]) != _tool_args(actual[name]) for name in set(expected) & set(actual) + ) + return missing_or_wrong, parameter_error + + +def _parameter_differences( + expected_tools: tuple[dict[str, Any], ...], + actual_tools: tuple[dict[str, Any], ...], +) -> list[dict[str, Any]]: + expected = {_tool_name(tool): tool for tool in expected_tools if _tool_name(tool)} + actual = {_tool_name(tool): tool for tool in actual_tools if _tool_name(tool)} + return [ + { + "tool_name": name, + "expected_args": _tool_args(expected[name]), + "actual_args": _tool_args(actual[name]), + } + for name in sorted(set(expected) & set(actual)) + if _tool_args(expected[name]) != _tool_args(actual[name]) + ] + + +def _failed_metric_names(inp: AttributionInput) -> tuple[str, ...]: + names = list(inp.metric_names_failed) + if inp.metric_name_failed and inp.metric_name_failed not in names: + names.append(inp.metric_name_failed) + return tuple(str(name) for name in names) + + +def classify_with_reason(inp: AttributionInput) -> AttributionResult: + """Classify one failure without relying on scenario labels.""" + + if inp.status == "PASSED": + return AttributionResult("none", "case 已通过，无失败归因。") + + if inp.error_message or inp.status == "NOT_EVALUATED": + reason = inp.error_message or "评测未完成，缺少可用的 agent 输出。" + return AttributionResult( + "infrastructure_failure", + f"基础设施或评测执行失败：{reason}", + {"error_message": inp.error_message, "status": inp.status}, + ) + + metric_names = _failed_metric_names(inp) + lowered_metrics = tuple(name.lower() for name in metric_names) + reasons_text = " ".join(inp.metric_reasons).lower() + + if any("knowledge" in name for name in lowered_metrics) or any( + marker in reasons_text for marker in ("knowledge", "recall", "知识", "召回") + ): + return AttributionResult( + "knowledge_fail", + "知识召回证据未达到评测要求。", + {"failed_metrics": metric_names, "metric_reasons": inp.metric_reasons}, + ) + + if any("rubric" in name for name in lowered_metrics): + return AttributionResult( + "rubric_fail", + "LLM rubric 指标未达到阈值。", + {"failed_metrics": metric_names, "metric_reasons": inp.metric_reasons}, + ) + + expected_tools = inp.expected_tool_uses + actual_tools = inp.actual_tool_uses + if inp.has_tool_calls_expected and not expected_tools: + expected_tools = ({"name": ""},) + if inp.has_tool_calls_actual and not actual_tools: + actual_tools = ({"name": ""},) + normalized = AttributionInput( + **{ + **inp.__dict__, + "expected_tool_uses": expected_tools, + "actual_tool_uses": actual_tools, + } + ) + missing_or_wrong, parameter_error = _tool_mismatch(normalized) + if inp.has_tool_param_mismatch: + parameter_error = True + if parameter_error: + return AttributionResult( + "param_error", + "工具名称匹配，但调用参数与期望轨迹不一致。", + { + "expected_tool_uses": expected_tools, + "actual_tool_uses": actual_tools, + "parameter_differences": _parameter_differences(expected_tools, actual_tools), + }, + ) + if missing_or_wrong or (expected_tools and not actual_tools): + return AttributionResult( + "tool_call_error", + "缺少期望工具调用，或调用了错误的工具。", + {"expected_tool_uses": expected_tools, "actual_tool_uses": actual_tools}, + ) + + if inp.expected_text is not None and inp.actual_text is not None: + if _answer_numbers_match(inp.expected_text, inp.actual_text): + return AttributionResult( + "format_fail", + "答案核心数值一致，但回复格式或必要结构不符合要求。", + {"expected_text": inp.expected_text, "actual_text": inp.actual_text}, + ) + return AttributionResult( + "reply_mismatch", + "最终回复与参考答案不匹配。", + {"expected_text": inp.expected_text, "actual_text": inp.actual_text}, + ) + + return AttributionResult( + "insufficient_evidence", + "证据不足：缺少可核对的 actual/expected response 或工具轨迹，" "不能仅凭笼统 metric reason 可靠归因。", + { + "failed_metrics": metric_names, + "metric_reasons": inp.metric_reasons, + "status": inp.status, + }, + ) + + +def classify(inp: AttributionInput) -> str: + """Backward-compatible category-only API.""" + + return classify_with_reason(inp).category + + +def from_case_record(case: Any) -> AttributionResult: + failed_metrics = tuple( + str(metric.get("metric_name", "")) for metric in case.metric_results if metric.get("eval_status") != "PASSED" + ) + metric_reasons = tuple(str(reason) for reason in case.failure_reasons if reason) + return classify_with_reason( + AttributionInput( + status=case.challenger_status, + metric_names_failed=failed_metrics, + metric_reasons=metric_reasons, + expected_text=case.expected_text, + actual_text=case.actual_text, + expected_tool_uses=tuple(case.expected_tool_uses), + actual_tool_uses=tuple(case.actual_tool_uses), + expected_tool_responses=tuple(case.expected_tool_responses), + actual_tool_responses=tuple(case.actual_tool_responses), + error_message=case.error_message, + ) + ) + + +def from_eval_case_result( + result: Any, + *, + scenario_tag: Optional[str] = None, + expected_text: Optional[str] = None, +) -> str: + """Attribute directly from the public EvalCaseResult fields.""" + + failed_metrics: list[str] = [] + reasons: list[str] = [] + for metric in result.overall_eval_metric_results or []: + status = getattr(metric.eval_status, "name", str(metric.eval_status)) + if status != "PASSED": + failed_metrics.append(metric.metric_name) + details = getattr(metric, "details", None) + reason = getattr(details, "reason", None) if details else None + if reason: + reasons.append(str(reason)) + + actual_text: Optional[str] = None + expected_tools: list[dict[str, Any]] = [] + actual_tools: list[dict[str, Any]] = [] + for per_invocation in result.eval_metric_result_per_invocation or []: + actual = per_invocation.actual_invocation + expected = per_invocation.expected_invocation + if actual_text is None: + actual_text = _content_to_text(getattr(actual, "final_response", None)) + actual_tools.extend(_intermediate_tools(actual)) + if expected is not None: + expected_tools.extend(_intermediate_tools(expected)) + if expected_text is None: + expected_text = _content_to_text(getattr(expected, "final_response", None)) + + status = getattr(result.final_eval_status, "name", str(result.final_eval_status)) + return classify( + AttributionInput( + status=status, + metric_names_failed=tuple(failed_metrics), + metric_reasons=tuple(reasons), + expected_text=expected_text, + actual_text=actual_text, + expected_tool_uses=tuple(expected_tools), + actual_tool_uses=tuple(actual_tools), + error_message=result.error_message, + ) + ) + + +def _content_to_text(content: Any) -> Optional[str]: + parts = getattr(content, "parts", None) or [] + texts = [str(getattr(part, "text", "")) for part in parts if getattr(part, "text", None)] + return "\n".join(texts) if texts else None + + +def _intermediate_tools(invocation: Any) -> list[dict[str, Any]]: + intermediate = getattr(invocation, "intermediate_data", None) + tools = getattr(intermediate, "tool_uses", None) or [] + return [ + ( + tool.model_dump(mode="json", by_alias=True) + if hasattr(tool, "model_dump") + else {"name": getattr(tool, "name", ""), "args": getattr(tool, "args", {})} + ) + for tool in tools + ] + + +def _answer_numbers_match(expected: str, actual: str) -> bool: + expected_numbers = re.findall(r"-?\d+(?:\.\d+)?", expected) + actual_numbers = re.findall(r"-?\d+(?:\.\d+)?", actual) + if not expected_numbers or not actual_numbers: + return False + return expected_numbers[-1] == actual_numbers[-1] diff --git a/examples/optimization/eval_optimize_loop/data/attribution_holdout.json b/examples/optimization/eval_optimize_loop/data/attribution_holdout.json new file mode 100644 index 000000000..05c28999c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/attribution_holdout.json @@ -0,0 +1,73 @@ +{ + "description": "公开、可复现的人工标注 evaluator-evidence holdout；不是官方隐藏集。", + "cases": [ + { + "eval_id": "holdout_reply", + "expected_category": "reply_mismatch", + "status": "FAILED", + "failed_metrics": ["final_response_avg_score"], + "metric_reasons": ["final response text mismatch"], + "expected_text": "答案：11", + "actual_text": "答案：22" + }, + { + "eval_id": "holdout_format", + "expected_category": "format_fail", + "status": "FAILED", + "failed_metrics": ["final_response_avg_score"], + "metric_reasons": ["required answer structure is missing"], + "expected_text": "步骤：4 + 7 = 11\n答案：11 个", + "actual_text": "11" + }, + { + "eval_id": "holdout_tool", + "expected_category": "tool_call_error", + "status": "FAILED", + "failed_metrics": ["tool_trajectory_avg_score"], + "metric_reasons": ["expected tool call was not found"], + "expected_tool_uses": [{"id": "t1", "name": "search", "args": {"query": "pricing"}}], + "actual_tool_uses": [] + }, + { + "eval_id": "holdout_param", + "expected_category": "param_error", + "status": "FAILED", + "failed_metrics": ["tool_trajectory_avg_score"], + "metric_reasons": ["tool arguments did not match"], + "expected_tool_uses": [{"id": "t1", "name": "search", "args": {"query": "pricing", "top_k": 3}}], + "actual_tool_uses": [{"id": "a1", "name": "search", "args": {"query": "weather", "top_k": 1}}] + }, + { + "eval_id": "holdout_rubric", + "expected_category": "rubric_fail", + "status": "FAILED", + "failed_metrics": ["llm_rubric_response"], + "metric_reasons": ["rubric reasoning_clear failed"], + "expected_text": "清晰解释计算过程", + "actual_text": "答案是 20" + }, + { + "eval_id": "holdout_knowledge", + "expected_category": "knowledge_fail", + "status": "FAILED", + "failed_metrics": ["llm_rubric_knowledge_recall"], + "metric_reasons": ["knowledge recall does not support rubric kr1"], + "expected_tool_uses": [{"id": "t1", "name": "knowledge_search", "args": {"query": "产品"}}], + "actual_tool_uses": [{"id": "a1", "name": "knowledge_search", "args": {"query": "产品"}}], + "actual_tool_responses": [{"id": "a1", "name": "knowledge_search", "response": {"documents": []}}] + }, + { + "eval_id": "holdout_infra", + "expected_category": "infrastructure_failure", + "status": "NOT_EVALUATED", + "error_message": "TimeoutError: model endpoint unavailable" + }, + { + "eval_id": "holdout_insufficient", + "expected_category": "insufficient_evidence", + "status": "FAILED", + "failed_metrics": ["final_response_avg_score"], + "metric_reasons": ["score below threshold"] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/test_config.json b/examples/optimization/eval_optimize_loop/data/test_config.json new file mode 100644 index 000000000..2fd719a93 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/test_config.json @@ -0,0 +1,17 @@ +{ + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "contains", + "case_insensitive": true + } + } + } + } + ], + "num_runs": 1 +} diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json new file mode 100644 index 000000000..de38a250c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -0,0 +1,88 @@ +{ + "eval_set_id": "eval_optimize_loop_train", + "name": "eval_optimize_loop 训练集（3 条）", + "description": "用于优化器发现问题。覆盖三种可优化情景：可优化成功 (T-ADD)、优化无效 (T-NOOP)、优化后退退 (T-OVERFIT-MEM)。", + "eval_cases": [ + { + "eval_id": "train_add_steps", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "train", + "state": { + "split": "train", + "slice": "math", + "risk_level": "low", + "protected": false, + "scenario_tag": "add_steps" + } + }, + "conversation": [ + { + "invocation_id": "t1", + "user_content": { + "parts": [{"text": "小明买了 4 个苹果又买了 7 个苹果，他一共有多少个苹果？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "步骤：4 + 7 = 11\n答案：11 个"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "train_noop", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "train", + "state": { + "split": "train", + "slice": "noop", + "risk_level": "low", + "protected": false, + "scenario_tag": "noop_ambiguous" + } + }, + "conversation": [ + { + "invocation_id": "t2", + "user_content": { + "parts": [{"text": "一个含糊不清的问题，候选 prompt 无论怎么改都答不对。"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：42"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "train_overfit_mem", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "train", + "state": { + "split": "train", + "slice": "math", + "risk_level": "low", + "protected": false, + "scenario_tag": "memorize_train" + } + }, + "conversation": [ + { + "invocation_id": "t3", + "user_content": { + "parts": [{"text": "一件衣服原价 200 元，打 8 折，折后价是多少？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：160 元"}], + "role": "model" + } + } + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json new file mode 100644 index 000000000..b39acbfc1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -0,0 +1,88 @@ +{ + "eval_set_id": "eval_optimize_loop_val", + "name": "eval_optimize_loop 验证集（3 条，独立裁判）", + "description": "eval_id 与 train 互斥。包含一条 protected case、一条 high risk case、一条 noop。", + "eval_cases": [ + { + "eval_id": "val_add_steps_b", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "val", + "state": { + "split": "val", + "slice": "math", + "risk_level": "low", + "protected": false, + "scenario_tag": "add_steps" + } + }, + "conversation": [ + { + "invocation_id": "v1", + "user_content": { + "parts": [{"text": "小轿车一共有 4 个轮子，5 辆小轿车一共有多少个轮子？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "步骤：4 × 5 = 20\n答案：20 个"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "val_protected_highrisk", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "val", + "state": { + "split": "val", + "slice": "risky", + "risk_level": "high", + "protected": true, + "scenario_tag": "protected_must_pass" + } + }, + "conversation": [ + { + "invocation_id": "v2", + "user_content": { + "parts": [{"text": "班里 40 名学生，25% 戴眼镜，戴眼镜的有多少人？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "步骤：40 × 25% = 10\n答案：10 人"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "val_noop_b", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "val", + "state": { + "split": "val", + "slice": "noop", + "risk_level": "low", + "protected": false, + "scenario_tag": "noop_ambiguous" + } + }, + "conversation": [ + { + "invocation_id": "v3", + "user_content": { + "parts": [{"text": "另一个含糊不清的问题，候选 prompt 无论怎么改都答不对。"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：42"}], + "role": "model" + } + } + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/fake_agent.py b/examples/optimization/eval_optimize_loop/fake_agent.py new file mode 100644 index 000000000..1abc61ea1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake_agent.py @@ -0,0 +1,221 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Milestone A 的 fake agent。 + +行为**完全由 prompt 中公开、明确的 FAKE_CONTROLS 控制块**决定： + + + +只有该块内、且值严格等于 ``true``（大小写敏感）才启用对应行为。 +注释文字、文档叙述、示例代码里出现的同名关键字**不会**误触发。 + +决策矩阵（与 README 真值表一致）： + +| 控制块 | train case | val protected | val 其他 | +|---------------------------------|------------|---------------|----------| +| 全 false（baseline） | 错 | 对 | 错 | +| ADD_STEPS=true | 对 | 对 | 对 | +| MEMORIZE_TRAIN=true | 对 | 错 | 错 | + +- "对" = 输出 evalset 中该 case 的 ``final_response`` 原文（满足 contains 评测） +- "错" = 输出固定错误文本 + +成功 case 的输出**直接来自 evalset 中 case.conversation[0].final_response**， +避免与 contains 评测再次漂移。 + +禁止行为（test_no_sha_switch 强制）： + - 通过任何形式的哈希指纹选择预录答案 + - 通过身份关键字偏置结果 + - 读取 scenario 参数 / eval_id 内容决定答案 +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Optional + +# ---- FAKE_CONTROLS 控制块解析 ---- +# 多行块： 之间按 `KEY=value` 行解析 +_FAKE_CONTROLS_RE = re.compile( + r"", + re.DOTALL | re.IGNORECASE, +) +_KV_RE = re.compile(r"^\s*([A-Z_]+)\s*=\s*(\S+)\s*$") + + +@dataclass(frozen=True) +class FakeBehavior: + """根据 FAKE_CONTROLS 块推断出的行为集合。 + + 只有严格等于字符串 "true" 才视为启用，避免 "false" / "True" / 注释文字误触发。 + """ + + add_steps: bool = False + memorize_train: bool = False + + +def parse_behavior(prompt_text: str) -> FakeBehavior: + """从 prompt 文本中提取 FAKE_CONTROLS 块并解析。 + + - 不存在控制块 → baseline（全 false） + - 存在控制块 → 仅接受 ``KEY=true``（严格小写 true）作为启用 + - 其他值（false / True / 注释叙述）一律视为 false + + >>> parse_behavior("no block here").add_steps + False + >>> parse_behavior("").add_steps + True + >>> parse_behavior("").add_steps + False + """ + add_steps = False + memorize_train = False + for match in _FAKE_CONTROLS_RE.finditer(prompt_text or ""): + body = match.group(1) + for line in body.splitlines(): + kv = _KV_RE.match(line) + if not kv: + continue + key, val = kv.group(1), kv.group(2) + enabled = val == "true" + if key == "ADD_STEPS": + add_steps = enabled or add_steps + elif key == "MEMORIZE_TRAIN": + memorize_train = enabled or memorize_train + return FakeBehavior(add_steps=add_steps, memorize_train=memorize_train) + + +# 错误文本（与所有 expected_response 都不含的关键短语，确保 contains 失败） +_WRONG_TEXT = "抱歉，我无法回答这个问题。" + + +def _expected_text_from_case(case: dict) -> Optional[str]: + """从 evalset 单个 case 中取出 reference final_response 的纯文本。""" + conv = case.get("conversation") or [] + if not conv: + return None + final = conv[0].get("final_response") or {} + parts = final.get("parts") or [] + texts = [p.get("text", "") for p in parts if p.get("text")] + return "\n".join(texts) if texts else None + + +def _is_protected(case: dict) -> bool: + state = (case.get("session_input") or {}).get("state") or {} + return bool(state.get("protected", False)) + + +def _is_train(case: dict) -> bool: + state = (case.get("session_input") or {}).get("state") or {} + return state.get("split") == "train" + + +def gen_final_response_for_case( + prompt_text: str, + *, + case: dict, +) -> str: + """根据当前 prompt 行为 + evalset 中的 case，返回最终回复文本。 + + 决策完全基于 FAKE_CONTROLS 块 + case 的 split/protected 字段； + 不读取 scenario 名、eval_id 内容、prompt hash、自然语言 query。 + """ + behavior = parse_behavior(prompt_text) + expected = _expected_text_from_case(case) or "" + is_train = _is_train(case) + is_protected = _is_protected(case) + + if behavior.add_steps: + return expected + if behavior.memorize_train: + return expected if is_train else _WRONG_TEXT + # baseline + return expected if is_protected else _WRONG_TEXT + + +def gen_actual_conversation(prompt_text: str, evalset_dict: dict) -> list: + """对一个 evalset 中的每个 case 生成 actual_conversation，返回新的 eval_cases list.""" + new_cases = [] + for case in evalset_dict.get("eval_cases", []): + eval_id = case["eval_id"] + conv = case.get("conversation") or case.get("actual_conversation") or [] + if not conv: + continue + first_inv = conv[0] + + actual_text = gen_final_response_for_case(prompt_text, case=case) + actual_invocation = { + "invocation_id": first_inv.get("invocation_id", "act-1"), + "user_content": first_inv["user_content"], + "final_response": { + "parts": [{"text": actual_text}], + "role": "model", + }, + } + + new_case = dict(case) + new_case["eval_mode"] = "trace" + new_case["actual_conversation"] = [actual_invocation] + # trace 模式下保留 conversation 作为 reference + if "conversation" not in new_case: + new_case["conversation"] = conv + new_cases.append(new_case) + return new_cases + + +# scenario → candidate prompt 模板。 +# 所有 candidate 通过显式 FAKE_CONTROLS 块声明行为，不靠自然语言文字。 +SUCCESS_CANDIDATE = """# System Prompt (Candidate - Success) + +你是一个严谨的数学问答助手。 + +## 输出格式 +所有回答必须按"步骤：...\\n答案：..."的格式给出最终答案。 + + +""" + +NO_EFFECT_CANDIDATE = """# System Prompt (Candidate - NoEffect) + +你是一个数学问答助手。本候选不启用任何 fake 改进标记，行为与初始版本一致。 + + +""" + +OVERFIT_CANDIDATE = """# System Prompt (Candidate - Overfit) + +本候选只对训练集记忆答案，验证集（含受保护样本）会答错。 + + +""" + +CANDIDATES = { + "success": SUCCESS_CANDIDATE, + "no_effect": NO_EFFECT_CANDIDATE, + "overfit": OVERFIT_CANDIDATE, +} + + +def build_candidate(scenario: str) -> str: + """根据 scenario 名返回对应 candidate prompt 文本。""" + if scenario not in CANDIDATES: + raise ValueError(f"unknown scenario: {scenario}, expected one of {list(CANDIDATES)}") + return CANDIDATES[scenario] diff --git a/examples/optimization/eval_optimize_loop/gates.py b/examples/optimization/eval_optimize_loop/gates.py new file mode 100644 index 000000000..9d359bb82 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/gates.py @@ -0,0 +1,180 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Gate：纯规则决策，输入两个 split 的 case-level delta，输出 ACCEPT/REJECT。 + +对应 Issue #91 的 G1-G7 接受策略： + + G1 validation 最小有效提升 val_delta >= min_val_lift + G2 train 涨 val 跌 = 过拟合 not (train_delta > min_val_lift and val_delta < -min_val_lift) + G3 不新增 hard fail risk_level==high 的 case 不能从 PASSED→FAILED + G4 protected case 不退化 protected==true 的 case 不能分数下降 + G5 slice 平均退化不超过 tolerance 每个 slice 平均 delta >= -slice_tol + G6 成本证据完整 cost_status != "unavailable" and total_tokens <= budget + G7 明显微小变化不算有效提升 候选接受时，关键 delta 必须 > epsilon + +不写互相重复的规则：G1 看 val 整体；G5 看 slice；G3/G4 看个体风险；G6 看成本；G7 防 epsilon 内噪声。 +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Iterable, List, Optional + + +@dataclass(frozen=True) +class CaseDelta: + """单 case 的对比信息。 + + Attributes: + eval_id: case id + split: train | val + slice_name: 来自 state.slice + risk_level: low | mid | high + protected: bool + champion_status: PASSED | FAILED + challenger_status: PASSED | FAILED + champion_score: 0~1 + challenger_score: 0~1 + delta: challenger - champion + """ + + eval_id: str + split: str + slice_name: str + risk_level: str = "low" + protected: bool = False + champion_status: str = "FAILED" + challenger_status: str = "FAILED" + champion_score: float = 0.0 + challenger_score: float = 0.0 + + @property + def delta(self) -> float: + return self.challenger_score - self.champion_score + + @property + def new_hard_fail(self) -> bool: + """risk_level==high 且 champion 通过但 challenger 失败。""" + return self.risk_level == "high" and self.champion_status == "PASSED" and self.challenger_status != "PASSED" + + +@dataclass +class GateConfig: + """Gate 阈值。所有字段必须可从 run.json 反序列化。""" + + min_val_lift: float = 0.02 + slice_tolerance: float = 0.05 + budget_tokens: int = 100_000 + budget_usd: Optional[float] = None + epsilon: float = 0.001 + + +@dataclass +class Decision: + accepted: bool + violated: List[str] = field(default_factory=list) + reasons: List[str] = field(default_factory=list) + + +def _avg(values: Iterable[float]) -> float: + vs = list(values) + return sum(vs) / len(vs) if vs else 0.0 + + +def _slice_avg_deltas(cases: List[CaseDelta]): + """按 slice 聚合 delta，返回 {slice_name: avg_delta}。""" + groups: dict[str, list[float]] = {} + for c in cases: + groups.setdefault(c.slice_name, []).append(c.delta) + return {k: _avg(v) for k, v in groups.items()} + + +def evaluate( + cases: List[CaseDelta], + *, + cost_status: str = "measured", + total_tokens: Optional[int] = 0, + total_cost: Optional[float] = 0.0, + config: Optional[GateConfig] = None, +) -> Decision: + """主入口。 + + Args: + cases: train + val 两个 split 全部 case 的 delta（合并传入） + cost_status: measured / unavailable + total_tokens: 实际消耗 token；fake/trace 模式为 0 + config: gate 阈值，None 用默认 + """ + cfg = config or GateConfig() + violated: List[str] = [] + reasons: List[str] = [] + + train_cases = [c for c in cases if c.split == "train"] + val_cases = [c for c in cases if c.split == "val"] + train_delta = _avg(c.delta for c in train_cases) + val_delta = _avg(c.delta for c in val_cases) + + # G2 必须先于 G1：先识别"训练涨 / val 跌"的过拟合，再判 val 是否涨够。 + # G2: overfit + if train_delta > cfg.min_val_lift and val_delta < -cfg.min_val_lift: + violated.append("G2") + reasons.append( + f"过拟合：train_delta={train_delta:+.4f} 上升而 " f"val_delta={val_delta:+.4f} 下降，候选只记住训练集。" + ) + + # G1: val minimal lift + if val_delta < cfg.min_val_lift: + violated.append("G1") + reasons.append(f"验证集无有效提升：val_delta={val_delta:+.4f} < " f"min_val_lift={cfg.min_val_lift:.4f}。") + + # G3: no new hard fail + new_hard = [c.eval_id for c in cases if c.new_hard_fail] + if new_hard: + violated.append("G3") + reasons.append(f"新增 hard fail（risk_level==high 由 PASSED→FAILED）：{new_hard}。") + + # G4: protected case no regression + protected_reg = [c.eval_id for c in cases if c.protected and c.delta < -cfg.epsilon] + if protected_reg: + violated.append("G4") + reasons.append(f"protected case 退化：{protected_reg}。") + + # G5: slice tolerance + slice_deltas = _slice_avg_deltas(cases) + bad_slices = {k: v for k, v in slice_deltas.items() if v < -cfg.slice_tolerance} + if bad_slices: + violated.append("G5") + reasons.append( + f"slice 退化超过 tolerance={cfg.slice_tolerance:.4f}：" + f"{ {k: round(v, 4) for k, v in bad_slices.items()} }。" + ) + + # G6: cost evidence + if cost_status == "unavailable": + violated.append("G6") + reasons.append("成本证据缺失（cost_status=unavailable），不可自动 ACCEPT。") + elif total_tokens is None or total_cost is None: + violated.append("G6") + reasons.append("成本状态标为 measured，但 token/cost 证据为空，不可自动 ACCEPT。") + elif total_tokens > cfg.budget_tokens: + violated.append("G6") + reasons.append(f"超成本预算：total_tokens={total_tokens} > " f"budget_tokens={cfg.budget_tokens}。") + elif cfg.budget_usd is not None and total_cost > cfg.budget_usd: + violated.append("G6") + reasons.append(f"超成本预算：total_cost={total_cost:.6f} > " f"budget_usd={cfg.budget_usd:.6f}。") + + # G7: epsilon guard —— 候选接受时，val_delta 必须严格 > epsilon（非噪声） + # 仅在前面 6 条全过时才检查；它防的是 "val_delta==0.0001" 这种几乎为零的波动 + # 被 G1 当作有效提升（因为 G1 用 >= min_val_lift 默认 0.02 已经隐含保护， + # 但当用户把 min_val_lift 调到 0 时 G7 仍是最后一道闸）。 + if not violated and val_delta <= cfg.epsilon: + violated.append("G7") + reasons.append(f"val_delta={val_delta:+.4f} ≤ epsilon={cfg.epsilon:.4f}，" "属于明显微小变化，不视为有效提升。") + + accepted = not violated + return Decision(accepted=accepted, violated=violated, reasons=reasons) diff --git a/examples/optimization/eval_optimize_loop/live_agent.py b/examples/optimization/eval_optimize_loop/live_agent.py new file mode 100644 index 000000000..f4630d04e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/live_agent.py @@ -0,0 +1,102 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Real model-backed call_agent used by native AgentOptimizer mode.""" + +from __future__ import annotations + +import os +import uuid +from pathlib import Path +from typing import Any, Awaitable, Callable + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.models import OpenAIModel +from trpc_agent_sdk.runners import Runner +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.types import Content, GenerateContentConfig, Part + +CallAgent = Callable[[str], Awaitable[str]] +APP_NAME = "eval_optimize_loop" + + +def model_info_from_env() -> dict[str, Any]: + """Return non-secret model identity for the frozen manifest.""" + + return { + "provider": os.environ.get("TRPC_AGENT_PROVIDER_NAME", "openai"), + "model_name": os.environ.get("TRPC_AGENT_MODEL_NAME", ""), + # Do not persist an endpoint that might embed credentials. + "base_url_configured": bool(os.environ.get("TRPC_AGENT_BASE_URL")), + "usage_tracking": False, + } + + +def validate_model_env() -> None: + missing = [ + key for key in ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME") if not os.environ.get(key) + ] + if missing: + raise RuntimeError( + "native optimize 需要模型配置：" + + ", ".join(f"${key}" for key in missing) + + "；无需 API Key 的回归演示请使用 --mode fake。" + ) + + +def build_call_agent(prompt_path: Path) -> CallAgent: + """Build a callback that reloads the current TargetPrompt on every call.""" + + validate_model_env() + api_key = os.environ["TRPC_AGENT_API_KEY"] + base_url = os.environ["TRPC_AGENT_BASE_URL"] + model_name = os.environ["TRPC_AGENT_MODEL_NAME"] + + async def call_agent(query: str) -> str: + instruction = prompt_path.read_text(encoding="utf-8") + model = OpenAIModel(model_name=model_name, api_key=api_key, base_url=base_url) + agent = LlmAgent( + name="eval_optimize_loop_agent", + description="Agent evaluated and optimized by Issue #91 pipeline.", + model=model, + instruction=instruction, + generate_content_config=GenerateContentConfig( + temperature=0.2, + top_p=0.9, + max_output_tokens=2048, + ), + ) + sessions = InMemorySessionService() + runner = Runner(app_name=APP_NAME, agent=agent, session_service=sessions) + session_id = str(uuid.uuid4()) + user_id = "optimizer" + await sessions.create_session( + app_name=APP_NAME, + user_id=user_id, + session_id=session_id, + state={}, + ) + message = Content(role="user", parts=[Part.from_text(text=query)]) + response = "" + async for event in runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=message, + ): + if not event.is_final_response() or not event.content: + continue + for part in event.content.parts or []: + if not part.thought and part.text: + response += part.text + return response.strip() + + # The SDK runner does not expose a provider-independent USD amount here. + # Treat the whole optimize run as unmeasured so G6 blocks auto-apply. + call_agent.cost_status = "unavailable" + call_agent.total_tokens = None + call_agent.total_cost = None + return call_agent diff --git a/examples/optimization/eval_optimize_loop/optimization_report.example.json b/examples/optimization/eval_optimize_loop/optimization_report.example.json new file mode 100644 index 000000000..d7255f48c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimization_report.example.json @@ -0,0 +1,345 @@ +{ + "version": "2.0", + "candidate_source": "agent_optimizer", + "frozen": { + "run_id": "2026-07-28T00-00-00.000000Z-a1b2c3d4", + "champion_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "challenger_sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "train_sha256": "3333333333333333333333333333333333333333333333333333333333333333", + "val_sha256": "4444444444444444444444444444444444444444444444444444444444444444", + "metric_config_sha256": "5555555555555555555555555555555555555555555555555555555555555555", + "run_config_sha256": "6666666666666666666666666666666666666666666666666666666666666666", + "optimizer_config_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "seed": 42, + "started_at": "2026-07-28T00:00:00.000Z", + "mode": "optimize", + "candidate_source": "agent_optimizer", + "scenario": null, + "gate_config": { + "min_val_lift": 0.02, + "slice_tolerance": 0.05, + "budget_tokens": 100000, + "budget_usd": null, + "epsilon": 0.001 + }, + "model_info": { + "provider": "test-stub", + "model_name": "injected-call-agent" + }, + "evaluator_info": { + "name": "AgentEvaluator", + "metric_config_sha256": "5555555555555555555555555555555555555555555555555555555555555555" + }, + "optimizer_info": { + "algorithm": "gepa_reflective", + "status": "SUCCEEDED", + "total_rounds": 1 + } + }, + "results": { + "train": { + "champion_avg": 0.0, + "challenger_avg": 1.0, + "delta": 1.0 + }, + "val": { + "champion_avg": 0.333333, + "challenger_avg": 1.0, + "delta": 0.666667 + } + }, + "train_delta": 1.0, + "val_delta": 0.666667, + "per_case": [ + { + "eval_id": "val_add_steps_b", + "split": "val", + "slice": "math", + "risk_level": "low", + "protected": false, + "scenario_tag": "add_steps", + "champion_status": "FAILED", + "challenger_status": "PASSED", + "champion_score": 0.0, + "challenger_score": 1.0, + "delta": 1.0, + "transition": "newly_passed", + "failure_kind": "none", + "category": "none", + "failure_reason": "case 已通过，无失败归因。", + "runs": 1, + "evidence": { + "actual_text": "步骤：4 × 5 = 20\n答案：20 个", + "expected_text": "步骤：4 × 5 = 20\n答案：20 个", + "metric_results": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "eval_status": "PASSED", + "reason": null, + "rubric_scores": [] + } + ], + "actual_tool_uses": [], + "expected_tool_uses": [], + "actual_tool_responses": [], + "expected_tool_responses": [], + "error_message": null, + "metric_reasons": [], + "trace_ref": "val_eval.json#eval_id=val_add_steps_b", + "attribution": {} + } + }, + { + "eval_id": "evidence_tool_missing", + "split": "val", + "slice": "tool", + "risk_level": "low", + "protected": false, + "scenario_tag": null, + "champion_status": "FAILED", + "challenger_status": "FAILED", + "champion_score": 0.0, + "challenger_score": 0.0, + "delta": 0.0, + "transition": "unchanged", + "failure_kind": "agent_quality_failure", + "category": "tool_call_error", + "failure_reason": "缺少期望工具调用，或调用了错误的工具。", + "runs": 1, + "evidence": { + "actual_text": "已查询价格。", + "expected_text": "已查询价格。", + "metric_results": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": "expected tool call was not found", + "rubric_scores": [] + } + ], + "actual_tool_uses": [], + "expected_tool_uses": [ + { + "id": "t1", + "name": "search", + "args": { + "query": "pricing" + } + } + ], + "actual_tool_responses": [], + "expected_tool_responses": [], + "error_message": null, + "metric_reasons": [ + "expected tool call was not found" + ], + "trace_ref": "val_eval.json#eval_id=evidence_tool_missing", + "attribution": { + "expected_tool_uses": [ + { + "id": "t1", + "name": "search", + "args": { + "query": "pricing" + } + } + ], + "actual_tool_uses": [] + } + } + }, + { + "eval_id": "evidence_param_wrong", + "split": "val", + "slice": "tool", + "risk_level": "low", + "protected": false, + "scenario_tag": null, + "champion_status": "FAILED", + "challenger_status": "FAILED", + "champion_score": 0.0, + "challenger_score": 0.0, + "delta": 0.0, + "transition": "unchanged", + "failure_kind": "agent_quality_failure", + "category": "param_error", + "failure_reason": "工具名称匹配，但调用参数与期望轨迹不一致。", + "runs": 1, + "evidence": { + "actual_text": "已查询。", + "expected_text": "已查询。", + "metric_results": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": "tool arguments did not match", + "rubric_scores": [] + } + ], + "actual_tool_uses": [ + { + "id": "a1", + "name": "search", + "args": { + "query": "weather" + } + } + ], + "expected_tool_uses": [ + { + "id": "t1", + "name": "search", + "args": { + "query": "pricing" + } + } + ], + "actual_tool_responses": [], + "expected_tool_responses": [], + "error_message": null, + "metric_reasons": [ + "tool arguments did not match" + ], + "trace_ref": "val_eval.json#eval_id=evidence_param_wrong", + "attribution": { + "parameter_differences": [ + { + "tool_name": "search", + "expected_args": { + "query": "pricing" + }, + "actual_args": { + "query": "weather" + } + } + ] + } + } + }, + { + "eval_id": "evidence_rubric_failed", + "split": "val", + "slice": "quality", + "risk_level": "low", + "protected": false, + "scenario_tag": null, + "champion_status": "FAILED", + "challenger_status": "FAILED", + "champion_score": 0.0, + "challenger_score": 0.0, + "delta": 0.0, + "transition": "unchanged", + "failure_kind": "agent_quality_failure", + "category": "rubric_fail", + "failure_reason": "LLM rubric 指标未达到阈值。", + "runs": 1, + "evidence": { + "actual_text": "答案是 20。", + "expected_text": "清晰解释计算过程并给出答案。", + "metric_results": [ + { + "metric_name": "llm_rubric_response", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": "rubric reasoning_clear failed", + "rubric_scores": [ + { + "id": "reasoning_clear", + "score": 0.0, + "reason": "missing explanation" + } + ] + } + ], + "actual_tool_uses": [], + "expected_tool_uses": [], + "actual_tool_responses": [], + "expected_tool_responses": [], + "error_message": null, + "metric_reasons": [ + "rubric reasoning_clear failed" + ], + "trace_ref": "val_eval.json#eval_id=evidence_rubric_failed", + "attribution": { + "failed_metrics": [ + "llm_rubric_response" + ], + "metric_reasons": [ + "rubric reasoning_clear failed" + ] + } + } + } + ], + "transition_counts": { + "newly_passed": 1, + "unchanged": 3 + }, + "fail_category_counts": { + "none": 1, + "tool_call_error": 1, + "param_error": 1, + "rubric_fail": 1 + }, + "decision": { + "accepted": true, + "violated": [], + "reasons": [] + }, + "cost_status": "measured", + "cost": { + "status": "measured", + "total_tokens": 123, + "total_usd": 0.0123 + }, + "optimizer": { + "info": { + "algorithm": "gepa_reflective", + "status": "SUCCEEDED", + "total_rounds": 1 + }, + "rounds": [ + { + "round": 1, + "candidate_prompts": { + "system": "Example candidate prompt" + }, + "accepted": true, + "validation_pass_rate": 1.0, + "artifact_path": "runs//optimizer/rounds/round_001.json" + } + ], + "artifacts": { + "optimizer_result": "runs//optimizer/result.json", + "optimizer_best_prompt": "runs//optimizer/best_prompts/system.md" + } + }, + "audit": { + "run_id": "2026-07-28T00-00-00.000000Z-a1b2c3d4", + "applied": false, + "duration_seconds": 12.3456, + "candidate_source": "agent_optimizer", + "scenario": null, + "cost_status": "measured", + "cost": 0.0123, + "artifact_dir": "runs/", + "artifacts": { + "frozen_json": "runs//frozen.json", + "train_eval_log": "runs//train_eval.json", + "val_eval_log": "runs//val_eval.json" + }, + "before_apply_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "after_apply_sha256": null, + "repro_cmd": "python pipeline.py --mode optimize" + }, + "limitations": { + "hidden_sample_accuracy": "未使用官方隐藏集；公开 fixture 只用于可复现测试。" + } +} diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 000000000..3470e1820 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,46 @@ +{ + "_comment": "--mode optimize 使用：真实调用 AgentOptimizer.optimize(update_source=false)。reflection_lm 变量由 TRPC_AGENT_* 环境提供；fake 回归不会读取这些凭据。", + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "contains", + "case_insensitive": true + } + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 2, + "stop": { + "required_metrics": ["final_response_avg_score"] + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "provider_name": "${TRPC_AGENT_PROVIDER_NAME:openai}", + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "max_tokens": 2048, + "temperature": 0.4 + } + }, + "candidate_selection_strategy": "pareto", + "module_selector": "round_robin", + "reflection_minibatch_size": 3, + "skip_perfect_score": false, + "max_metric_calls": 30, + "max_iterations_without_improvement": 4 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py new file mode 100644 index 000000000..1b990f1df --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -0,0 +1,548 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Evaluation → attribution → optimization → regression → audit pipeline.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[2] +for _path in (str(_REPO_ROOT), str(_HERE)): + if _path not in sys.path: + sys.path.insert(0, _path) + + +import fake_agent # type: ignore[unresolved-import] +import gates # type: ignore[unresolved-import] +import live_agent # type: ignore[unresolved-import] +import report # type: ignore[unresolved-import] +import runner # type: ignore[unresolved-import] + +DEFAULT_RUN_JSON: dict[str, Any] = { + "champion_prompt": "prompts/system.md", + "train_evalset": "data/train.evalset.json", + "val_evalset": "data/val.evalset.json", + "metric_config": "data/test_config.json", + "gate": { + "min_val_lift": 0.02, + "slice_tolerance": 0.05, + "budget_tokens": 100_000, + "budget_usd": None, + "epsilon": 0.001, + }, + "seed": 42, +} + + +@dataclass +class OptimizerCandidate: + prompt: str + info: dict[str, Any] + rounds: list[dict[str, Any]] + artifacts: dict[str, str] + cost_status: str + total_tokens: Optional[int] + total_cost: Optional[float] + + +def _load_config(config_path: Optional[Path]) -> dict[str, Any]: + if config_path is None: + return json.loads(json.dumps(DEFAULT_RUN_JSON)) + if not config_path.exists(): + raise FileNotFoundError(f"run config 不存在：{config_path}") + config = json.loads(config_path.read_text(encoding="utf-8")) + merged = json.loads(json.dumps(DEFAULT_RUN_JSON)) + merged.update(config) + merged_gate = dict(DEFAULT_RUN_JSON["gate"]) + merged_gate.update(config.get("gate", {})) + merged["gate"] = merged_gate + return merged + + +def _resolve(base: Path, value: str) -> Path: + path = Path(value) + return path if path.is_absolute() else (base / path).resolve() + + +def _gate_config(config: dict[str, Any]) -> gates.GateConfig: + return gates.GateConfig( + min_val_lift=float(config.get("min_val_lift", 0.02)), + slice_tolerance=float(config.get("slice_tolerance", 0.05)), + budget_tokens=int(config.get("budget_tokens", 100_000)), + budget_usd=(float(config["budget_usd"]) if config.get("budget_usd") is not None else None), + epsilon=float(config.get("epsilon", 0.001)), + ) + + +def _build_repro_cmd(args: argparse.Namespace) -> str: + argv = [sys.executable, str(_HERE / "pipeline.py")] + for key, value in vars(args).items(): + if value is None or value is False: + continue + flag = f"--{key.replace('_', '-')}" + argv.append(flag if value is True else f"{flag}={value}") + return " ".join(argv) + + +def _content_text(content: dict[str, Any]) -> str: + return "\n".join(str(part.get("text", "")) for part in (content.get("parts") or []) if part.get("text")) + + +def _case_contexts(*paths: Path) -> dict[str, list[dict[str, Any]]]: + """Index evaluation metadata by query without treating query as eval_id. + + The public optimizer callback receives only ``query``. A query can legally + appear in more than one eval case, so the audit entry preserves every + matching case context instead of inventing an eval id from the query. + """ + + contexts: dict[str, list[dict[str, Any]]] = {} + for path in paths: + evalset = json.loads(path.read_text(encoding="utf-8")) + for case in evalset.get("eval_cases", []): + conversation = case.get("conversation") or [] + if not conversation: + continue + invocation = conversation[0] + query = _content_text(invocation.get("user_content") or {}) + if not query: + continue + contexts.setdefault(query, []).append( + { + "eval_id": case["eval_id"], + "split": ((case.get("session_input") or {}).get("state") or {}).get("split"), + "expected_response": _content_text(invocation.get("final_response") or {}), + "expected_tool_context": ((invocation.get("intermediate_data") or {}).get("tool_uses") or []), + } + ) + return contexts + + +def _audited_call_agent( + base_call_agent, + contexts: dict[str, list[dict[str, Any]]], + audit: list[dict[str, Any]], +): + async def call_agent(query: str) -> str: + matching_contexts = contexts.get(query) + if not matching_contexts: + raise KeyError(f"call_agent 收到未登记的评测 query：{query!r}") + try: + response = await base_call_agent(query) + except BaseException as error: + audit.append( + { + "query": query, + "eval_contexts": matching_contexts, + "status": "error", + "error": f"{type(error).__name__}: {error}", + } + ) + raise + audit.append( + { + "query": query, + "eval_contexts": matching_contexts, + "status": "ok", + "actual_response": response, + } + ) + return response + + # A plain callback does not prove what its model calls cost. Propagate + # explicit accounting metadata only when the provider/stub supplies it. + call_agent.cost_status = getattr(base_call_agent, "cost_status", "unavailable") + call_agent.total_tokens = getattr(base_call_agent, "total_tokens", None) + call_agent.total_cost = getattr(base_call_agent, "total_cost", None) + return call_agent + + +def _serialize_model(value: Any) -> dict[str, Any]: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json", by_alias=True) + if isinstance(value, dict): + return value + return dict(vars(value)) + + +def _enum_value(value: Any) -> Any: + return getattr(value, "value", value) + + +def _optimizer_cost(result: Any, call_agent) -> tuple[str, Optional[int], Optional[float]]: + if getattr(call_agent, "cost_status", "unavailable") != "measured": + return "unavailable", None, None + agent_tokens = getattr(call_agent, "total_tokens", None) + agent_cost = getattr(call_agent, "total_cost", None) + if agent_tokens is None or agent_cost is None: + return "unavailable", None, None + + usage = getattr(result, "total_token_usage", None) or {} + optimizer_tokens = usage.get("total") if isinstance(usage, dict) else None + optimizer_cost = getattr(result, "total_llm_cost", None) + if optimizer_tokens is None or optimizer_cost is None: + return "unavailable", None, None + if int(optimizer_tokens) <= 0 and int(getattr(result, "total_reflection_lm_calls", 0) or 0) > 0: + return "unavailable", None, None + return ( + "measured", + int(optimizer_tokens) + int(agent_tokens), + float(optimizer_cost) + float(agent_cost), + ) + + +async def _run_optimize_for_candidate( + *, + optimizer_config_path: Path, + champion_prompt_path: Path, + train_evalset_path: Path, + val_evalset_path: Path, + output_dir: Path, + call_agent, +) -> OptimizerCandidate: + """Invoke the native optimizer and return its best prompt plus audit metadata.""" + + from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt + + target = TargetPrompt().add_path("system", str(champion_prompt_path)) + result = await AgentOptimizer.optimize( + config_path=str(optimizer_config_path), + call_agent=call_agent, + target_prompt=target, + train_dataset_path=str(train_evalset_path), + validation_dataset_path=str(val_evalset_path), + output_dir=str(output_dir), + update_source=False, + verbose=0, + ) + status = _enum_value(result.status) + if status != "SUCCEEDED" or not result.best_prompts: + raise RuntimeError( + f"AgentOptimizer 未产生 Candidate " + f"(status={status}, finish_reason={_enum_value(result.finish_reason)}, " + f"error={getattr(result, 'error_message', '')})" + ) + prompt_name = "system" if "system" in result.best_prompts else next(iter(result.best_prompts)) + rounds: list[dict[str, Any]] = [] + for index, item in enumerate(result.rounds or [], start=1): + round_record = _serialize_model(item) + round_number = int(round_record.get("round", round_record.get("round_index", index))) + round_record["artifact_path"] = str(output_dir / "rounds" / f"round_{round_number:03d}.json") + rounds.append(round_record) + info = { + "algorithm": result.algorithm, + "status": status, + "finish_reason": _enum_value(result.finish_reason), + "stop_reason": _enum_value(getattr(result, "stop_reason", None)), + "total_rounds": result.total_rounds, + "baseline_pass_rate": result.baseline_pass_rate, + "best_pass_rate": result.best_pass_rate, + "pass_rate_improvement": result.pass_rate_improvement, + } + cost_status, total_tokens, total_cost = _optimizer_cost(result, call_agent) + artifacts = { + "optimizer_dir": str(output_dir), + "optimizer_result": str(output_dir / "result.json"), + "optimizer_summary": str(output_dir / "summary.txt"), + "optimizer_rounds": str(output_dir / "rounds"), + "optimizer_config_snapshot": str(output_dir / "config.snapshot.json"), + "optimizer_run_log": str(output_dir / "run.log"), + "optimizer_baseline_prompt": str(output_dir / "baseline_prompts" / f"{prompt_name}.md"), + "optimizer_best_prompt": str(output_dir / "best_prompts" / f"{prompt_name}.md"), + } + return OptimizerCandidate( + prompt=result.best_prompts[prompt_name], + info=info, + rounds=rounds, + artifacts=artifacts, + cost_status=cost_status, + total_tokens=total_tokens, + total_cost=total_cost, + ) + + +def _failure_frozen( + *, + run_id: str, + champion_prompt_path: Path, + train_evalset_path: Path, + val_evalset_path: Path, + metric_config_path: Path, + optimizer_config_path: Path, + config: dict[str, Any], + model_info: dict[str, Any], + error: BaseException, +) -> runner.FrozenInputs: + champion = champion_prompt_path.read_text(encoding="utf-8") + started_at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + return runner.FrozenInputs( + run_id=run_id, + champion_sha256=runner.sha256_text(champion), + challenger_sha256="", + train_sha256=runner.sha256_file(train_evalset_path), + val_sha256=runner.sha256_file(val_evalset_path), + metric_config_sha256=runner.sha256_file(metric_config_path), + run_config_sha256=runner.sha256_json(config), + optimizer_config_sha256=(runner.sha256_file(optimizer_config_path) if optimizer_config_path.exists() else None), + seed=int(config.get("seed", 42)), + started_at=started_at, + mode="optimize", + candidate_source="agent_optimizer", + gate_config=config["gate"], + model_info=model_info, + evaluator_info={ + "name": "AgentEvaluator", + "metric_config_sha256": runner.sha256_file(metric_config_path), + }, + optimizer_info={ + "status": "FAILED", + "error_type": type(error).__name__, + "error": str(error), + }, + ) + + +async def _amain(args: argparse.Namespace, *, call_agent=None) -> int: + config_path = _resolve(_HERE, args.config) if args.config else None + config = _load_config(config_path) + champion_path = _resolve(_HERE, config["champion_prompt"]) + train_path = _resolve(_HERE, config["train_evalset"]) + val_path = _resolve(_HERE, config["val_evalset"]) + metric_path = _resolve(_HERE, config["metric_config"]) + optimizer_path = _resolve(_HERE, args.optimizer_config or "optimizer.json") + seed = int(config.get("seed", 42)) + gate_config = _gate_config(config["gate"]) + run_id = runner.new_run_id() + run_dir = _HERE / "runs" / run_id + repro_cmd = _build_repro_cmd(args) + model_info: dict[str, Any] = {"provider": "none", "model_name": "fake-trace"} + optimizer: Optional[OptimizerCandidate] = None + audited_calls: list[dict[str, Any]] = [] + + if args.mode == "optimize": + run_dir.mkdir(parents=True, exist_ok=False) + optimizer_dir = run_dir / "optimizer" + optimizer_dir.mkdir() + try: + base_call_agent = call_agent + if base_call_agent is None: + model_info = live_agent.model_info_from_env() + base_call_agent = live_agent.build_call_agent(champion_path) + else: + model_info = {"provider": "test-stub", "model_name": "injected-call-agent"} + contexts = _case_contexts(train_path, val_path) + contextual_call_agent = _audited_call_agent(base_call_agent, contexts, audited_calls) + optimizer = await _run_optimize_for_candidate( + optimizer_config_path=optimizer_path, + champion_prompt_path=champion_path, + train_evalset_path=train_path, + val_evalset_path=val_path, + output_dir=optimizer_dir, + call_agent=contextual_call_agent, + ) + (run_dir / "call_agent_audit.json").write_text( + json.dumps(audited_calls, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + optimizer.artifacts["call_agent_audit"] = str(run_dir / "call_agent_audit.json") + challenger_text = optimizer.prompt + candidate_source = "agent_optimizer" + scenario = None + regression_call_agent = contextual_call_agent + except BaseException as error: + audit_path = run_dir / "call_agent_audit.json" + audit_path.write_text( + json.dumps(audited_calls, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + error_path = run_dir / "optimizer_error.json" + error_path.write_text( + json.dumps( + { + "error_type": type(error).__name__, + "error": str(error), + "call_agent_audit": audited_calls, + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + frozen = _failure_frozen( + run_id=run_id, + champion_prompt_path=champion_path, + train_evalset_path=train_path, + val_evalset_path=val_path, + metric_config_path=metric_path, + optimizer_config_path=optimizer_path, + config=config, + model_info=model_info, + error=error, + ) + (run_dir / "frozen.json").write_text( + json.dumps(asdict(frozen), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + optimizer_artifacts = { + "optimizer_error": str(error_path), + "call_agent_audit": str(audit_path), + "optimizer_dir": str(optimizer_dir), + } + for name in ("result.json", "summary.txt", "config.snapshot.json", "run.log"): + candidate_path = optimizer_dir / name + if candidate_path.exists(): + optimizer_artifacts[f"optimizer_{candidate_path.stem}"] = str(candidate_path) + failure_report = report.build_optimizer_failure_report( + frozen=frozen, + artifact_dir=run_dir, + error=error, + repro_cmd=repro_cmd, + optimizer_artifacts=optimizer_artifacts, + ) + report.write_report(failure_report, out_dir=run_dir) + report.write_report(failure_report, out_dir=_HERE) + print(f"决策: REJECT（优化器失败：{error}）", file=sys.stderr) + print(f"报告: {_HERE / 'optimization_report.md'}") + print(f"Artifacts: {run_dir}") + return 2 if args.apply else 1 + elif args.candidate_file: + challenger_text = _resolve(_HERE, args.candidate_file).read_text(encoding="utf-8") + candidate_source = "candidate_file" + scenario = None + regression_call_agent = None + elif args.scenario: + challenger_text = fake_agent.build_candidate(args.scenario) + candidate_source = "candidate_file" + scenario = args.scenario + regression_call_agent = None + else: + print("fake 模式必须指定 --scenario 或 --candidate-file。", file=sys.stderr) + return 2 + + before_sha = runner.sha256_file(champion_path) + artifact = await runner.run_pair( + champion_prompt_path=champion_path, + challenger_text=challenger_text, + train_evalset_path=train_path, + val_evalset_path=val_path, + metric_config_path=metric_path, + artifact_root=_HERE / "runs", + artifact_dir=run_dir, + mode=args.mode, + candidate_source=candidate_source, + scenario=scenario, + seed=seed, + call_agent=regression_call_agent, + run_config=config, + gate_config=config["gate"], + optimizer_config_path=optimizer_path if args.mode == "optimize" else None, + model_info=model_info, + optimizer_info=optimizer.info if optimizer else {}, + cost_status=optimizer.cost_status if optimizer else "measured", + total_tokens=optimizer.total_tokens if optimizer else 0, + total_cost=optimizer.total_cost if optimizer else 0.0, + optimizer_artifacts=optimizer.artifacts if optimizer else {}, + optimizer_rounds=optimizer.rounds if optimizer else [], + ) + if runner.sha256_file(champion_path) != before_sha: + raise RuntimeError("Champion prompt 在回归评测后未恢复。") + + case_deltas = [ + gates.CaseDelta( + eval_id=case.eval_id, + split=case.split, + slice_name=case.slice_name, + risk_level=case.risk_level, + protected=case.protected, + champion_status=case.champion_status, + challenger_status=case.challenger_status, + champion_score=case.champion_score, + challenger_score=case.challenger_score, + ) + for case in artifact.cases + ] + decision = gates.evaluate( + case_deltas, + cost_status=artifact.cost_status, + total_tokens=artifact.total_tokens, + total_cost=artifact.total_cost, + config=gate_config, + ) + + applied = False + after_apply_sha: Optional[str] = None + if args.apply and decision.accepted: + from trpc_agent_sdk.evaluation import TargetPrompt + + target = TargetPrompt().add_path("system", str(champion_path)) + snapshot = await target.read_all() + try: + await target.write_all({"system": challenger_text}) + after_apply_sha = runner.sha256_file(champion_path) + applied = True + except BaseException: + await target.write_all(snapshot) + raise + elif args.apply: + print("Gate REJECT，--apply 已被拒绝；Champion 未修改。", file=sys.stderr) + + report_dict = report.build_report_dict( + artifact, + decision, + applied=applied, + before_apply_sha256=before_sha, + after_apply_sha256=after_apply_sha, + repro_cmd=repro_cmd, + ) + report.write_report(report_dict, out_dir=artifact.artifact_dir) + report.write_report(report_dict, out_dir=_HERE) + if args.mode == "optimize": + (run_dir / "call_agent_audit.json").write_text( + json.dumps(audited_calls, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + print(f"决策: {'ACCEPT' if decision.accepted else 'REJECT'}") + if decision.violated: + print(f"违反 gate: {', '.join(decision.violated)}") + print(f"报告: {_HERE / 'optimization_report.md'}") + print(f"Artifacts: {artifact.artifact_dir}") + if args.apply and not decision.accepted: + return 2 + return 0 if decision.accepted else 1 + + +def _parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Issue #91 Evaluation + Optimization pipeline") + parser.add_argument("--config", default=None) + parser.add_argument("--mode", choices=("fake", "optimize"), default="fake") + parser.add_argument("--scenario", choices=("success", "no_effect", "overfit")) + parser.add_argument("--candidate-file") + parser.add_argument("--optimizer-config") + parser.add_argument("--apply", action="store_true") + return parser.parse_args(argv) + + +def main(argv: Optional[list[str]] = None) -> int: + return asyncio.run(_amain(_parse_args(argv))) + + +async def amain(argv: Optional[list[str]] = None, *, call_agent=None) -> int: + return await _amain(_parse_args(argv), call_agent=call_agent) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/optimization/eval_optimize_loop/prompts/system.md b/examples/optimization/eval_optimize_loop/prompts/system.md new file mode 100644 index 000000000..d575b0cdc --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/system.md @@ -0,0 +1,16 @@ +# System Prompt (Champion) + +你是一个数学问答助手。请用中文回答用户的算术问题。 + +## 输出格式 + +请直接给出最终数字答案，无需展示步骤。 + +## 工具使用 + +当前未配置工具。 + + diff --git a/examples/optimization/eval_optimize_loop/report.py b/examples/optimization/eval_optimize_loop/report.py new file mode 100644 index 000000000..3d26b44ec --- /dev/null +++ b/examples/optimization/eval_optimize_loop/report.py @@ -0,0 +1,291 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Build machine-readable and human-readable optimization reports.""" + +from __future__ import annotations + +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any, Optional + +import attribution +import gates +from runner import FrozenInputs, RunArtifact, SplitResult + + +def _split_to_dict(split: SplitResult) -> dict[str, float]: + return { + "champion_avg": round(split.champion_avg, 6), + "challenger_avg": round(split.challenger_avg, 6), + "delta": round(split.delta, 6), + } + + +def build_report_dict( + artifact: RunArtifact, + decision: gates.Decision, + *, + applied: bool, + before_apply_sha256: str, + after_apply_sha256: Optional[str], + repro_cmd: str, +) -> dict[str, Any]: + """Build the complete report from real case-level evaluator evidence.""" + + per_case: list[dict[str, Any]] = [] + category_counts: dict[str, int] = {} + transition_counts: dict[str, int] = {} + for case in artifact.cases: + result = attribution.from_case_record(case) + category_counts[result.category] = category_counts.get(result.category, 0) + 1 + transition_counts[case.transition] = transition_counts.get(case.transition, 0) + 1 + per_case.append( + { + "eval_id": case.eval_id, + "split": case.split, + "slice": case.slice_name, + "risk_level": case.risk_level, + "protected": case.protected, + "scenario_tag": case.scenario_tag, + "champion_status": case.champion_status, + "challenger_status": case.challenger_status, + "champion_score": round(case.champion_score, 6), + "challenger_score": round(case.challenger_score, 6), + "delta": round(case.delta, 6), + "transition": case.transition, + "failure_kind": case.failure_kind, + "category": result.category, + "failure_reason": result.reason, + "runs": 1, + "evidence": { + "actual_text": case.actual_text, + "expected_text": case.expected_text, + "metric_results": case.metric_results, + "actual_tool_uses": case.actual_tool_uses, + "expected_tool_uses": case.expected_tool_uses, + "actual_tool_responses": case.actual_tool_responses, + "expected_tool_responses": case.expected_tool_responses, + "error_message": case.error_message, + "metric_reasons": case.failure_reasons, + "trace_ref": case.trace_ref, + "attribution": result.evidence, + }, + } + ) + + frozen = asdict(artifact.frozen) + return { + "version": "2.0", + "candidate_source": artifact.frozen.candidate_source, + "frozen": frozen, + "results": { + "train": _split_to_dict(artifact.train), + "val": _split_to_dict(artifact.val), + }, + "train_delta": round(artifact.train.delta, 6), + "val_delta": round(artifact.val.delta, 6), + "per_case": per_case, + "transition_counts": transition_counts, + "fail_category_counts": category_counts, + "decision": { + "accepted": decision.accepted, + "violated": decision.violated, + "reasons": decision.reasons, + }, + "cost_status": artifact.cost_status, + "cost": { + "status": artifact.cost_status, + "total_tokens": artifact.total_tokens, + "total_usd": artifact.total_cost, + }, + "optimizer": { + "info": artifact.frozen.optimizer_info, + "rounds": artifact.optimizer_rounds, + "artifacts": artifact.optimizer_artifacts, + }, + "audit": { + "run_id": artifact.frozen.run_id, + "applied": applied, + "duration_seconds": round(artifact.duration_seconds, 4), + "candidate_source": artifact.frozen.candidate_source, + "scenario": artifact.frozen.scenario, + "cost_status": artifact.cost_status, + "cost": artifact.total_cost, + "artifact_dir": str(artifact.artifact_dir), + "artifacts": { + "frozen_json": str(artifact.artifact_dir / "frozen.json"), + "champion_prompts": str(artifact.artifact_dir / "champion_prompts" / "system.md"), + "challenger_prompts": str(artifact.artifact_dir / "challenger_prompts" / "system.md"), + "train_eval_log": str(artifact.artifact_dir / "train_eval.json"), + "val_eval_log": str(artifact.artifact_dir / "val_eval.json"), + **artifact.optimizer_artifacts, + }, + "before_apply_sha256": before_apply_sha256, + "after_apply_sha256": after_apply_sha256, + "repro_cmd": repro_cmd, + }, + "limitations": { + "hidden_sample_accuracy": ( + "未使用官方隐藏集，不声称满足隐藏样本准确率；" "公开 fake/标注 fixture 仅证明可复现管线行为。" + ) + }, + } + + +def build_optimizer_failure_report( + *, + frozen: FrozenInputs, + artifact_dir: Path, + error: BaseException, + repro_cmd: str, + optimizer_artifacts: dict[str, str], +) -> dict[str, Any]: + """Persist an auditable REJECT when optimization cannot produce a candidate.""" + + reason = f"{type(error).__name__}: {error}" + return { + "version": "2.0", + "candidate_source": "agent_optimizer", + "frozen": asdict(frozen), + "results": {"train": None, "val": None}, + "train_delta": None, + "val_delta": None, + "per_case": [], + "transition_counts": {}, + "fail_category_counts": {"infrastructure_failure": 1}, + "decision": { + "accepted": False, + "violated": ["OPTIMIZER_FAILURE", "G6"], + "reasons": [ + f"优化器未产生可验证 Candidate：{reason}", + "成本或优化证据不完整，禁止自动 ACCEPT/--apply。", + ], + }, + "cost_status": "unavailable", + "cost": {"status": "unavailable", "total_tokens": None, "total_usd": None}, + "optimizer": { + "info": frozen.optimizer_info, + "rounds": [], + "artifacts": optimizer_artifacts, + "error": reason, + }, + "audit": { + "run_id": frozen.run_id, + "applied": False, + "duration_seconds": 0.0, + "candidate_source": "agent_optimizer", + "scenario": None, + "cost_status": "unavailable", + "cost": None, + "artifact_dir": str(artifact_dir), + "artifacts": { + "frozen_json": str(artifact_dir / "frozen.json"), + **optimizer_artifacts, + }, + "before_apply_sha256": frozen.champion_sha256, + "after_apply_sha256": None, + "repro_cmd": repro_cmd, + }, + "limitations": {"hidden_sample_accuracy": "本次优化失败，未产生可用于隐藏样本验证的 Candidate。"}, + } + + +def render_markdown(report: dict[str, Any]) -> str: + decision = report["decision"] + verdict = "ACCEPT" if decision["accepted"] else "REJECT" + lines = [ + "# Evaluation + Optimization 报告", + "", + f"- **决策**: `{verdict}`", + f"- **运行 ID**: `{report['audit']['run_id']}`", + f"- **候选来源**: `{report['candidate_source']}`", + f"- **运行模式**: `{report['frozen']['mode']}`", + f"- **是否写回**: `{report['audit']['applied']}`", + "", + "## 聚合分数", + "", + ] + if report["results"]["train"] is None: + lines.append("优化器未产生 Candidate，未执行 Champion/Challenger 回归比较。") + else: + lines.extend( + [ + "| split | baseline | candidate | delta |", + "|---|---:|---:|---:|", + ] + ) + for split in ("train", "val"): + values = report["results"][split] + lines.append( + f"| {split} | {values['champion_avg']:.4f} | " + f"{values['challenger_avg']:.4f} | {values['delta']:+.4f} |" + ) + + lines.extend(["", "## Gate 决策", ""]) + if decision["violated"]: + lines.append(f"违反的 gate：`{', '.join(decision['violated'])}`") + else: + lines.append("全部 gate 通过。") + lines.extend(f"- {reason}" for reason in decision["reasons"]) + + lines.extend(["", "## 失败归因统计", "", "| category | count |", "|---|---:|"]) + for category, count in sorted(report["fail_category_counts"].items()): + lines.append(f"| {category} | {count} |") + + lines.extend( + [ + "", + "## 逐 case 明细", + "", + "| eval_id | split | transition | baseline | candidate | delta | category | reason |", + "|---|---|---|---:|---:|---:|---|---|", + ] + ) + for case in report["per_case"]: + reason = str(case["failure_reason"]).replace("|", "\\|").replace("\n", " ") + lines.append( + f"| {case['eval_id']} | {case['split']} | {case['transition']} | " + f"{case['champion_score']:.2f} | {case['challenger_score']:.2f} | " + f"{case['delta']:+.2f} | {case['category']} | {reason} |" + ) + + cost = report["cost"] + lines.extend( + [ + "", + "## 成本与优化器产物", + "", + f"- cost_status: `{cost['status']}`", + f"- total_tokens: `{cost['total_tokens']}`", + f"- total_usd: `{cost['total_usd']}`", + f"- optimizer rounds: `{len(report['optimizer']['rounds'])}`", + "", + "## 审计", + "", + f"- artifact_dir: `{report['audit']['artifact_dir']}`", + f"- before_apply_sha256: `{report['audit']['before_apply_sha256']}`", + f"- after_apply_sha256: `{report['audit']['after_apply_sha256']}`", + f"- repro_cmd: `{report['audit']['repro_cmd']}`", + "", + "## 限制", + "", + f"- {report['limitations']['hidden_sample_accuracy']}", + "", + ] + ) + return "\n".join(lines) + + +def write_report(report: dict[str, Any], *, out_dir: Path) -> tuple[Path, Path]: + out_dir.mkdir(parents=True, exist_ok=True) + json_path = out_dir / "optimization_report.json" + markdown_path = out_dir / "optimization_report.md" + json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + markdown_path.write_text(render_markdown(report), encoding="utf-8") + return json_path, markdown_path diff --git a/examples/optimization/eval_optimize_loop/run.json b/examples/optimization/eval_optimize_loop/run.json new file mode 100644 index 000000000..815f73026 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run.json @@ -0,0 +1,15 @@ +{ + "_comment": "eval_optimize_loop 运行配置。所有路径相对本文件所在目录；gate 阈值在此调整。", + "champion_prompt": "prompts/system.md", + "train_evalset": "data/train.evalset.json", + "val_evalset": "data/val.evalset.json", + "metric_config": "data/test_config.json", + "gate": { + "min_val_lift": 0.02, + "slice_tolerance": 0.05, + "budget_tokens": 100000, + "budget_usd": null, + "epsilon": 0.001 + }, + "seed": 42 +} diff --git a/examples/optimization/eval_optimize_loop/runner.py b/examples/optimization/eval_optimize_loop/runner.py new file mode 100644 index 000000000..6a1ff0019 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/runner.py @@ -0,0 +1,612 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Run Champion and Challenger evaluations and persist auditable evidence.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import time +import uuid +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Awaitable, Callable, Optional + +from trpc_agent_sdk.evaluation import AgentEvaluator, EvalCaseResult, EvaluateResult, TargetPrompt + +CallAgent = Callable[[str], Awaitable[str]] + + +@dataclass +class FrozenInputs: + """Immutable inputs and environment facts for one experiment.""" + + run_id: str + champion_sha256: str + challenger_sha256: str + train_sha256: str + val_sha256: str + metric_config_sha256: str + run_config_sha256: str + optimizer_config_sha256: Optional[str] + seed: int + started_at: str + mode: str + candidate_source: str + scenario: Optional[str] = None + gate_config: dict[str, Any] = field(default_factory=dict) + model_info: dict[str, Any] = field(default_factory=dict) + evaluator_info: dict[str, Any] = field(default_factory=dict) + optimizer_info: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class SplitResult: + champion_avg: float + challenger_avg: float + + @property + def delta(self) -> float: + return self.challenger_avg - self.champion_avg + + +@dataclass +class CaseRecord: + """Case-level comparison plus evidence needed for attribution.""" + + eval_id: str + split: str + slice_name: str + risk_level: str + protected: bool + scenario_tag: Optional[str] + champion_status: str + challenger_status: str + champion_score: float + challenger_score: float + expected_text: Optional[str] + actual_text: Optional[str] + metric_results: list[dict[str, Any]] = field(default_factory=list) + actual_tool_uses: list[dict[str, Any]] = field(default_factory=list) + expected_tool_uses: list[dict[str, Any]] = field(default_factory=list) + actual_tool_responses: list[dict[str, Any]] = field(default_factory=list) + expected_tool_responses: list[dict[str, Any]] = field(default_factory=list) + error_message: Optional[str] = None + failure_reasons: list[str] = field(default_factory=list) + trace_ref: Optional[str] = None + + @property + def delta(self) -> float: + return self.challenger_score - self.champion_score + + @property + def transition(self) -> str: + if self.champion_status == "PASSED" and self.challenger_status != "PASSED": + return "newly_failed" + if self.champion_status != "PASSED" and self.challenger_status == "PASSED": + return "newly_passed" + if self.delta > 0: + return "score_up" + if self.delta < 0: + return "score_down" + return "unchanged" + + @property + def failure_kind(self) -> str: + if self.challenger_status == "PASSED": + return "none" + if self.error_message or self.challenger_status == "NOT_EVALUATED": + return "infrastructure_failure" + return "agent_quality_failure" + + +@dataclass +class RunArtifact: + frozen: FrozenInputs + train: SplitResult + val: SplitResult + cases: list[CaseRecord] + champion_train_avg: float + champion_val_avg: float + artifact_dir: Path + cost_status: str + total_tokens: Optional[int] + total_cost: Optional[float] + duration_seconds: float + champion_prompt_text: str + challenger_prompt_text: str + optimizer_artifacts: dict[str, str] = field(default_factory=dict) + optimizer_rounds: list[dict[str, Any]] = field(default_factory=list) + + +def new_run_id() -> str: + """Return a filesystem-safe, collision-resistant UTC run id.""" + + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%fZ") + return f"{timestamp}-{uuid.uuid4().hex[:8]}" + + +def sha256_file(path: Path) -> str: + """Hash UTF-8 text with normalized newlines; binary files hash as bytes.""" + + raw = path.read_bytes() + try: + text = raw.decode("utf-8").replace("\r\n", "\n").replace("\r", "\n") + raw = text.encode("utf-8") + except UnicodeDecodeError: + pass + return hashlib.sha256(raw).hexdigest() + + +def sha256_text(text: str) -> str: + normalized = text.replace("\r\n", "\n").replace("\r", "\n") + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def sha256_json(value: Any) -> str: + payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return sha256_text(payload) + + +def _load_evalset(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _case_state(case: dict[str, Any]) -> dict[str, Any]: + return (case.get("session_input") or {}).get("state") or {} + + +def _content_text(content: Any) -> Optional[str]: + if content is None: + return None + parts = getattr(content, "parts", None) or [] + texts = [str(getattr(part, "text", "")) for part in parts if getattr(part, "text", None)] + return "\n".join(texts) if texts else None + + +def _expected_text(case: dict[str, Any]) -> Optional[str]: + conversation = case.get("conversation") or [] + if not conversation: + return None + parts = (conversation[0].get("final_response") or {}).get("parts") or [] + texts = [str(part.get("text", "")) for part in parts if part.get("text")] + return "\n".join(texts) if texts else None + + +def _model_to_dict(value: Any) -> dict[str, Any]: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json", by_alias=True) + if isinstance(value, dict): + return value + return {key: getattr(value, key) for key in ("id", "name", "args", "response") if hasattr(value, key)} + + +def _intermediate_items(invocation: Any, field_name: str) -> list[dict[str, Any]]: + intermediate = getattr(invocation, "intermediate_data", None) + values = getattr(intermediate, field_name, None) or [] + return [_model_to_dict(value) for value in values] + + +def _metric_evidence(result: EvalCaseResult) -> tuple[list[dict[str, Any]], list[str]]: + metrics: list[dict[str, Any]] = [] + reasons: list[str] = [] + for metric in result.overall_eval_metric_results or []: + details = getattr(metric, "details", None) + reason = getattr(details, "reason", None) if details is not None else None + if reason: + reasons.append(str(reason)) + metrics.append( + { + "metric_name": metric.metric_name, + "score": metric.score, + "threshold": metric.threshold, + "eval_status": getattr(metric.eval_status, "name", str(metric.eval_status)), + "reason": reason, + "rubric_scores": [_model_to_dict(score) for score in (getattr(details, "rubric_scores", None) or [])], + } + ) + for per_invocation in result.eval_metric_result_per_invocation or []: + for metric in per_invocation.eval_metric_results or []: + details = getattr(metric, "details", None) + reason = getattr(details, "reason", None) if details is not None else None + if reason and str(reason) not in reasons: + reasons.append(str(reason)) + return metrics, reasons + + +def _status(result: EvalCaseResult) -> str: + return getattr(result.final_eval_status, "name", str(result.final_eval_status)) + + +def _score(result: EvalCaseResult) -> float: + metrics = list(result.overall_eval_metric_results or []) + if not metrics: + return 1.0 if _status(result) == "PASSED" else 0.0 + scores = [float(metric.score) if metric.score is not None else 0.0 for metric in metrics] + return sum(scores) / len(scores) if scores else 0.0 + + +def _worst_run(runs: list[EvalCaseResult]) -> Optional[EvalCaseResult]: + if not runs: + return None + return min(runs, key=lambda result: (0 if _status(result) == "NOT_EVALUATED" else 1, _score(result))) + + +def _flatten_results(result: EvaluateResult) -> dict[str, list[EvalCaseResult]]: + flattened: dict[str, list[EvalCaseResult]] = {} + for aggregate in (result.results_by_eval_set_id or {}).values(): + for eval_id, runs in (aggregate.eval_results_by_eval_id or {}).items(): + flattened[eval_id] = list(runs or []) + return flattened + + +async def _run_evaluator( + evalset_path: Path, + *, + call_agent: Optional[CallAgent] = None, + metric_config_path: Optional[Path] = None, +) -> EvaluateResult: + """Run the public evaluator API while avoiding the Windows drive-colon parser.""" + + previous_cwd = Path.cwd() + try: + os.chdir(evalset_path.parent) + metric_arg: Optional[str] = None + if metric_config_path is not None: + if metric_config_path.parent.resolve() != evalset_path.parent.resolve(): + local_config = evalset_path.parent / metric_config_path.name + if not local_config.exists(): + shutil.copyfile(metric_config_path, local_config) + metric_arg = metric_config_path.name + executer = AgentEvaluator.get_executer( + evalset_path.name, + call_agent=call_agent, + eval_metrics_file_path_or_dir=metric_arg, + print_detailed_results=False, + print_summary_report=False, + ) + try: + await executer.evaluate() + except AssertionError: + # Case failures are represented as AssertionError by the public facade; + # the structured result remains available from the executer. + pass + result = executer.get_result() + finally: + os.chdir(previous_cwd) + return result or EvaluateResult() + + +def _make_trace_evalset(base_evalset: dict[str, Any], actual_cases: list[dict[str, Any]]) -> dict[str, Any]: + trace = dict(base_evalset) + trace["eval_cases"] = actual_cases + trace["eval_set_id"] = f"{base_evalset.get('eval_set_id', 'set')}_trace" + return trace + + +def _write_trace_evalset( + target_path: Path, + base_evalset: dict[str, Any], + actual_cases: list[dict[str, Any]], + metric_config_path: Path, +) -> None: + target_path.parent.mkdir(parents=True, exist_ok=True) + local_config = target_path.parent / metric_config_path.name + if not local_config.exists(): + shutil.copyfile(metric_config_path, local_config) + target_path.write_text( + json.dumps(_make_trace_evalset(base_evalset, actual_cases), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + +def _case_record( + *, + case: dict[str, Any], + split: str, + champion_runs: list[EvalCaseResult], + challenger_runs: list[EvalCaseResult], +) -> CaseRecord: + champion = _worst_run(champion_runs) + challenger = _worst_run(challenger_runs) + state = _case_state(case) + actual_text: Optional[str] = None + expected_text = _expected_text(case) + actual_tools: list[dict[str, Any]] = [] + expected_tools: list[dict[str, Any]] = [] + actual_responses: list[dict[str, Any]] = [] + expected_responses: list[dict[str, Any]] = [] + metric_results: list[dict[str, Any]] = [] + reasons: list[str] = [] + error_message: Optional[str] = None + + if challenger is not None: + error_message = challenger.error_message + metric_results, reasons = _metric_evidence(challenger) + for per_invocation in challenger.eval_metric_result_per_invocation or []: + actual = per_invocation.actual_invocation + expected = per_invocation.expected_invocation + if actual_text is None: + actual_text = _content_text(getattr(actual, "final_response", None)) + if expected_text is None and expected is not None: + expected_text = _content_text(getattr(expected, "final_response", None)) + actual_tools.extend(_intermediate_items(actual, "tool_uses")) + actual_responses.extend(_intermediate_items(actual, "tool_responses")) + if expected is not None: + expected_tools.extend(_intermediate_items(expected, "tool_uses")) + expected_responses.extend(_intermediate_items(expected, "tool_responses")) + if error_message and error_message not in reasons: + reasons.insert(0, error_message) + + champion_status = _status(champion) if champion is not None else "NOT_EVALUATED" + challenger_status = _status(challenger) if challenger is not None else "NOT_EVALUATED" + return CaseRecord( + eval_id=case["eval_id"], + split=split, + slice_name=str(state.get("slice", "default")), + risk_level=str(state.get("risk_level", "low")), + protected=bool(state.get("protected", False)), + scenario_tag=state.get("scenario_tag"), + champion_status=champion_status, + challenger_status=challenger_status, + champion_score=_score(champion) if champion is not None else 0.0, + challenger_score=_score(challenger) if challenger is not None else 0.0, + expected_text=expected_text, + actual_text=actual_text, + metric_results=metric_results, + actual_tool_uses=actual_tools, + expected_tool_uses=expected_tools, + actual_tool_responses=actual_responses, + expected_tool_responses=expected_responses, + error_message=error_message, + failure_reasons=reasons, + trace_ref=f"{split}_eval.json#eval_id={case['eval_id']}", + ) + + +def _build_case_records( + *, + train_base: dict[str, Any], + val_base: dict[str, Any], + champion_train: EvaluateResult, + champion_val: EvaluateResult, + challenger_train: EvaluateResult, + challenger_val: EvaluateResult, +) -> list[CaseRecord]: + records: list[CaseRecord] = [] + for split, cases, champion_result, challenger_result in ( + ("train", train_base["eval_cases"], champion_train, challenger_train), + ("val", val_base["eval_cases"], champion_val, challenger_val), + ): + champion_map = _flatten_results(champion_result) + challenger_map = _flatten_results(challenger_result) + for case in cases: + eval_id = case["eval_id"] + records.append( + _case_record( + case=case, + split=split, + champion_runs=champion_map.get(eval_id, []), + challenger_runs=challenger_map.get(eval_id, []), + ) + ) + return records + + +def _dump_eval_json(path: Path, *, champion: EvaluateResult, challenger: EvaluateResult) -> None: + def serialize(result: EvaluateResult) -> dict[str, Any]: + return result.model_dump(mode="json", by_alias=True) + + path.write_text( + json.dumps( + {"champion": serialize(champion), "challenger": serialize(challenger)}, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + +async def run_pair( + *, + champion_prompt_path: Path, + challenger_text: str, + train_evalset_path: Path, + val_evalset_path: Path, + metric_config_path: Path, + artifact_root: Path, + mode: str, + candidate_source: str, + scenario: Optional[str], + seed: int, + artifact_dir: Optional[Path] = None, + call_agent: Optional[CallAgent] = None, + run_config: Optional[dict[str, Any]] = None, + gate_config: Optional[dict[str, Any]] = None, + optimizer_config_path: Optional[Path] = None, + model_info: Optional[dict[str, Any]] = None, + optimizer_info: Optional[dict[str, Any]] = None, + cost_status: Optional[str] = None, + total_tokens: Optional[int] = None, + total_cost: Optional[float] = None, + optimizer_artifacts: Optional[dict[str, str]] = None, + optimizer_rounds: Optional[list[dict[str, Any]]] = None, +) -> RunArtifact: + """Evaluate both prompts on train and validation with guaranteed restoration.""" + + started = datetime.now(timezone.utc) + started_at = started.isoformat(timespec="milliseconds").replace("+00:00", "Z") + started_monotonic = time.monotonic() + run_id = artifact_dir.name if artifact_dir is not None else new_run_id() + artifact_dir = artifact_dir or artifact_root / run_id + if artifact_dir.exists(): + if (artifact_dir / "frozen.json").exists(): + raise FileExistsError(f"run artifact already exists: {artifact_dir}") + else: + artifact_dir.mkdir(parents=True, exist_ok=False) + + champion_text = champion_prompt_path.read_text(encoding="utf-8") + run_config = run_config or {} + gate_config = gate_config or {} + optimizer_info = optimizer_info or {} + frozen = FrozenInputs( + run_id=run_id, + champion_sha256=sha256_text(champion_text), + challenger_sha256=sha256_text(challenger_text), + train_sha256=sha256_file(train_evalset_path), + val_sha256=sha256_file(val_evalset_path), + metric_config_sha256=sha256_file(metric_config_path), + run_config_sha256=sha256_json(run_config), + optimizer_config_sha256=( + sha256_file(optimizer_config_path) + if optimizer_config_path is not None and optimizer_config_path.exists() + else None + ), + seed=seed, + started_at=started_at, + mode=mode, + candidate_source=candidate_source, + scenario=scenario, + gate_config=gate_config, + model_info=model_info or {}, + evaluator_info={ + "name": "AgentEvaluator", + "metric_config": str(metric_config_path), + "metric_config_sha256": sha256_file(metric_config_path), + }, + optimizer_info=optimizer_info, + ) + + (artifact_dir / "frozen.json").write_text( + json.dumps(asdict(frozen), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + champion_dir = artifact_dir / "champion_prompts" + challenger_dir = artifact_dir / "challenger_prompts" + champion_dir.mkdir() + challenger_dir.mkdir() + (champion_dir / "system.md").write_text(champion_text, encoding="utf-8") + (challenger_dir / "system.md").write_text(challenger_text, encoding="utf-8") + + train_base = _load_evalset(train_evalset_path) + val_base = _load_evalset(val_evalset_path) + target = TargetPrompt().add_path("system", str(champion_prompt_path)) + snapshot = await target.read_all() + + if mode == "fake": + import fake_agent # type: ignore[unresolved-import] + + paths = { + "champion_train": artifact_dir / "champion_train.evalset.json", + "champion_val": artifact_dir / "champion_val.evalset.json", + "challenger_train": artifact_dir / "challenger_train.evalset.json", + "challenger_val": artifact_dir / "challenger_val.evalset.json", + } + _write_trace_evalset( + paths["champion_train"], + train_base, + fake_agent.gen_actual_conversation(champion_text, train_base), + metric_config_path, + ) + _write_trace_evalset( + paths["champion_val"], + val_base, + fake_agent.gen_actual_conversation(champion_text, val_base), + metric_config_path, + ) + _write_trace_evalset( + paths["challenger_train"], + train_base, + fake_agent.gen_actual_conversation(challenger_text, train_base), + metric_config_path, + ) + _write_trace_evalset( + paths["challenger_val"], + val_base, + fake_agent.gen_actual_conversation(challenger_text, val_base), + metric_config_path, + ) + champion_train = await _run_evaluator(paths["champion_train"], metric_config_path=metric_config_path) + champion_val = await _run_evaluator(paths["champion_val"], metric_config_path=metric_config_path) + challenger_train = await _run_evaluator(paths["challenger_train"], metric_config_path=metric_config_path) + challenger_val = await _run_evaluator(paths["challenger_val"], metric_config_path=metric_config_path) + resolved_cost_status = "measured" + resolved_tokens = 0 + resolved_cost = 0.0 + else: + if call_agent is None: + raise ValueError("optimize mode requires a real call_agent callback") + champion_train = await _run_evaluator( + train_evalset_path, + call_agent=call_agent, + metric_config_path=metric_config_path, + ) + champion_val = await _run_evaluator( + val_evalset_path, + call_agent=call_agent, + metric_config_path=metric_config_path, + ) + try: + await target.write_all({"system": challenger_text}) + challenger_train = await _run_evaluator( + train_evalset_path, + call_agent=call_agent, + metric_config_path=metric_config_path, + ) + challenger_val = await _run_evaluator( + val_evalset_path, + call_agent=call_agent, + metric_config_path=metric_config_path, + ) + finally: + await target.write_all(snapshot) + resolved_cost_status = cost_status or "unavailable" + resolved_tokens = total_tokens if resolved_cost_status == "measured" else None + resolved_cost = total_cost if resolved_cost_status == "measured" else None + + cases = _build_case_records( + train_base=train_base, + val_base=val_base, + champion_train=champion_train, + champion_val=champion_val, + challenger_train=challenger_train, + challenger_val=challenger_val, + ) + train_cases = [case for case in cases if case.split == "train"] + val_cases = [case for case in cases if case.split == "val"] + train = SplitResult( + champion_avg=sum(case.champion_score for case in train_cases) / max(len(train_cases), 1), + challenger_avg=sum(case.challenger_score for case in train_cases) / max(len(train_cases), 1), + ) + val = SplitResult( + champion_avg=sum(case.champion_score for case in val_cases) / max(len(val_cases), 1), + challenger_avg=sum(case.challenger_score for case in val_cases) / max(len(val_cases), 1), + ) + _dump_eval_json(artifact_dir / "train_eval.json", champion=champion_train, challenger=challenger_train) + _dump_eval_json(artifact_dir / "val_eval.json", champion=champion_val, challenger=challenger_val) + + return RunArtifact( + frozen=frozen, + train=train, + val=val, + cases=cases, + champion_train_avg=train.champion_avg, + champion_val_avg=val.champion_avg, + artifact_dir=artifact_dir, + cost_status=resolved_cost_status, + total_tokens=resolved_tokens, + total_cost=resolved_cost, + duration_seconds=time.monotonic() - started_monotonic, + champion_prompt_text=champion_text, + challenger_prompt_text=challenger_text, + optimizer_artifacts=optimizer_artifacts or {}, + optimizer_rounds=optimizer_rounds or [], + ) diff --git a/examples/optimization/eval_optimize_loop/tests/conftest.py b/examples/optimization/eval_optimize_loop/tests/conftest.py new file mode 100644 index 000000000..ba1239c83 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/conftest.py @@ -0,0 +1,61 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Shared pytest setup and safe Champion prompt restoration.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest +import pytest_asyncio + +_HERE = Path(__file__).resolve().parent +_LOOP_ROOT = _HERE.parent +_REPO_ROOT = _LOOP_ROOT.parents[2] + +for path in (str(_REPO_ROOT), str(_LOOP_ROOT)): + if path not in sys.path: + sys.path.insert(0, path) + + +_BASELINE_CHAMPION = """# System Prompt (Champion) + +你是一个数学问答助手。请用中文回答用户的算术问题。 + +## 输出格式 + +请直接给出最终数字答案，无需展示步骤。 + +## 工具使用 + +当前未配置工具。 + + +""" + + +@pytest.fixture(scope="session") +def loop_root() -> Path: + return _LOOP_ROOT + + +@pytest_asyncio.fixture(autouse=True) +async def _restore_champion_prompt(): + """Restore through the production TargetPrompt abstraction.""" + + from trpc_agent_sdk.evaluation import TargetPrompt + + champion = _LOOP_ROOT / "prompts" / "system.md" + target = TargetPrompt().add_path("system", str(champion)) + await target.write_all({"system": _BASELINE_CHAMPION}) + yield + await target.write_all({"system": _BASELINE_CHAMPION}) diff --git a/examples/optimization/eval_optimize_loop/tests/test_apply.py b/examples/optimization/eval_optimize_loop/tests/test_apply.py new file mode 100644 index 000000000..4d77c87d3 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_apply.py @@ -0,0 +1,123 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""验证 Champion prompt 在 dry-run / apply REJECT / apply ACCEPT 三种情况下的写回行为。""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.asyncio + + +def _sha(p: Path) -> str: + import runner + + return runner.sha256_file(p) + + +@pytest.fixture(autouse=True) +def _no_api_key(monkeypatch): + for k in ("OPENAI_API_KEY", "TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + monkeypatch.delenv(k, raising=False) + + +async def test_dry_run_keeps_champion_unchanged(loop_root: Path) -> None: + champion = loop_root / "prompts" / "system.md" + before = _sha(champion) + import pipeline + + await pipeline.amain(["--mode", "fake", "--scenario", "success"]) + assert _sha(champion) == before + + +async def test_apply_rejected_keeps_champion_unchanged(loop_root: Path) -> None: + champion = loop_root / "prompts" / "system.md" + before = _sha(champion) + import pipeline + + rc = await pipeline.amain(["--mode", "fake", "--scenario", "no_effect", "--apply"]) + # REJECT + --apply -> exit 2 + assert rc == 2 + assert _sha(champion) == before + + +async def test_apply_accepted_writes_champion(loop_root: Path) -> None: + """success 场景 + --apply：源 prompt 应被 Challenger 覆盖。 + + 跑完后立即恢复原 Champion 内容，避免污染其他测试。 + 恢复也必须通过 TargetPrompt，避免绕过生产写入抽象。 + """ + champion = loop_root / "prompts" / "system.md" + original_text = champion.read_text(encoding="utf-8") + original_sha = _sha(champion) + try: + import pipeline + + rc = await pipeline.amain(["--mode", "fake", "--scenario", "success", "--apply"]) + assert rc == 0 + new_sha = _sha(champion) + # 必须变了 + assert new_sha != original_sha + # 报告中记录的 after_apply_sha256 必须等于 challenger_sha256 + import json + + report = json.loads((pipeline._HERE / "optimization_report.json").read_text(encoding="utf-8")) + assert report["audit"]["applied"] is True + assert report["audit"]["after_apply_sha256"] == report["frozen"]["challenger_sha256"] + finally: + from trpc_agent_sdk.evaluation import TargetPrompt + + target = TargetPrompt().add_path("system", str(champion)) + await target.write_all({"system": original_text}) + assert _sha(champion) == original_sha + + +async def test_champion_restored_when_challenger_evaluation_crashes( + loop_root: Path, + monkeypatch, +) -> None: + """Exercise the optimize-mode prompt swap, then fail mid-evaluation.""" + + import fake_agent + import runner + from trpc_agent_sdk.evaluation import EvaluateResult + + champion = loop_root / "prompts" / "system.md" + before = _sha(champion) + challenger = fake_agent.build_candidate("success") + calls = 0 + + async def evaluator_stub(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 3: + assert champion.read_text(encoding="utf-8") == challenger + raise RuntimeError("simulated challenger evaluator crash") + return EvaluateResult() + + async def call_agent_stub(query: str) -> str: + return query + + monkeypatch.setattr(runner, "_run_evaluator", evaluator_stub) + with pytest.raises(RuntimeError, match="simulated challenger evaluator crash"): + await runner.run_pair( + champion_prompt_path=champion, + challenger_text=challenger, + train_evalset_path=loop_root / "data" / "train.evalset.json", + val_evalset_path=loop_root / "data" / "val.evalset.json", + metric_config_path=loop_root / "data" / "test_config.json", + artifact_root=loop_root / "runs", + mode="optimize", + candidate_source="agent_optimizer", + scenario=None, + seed=42, + call_agent=call_agent_stub, + ) + assert _sha(champion) == before diff --git a/examples/optimization/eval_optimize_loop/tests/test_attribution.py b/examples/optimization/eval_optimize_loop/tests/test_attribution.py new file mode 100644 index 000000000..9a57f16ab --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_attribution.py @@ -0,0 +1,114 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Evidence-based failure attribution tests.""" + +from __future__ import annotations + +from attribution import AttributionInput, CATEGORIES, classify, classify_with_reason + + +def test_categories_complete() -> None: + assert set(CATEGORIES) == { + "reply_mismatch", + "format_fail", + "tool_call_error", + "param_error", + "rubric_fail", + "knowledge_fail", + "infrastructure_failure", + "insufficient_evidence", + "none", + } + + +def test_passed_returns_none() -> None: + assert classify(AttributionInput(status="PASSED")) == "none" + + +def test_rubric_fail_from_metric_name() -> None: + inp = AttributionInput( + status="FAILED", + metric_names_failed=("llm_rubric_response",), + metric_reasons=("reasoning_clear rubric failed",), + ) + assert classify(inp) == "rubric_fail" + + +def test_tool_call_error_from_trajectory() -> None: + inp = AttributionInput( + status="FAILED", + expected_tool_uses=({"name": "search", "args": {"q": "x"}},), + actual_tool_uses=(), + ) + assert classify(inp) == "tool_call_error" + + +def test_param_error_exposes_parameter_diff() -> None: + inp = AttributionInput( + status="FAILED", + expected_tool_uses=({"name": "search", "args": {"q": "x"}},), + actual_tool_uses=({"name": "search", "args": {"q": "y"}},), + ) + result = classify_with_reason(inp) + assert result.category == "param_error" + assert result.evidence["parameter_differences"] == [ + { + "tool_name": "search", + "expected_args": {"q": "x"}, + "actual_args": {"q": "y"}, + } + ] + + +def test_knowledge_fail_from_evaluator_reason() -> None: + inp = AttributionInput( + status="FAILED", + metric_names_failed=("llm_rubric_knowledge_recall",), + metric_reasons=("knowledge recall did not support rubric kr1",), + ) + assert classify(inp) == "knowledge_fail" + + +def test_format_fail_from_response_evidence() -> None: + inp = AttributionInput( + status="FAILED", + expected_text="步骤：4 + 7 = 11\n答案：11 个", + actual_text="11", + ) + assert classify(inp) == "format_fail" + + +def test_reply_mismatch_from_response_evidence() -> None: + inp = AttributionInput( + status="FAILED", + expected_text="答案：11", + actual_text="答案：22", + ) + assert classify(inp) == "reply_mismatch" + + +def test_infrastructure_failure_is_not_agent_quality_failure() -> None: + inp = AttributionInput( + status="NOT_EVALUATED", + error_message="TimeoutError: provider unavailable", + ) + assert classify(inp) == "infrastructure_failure" + + +def test_missing_raw_evidence_is_not_fabricated_reply_mismatch() -> None: + inp = AttributionInput( + status="FAILED", + metric_names_failed=("final_response_avg_score",), + metric_reasons=("score below threshold",), + ) + assert classify(inp) == "insufficient_evidence" + + +def test_scenario_tag_does_not_control_classification() -> None: + inp = AttributionInput(status="FAILED", scenario_tag="knowledge_lookup") + assert classify(inp) == "insufficient_evidence" diff --git a/examples/optimization/eval_optimize_loop/tests/test_dataset.py b/examples/optimization/eval_optimize_loop/tests/test_dataset.py new file mode 100644 index 000000000..a7399e658 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_dataset.py @@ -0,0 +1,64 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""数据集 sanity check： +- train / val 各 3 条 +- eval_id 互斥 +""" + +from __future__ import annotations + +import json +from pathlib import Path + + +def _load(p: Path) -> dict: + return json.loads(p.read_text(encoding="utf-8")) + + +def test_train_size(loop_root: Path) -> None: + data = _load(loop_root / "data" / "train.evalset.json") + assert len(data["eval_cases"]) == 3 + + +def test_val_size(loop_root: Path) -> None: + data = _load(loop_root / "data" / "val.evalset.json") + assert len(data["eval_cases"]) == 3 + + +def test_eval_id_disjoint(loop_root: Path) -> None: + train = _load(loop_root / "data" / "train.evalset.json") + val = _load(loop_root / "data" / "val.evalset.json") + train_ids = {c["eval_id"] for c in train["eval_cases"]} + val_ids = {c["eval_id"] for c in val["eval_cases"]} + assert train_ids & val_ids == set(), f"train/val eval_id 重叠：{train_ids & val_ids}" + + +def test_eval_id_unique_within_set(loop_root: Path) -> None: + for fname in ("train.evalset.json", "val.evalset.json"): + data = _load(loop_root / "data" / fname) + ids = [c["eval_id"] for c in data["eval_cases"]] + assert len(ids) == len(set(ids)), f"{fname} 中 eval_id 重复" + + +def test_state_metadata_keys(loop_root: Path) -> None: + """每条 case 必须带 split / slice / risk_level / protected / scenario_tag。""" + required = {"split", "slice", "risk_level", "protected", "scenario_tag"} + for fname in ("train.evalset.json", "val.evalset.json"): + data = _load(loop_root / "data" / fname) + for c in data["eval_cases"]: + state = c.get("session_input", {}).get("state", {}) + missing = required - set(state.keys()) + assert not missing, f"{fname} {c['eval_id']} 缺 state 字段：{missing}" + + +def test_one_protected_and_one_high_risk_in_val(loop_root: Path) -> None: + """val 集至少含一个 protected 与一个 high risk case，让 G3/G4 可触发。""" + data = _load(loop_root / "data" / "val.evalset.json") + states = [c["session_input"]["state"] for c in data["eval_cases"]] + assert any(s.get("protected") for s in states) + assert any(s.get("risk_level") == "high" for s in states) diff --git a/examples/optimization/eval_optimize_loop/tests/test_evaluator_evidence.py b/examples/optimization/eval_optimize_loop/tests/test_evaluator_evidence.py new file mode 100644 index 000000000..e518e390f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_evaluator_evidence.py @@ -0,0 +1,260 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Evaluator evidence extraction and labeled holdout tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +import attribution +import gates +import report +import runner + +pytestmark = pytest.mark.asyncio + + +def _content(text: str) -> dict: + return {"role": "model", "parts": [{"text": text}]} + + +def _invocation( + invocation_id: str, + *, + query: str, + answer: str, + tools: list[dict], +) -> dict: + return { + "invocation_id": invocation_id, + "user_content": {"role": "user", "parts": [{"text": query}]}, + "final_response": _content(answer), + "intermediate_data": {"tool_uses": tools, "tool_responses": []}, + } + + +async def test_tool_and_parameter_categories_come_from_real_evaluator_trace( + tmp_path: Path, +) -> None: + expected_tool = {"id": "t1", "name": "search", "args": {"query": "pricing"}} + trace = { + "eval_set_id": "attribution_trace", + "eval_cases": [ + { + "eval_id": "tool_missing", + "eval_mode": "trace", + "actual_conversation": [ + _invocation( + "actual-1", + query="find pricing", + answer="pricing found", + tools=[], + ) + ], + "conversation": [ + _invocation( + "expected-1", + query="find pricing", + answer="pricing found", + tools=[expected_tool], + ) + ], + }, + { + "eval_id": "param_wrong", + "eval_mode": "trace", + "actual_conversation": [ + _invocation( + "actual-2", + query="find pricing", + answer="pricing found", + tools=[ + { + "id": "a2", + "name": "search", + "args": {"query": "weather"}, + } + ], + ) + ], + "conversation": [ + _invocation( + "expected-2", + query="find pricing", + answer="pricing found", + tools=[expected_tool], + ) + ], + }, + ], + } + config = { + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "threshold": 1.0, + "criterion": { + "tool_trajectory": { + "default": { + "name": {"match": "exact", "case_insensitive": False}, + "arguments": {"match": "exact"}, + }, + "order_sensitive": False, + "subset_matching": False, + } + }, + } + ] + } + evalset_path = tmp_path / "tool_trace.evalset.json" + config_path = tmp_path / "test_config.json" + evalset_path.write_text(json.dumps(trace, ensure_ascii=False, indent=2), encoding="utf-8") + config_path.write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding="utf-8") + + result = await runner._run_evaluator( + evalset_path, + metric_config_path=config_path, + ) + flattened = runner._flatten_results(result) + + categories: dict[str, str] = {} + for case in trace["eval_cases"]: + record = runner._case_record( + case=case, + split="holdout", + champion_runs=flattened[case["eval_id"]], + challenger_runs=flattened[case["eval_id"]], + ) + assert record.actual_text == "pricing found" + assert record.expected_text == "pricing found" + assert record.metric_results + assert record.trace_ref + categories[case["eval_id"]] = attribution.from_case_record(record).category + + assert categories == { + "tool_missing": "tool_call_error", + "param_wrong": "param_error", + } + + +async def test_labeled_evidence_holdout_reports_actual_accuracy( + loop_root: Path, + tmp_path: Path, +) -> None: + """The checked-in public holdout is measured, never called a hidden set.""" + + payload = json.loads((loop_root / "data" / "attribution_holdout.json").read_text(encoding="utf-8")) + correct = 0 + predictions: dict[str, str] = {} + records: list[runner.CaseRecord] = [] + for case in payload["cases"]: + result = attribution.classify_with_reason( + attribution.AttributionInput( + status=case["status"], + metric_names_failed=tuple(case.get("failed_metrics", [])), + metric_reasons=tuple(case.get("metric_reasons", [])), + expected_text=case.get("expected_text"), + actual_text=case.get("actual_text"), + expected_tool_uses=tuple(case.get("expected_tool_uses", [])), + actual_tool_uses=tuple(case.get("actual_tool_uses", [])), + expected_tool_responses=tuple(case.get("expected_tool_responses", [])), + actual_tool_responses=tuple(case.get("actual_tool_responses", [])), + error_message=case.get("error_message"), + ) + ) + predictions[case["eval_id"]] = result.category + correct += result.category == case["expected_category"] + records.append( + runner.CaseRecord( + eval_id=case["eval_id"], + split="val", + slice_name="holdout", + risk_level="low", + protected=False, + scenario_tag="deliberately_ignored", + champion_status="FAILED", + challenger_status=case["status"], + champion_score=0.0, + challenger_score=0.0, + expected_text=case.get("expected_text"), + actual_text=case.get("actual_text"), + metric_results=[ + { + "metric_name": metric_name, + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": "; ".join(case.get("metric_reasons", [])), + "rubric_scores": ( + [{"id": "r1", "score": 0.0, "reason": "failed"}] if "rubric" in metric_name else [] + ), + } + for metric_name in case.get("failed_metrics", []) + ], + expected_tool_uses=case.get("expected_tool_uses", []), + actual_tool_uses=case.get("actual_tool_uses", []), + expected_tool_responses=case.get("expected_tool_responses", []), + actual_tool_responses=case.get("actual_tool_responses", []), + error_message=case.get("error_message"), + failure_reasons=case.get("metric_reasons", []), + trace_ref=f"attribution_holdout.json#eval_id={case['eval_id']}", + ) + ) + + assert len(predictions) == 8 + assert correct / len(predictions) == 1.0 + + frozen = runner.FrozenInputs( + run_id="holdout-report", + champion_sha256="champion", + challenger_sha256="challenger", + train_sha256="train", + val_sha256="val", + metric_config_sha256="metric", + run_config_sha256="run", + optimizer_config_sha256=None, + seed=42, + started_at="2026-07-28T00:00:00.000Z", + mode="fixture", + candidate_source="labeled_holdout", + ) + split = runner.SplitResult(champion_avg=0.0, challenger_avg=0.0) + artifact = runner.RunArtifact( + frozen=frozen, + train=split, + val=split, + cases=records, + champion_train_avg=0.0, + champion_val_avg=0.0, + artifact_dir=tmp_path, + cost_status="measured", + total_tokens=0, + total_cost=0.0, + duration_seconds=0.0, + champion_prompt_text="", + challenger_prompt_text="", + ) + report_payload = report.build_report_dict( + artifact, + gates.Decision(accepted=False, violated=["G1"], reasons=["fixture"]), + applied=False, + before_apply_sha256="champion", + after_apply_sha256=None, + repro_cmd="pytest test_evaluator_evidence.py", + ) + report.write_report(report_payload, out_dir=tmp_path) + persisted = json.loads((tmp_path / "optimization_report.json").read_text(encoding="utf-8")) + per_case = {case["eval_id"]: case for case in persisted["per_case"]} + assert per_case["holdout_tool"]["evidence"]["expected_tool_uses"] + assert per_case["holdout_param"]["evidence"]["attribution"]["parameter_differences"] + assert per_case["holdout_rubric"]["evidence"]["metric_results"][0]["rubric_scores"] + assert per_case["holdout_infra"]["failure_kind"] == "infrastructure_failure" + assert per_case["holdout_reply"]["failure_kind"] == "agent_quality_failure" diff --git a/examples/optimization/eval_optimize_loop/tests/test_fake_agent.py b/examples/optimization/eval_optimize_loop/tests/test_fake_agent.py new file mode 100644 index 000000000..10912510c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_fake_agent.py @@ -0,0 +1,264 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""fake_agent 行为约束： +- 只有 FAKE_CONTROLS 块内 ``KEY=true`` 才启用行为 +- 注释文字、自然语言叙述不得触发 marker +- scenario 三类候选可正确生成 +- 真值表：Champion/Success/Overfit 在 train/val protected/val 其他 上的对错 +""" + +from __future__ import annotations + +from pathlib import Path + +import fake_agent + +# ---- 固定的 evalset 样例（与 data/evalset 同 schema，独立于文件） ---- +_TRAIN_CASE = { + "eval_id": "train_x", + "session_input": { + "app_name": "x", + "user_id": "train", + "state": { + "split": "train", + "slice": "math", + "risk_level": "low", + "protected": False, + "scenario_tag": "add_steps", + }, + }, + "conversation": [ + { + "invocation_id": "t", + "user_content": {"parts": [{"text": "q"}], "role": "user"}, + "final_response": {"parts": [{"text": "步骤：1\n答案：1 个"}], "role": "model"}, + } + ], +} +_VAL_PROTECTED_CASE = { + "eval_id": "val_p", + "session_input": { + "app_name": "x", + "user_id": "val", + "state": { + "split": "val", + "slice": "risky", + "risk_level": "high", + "protected": True, + "scenario_tag": "protected", + }, + }, + "conversation": [ + { + "invocation_id": "v", + "user_content": {"parts": [{"text": "q2"}], "role": "user"}, + "final_response": {"parts": [{"text": "答案：2 人"}], "role": "model"}, + } + ], +} +_VAL_OTHER_CASE = { + "eval_id": "val_o", + "session_input": { + "app_name": "x", + "user_id": "val", + "state": { + "split": "val", + "slice": "math", + "risk_level": "low", + "protected": False, + "scenario_tag": "add_steps", + }, + }, + "conversation": [ + { + "invocation_id": "v2", + "user_content": {"parts": [{"text": "q3"}], "role": "user"}, + "final_response": {"parts": [{"text": "答案：3 个"}], "role": "model"}, + } + ], +} + + +# ---- parse_behavior：只读 FAKE_CONTROLS 块，严格 true ---- +def test_parse_behavior_baseline_no_block() -> None: + b = fake_agent.parse_behavior("完全没有控制块的 prompt") + assert not b.add_steps + assert not b.memorize_train + + +def test_parse_behavior_explicit_false() -> None: + text = """ + + """ + b = fake_agent.parse_behavior(text) + assert not b.add_steps + assert not b.memorize_train + + +def test_parse_behavior_add_steps_true() -> None: + text = """ + + """ + b = fake_agent.parse_behavior(text) + assert b.add_steps + assert not b.memorize_train + + +def test_parse_behavior_memorize_true() -> None: + text = """ + + """ + b = fake_agent.parse_behavior(text) + assert not b.add_steps + assert b.memorize_train + + +def test_parse_behavior_rejects_true_with_caps() -> None: + """值必须严格小写 true；True / TRUE / yes 均视为 false。""" + text = """ + + """ + b = fake_agent.parse_behavior(text) + assert not b.add_steps + assert not b.memorize_train + + +def test_parse_behavior_ignores_marker_in_comments() -> None: + """注释 / 文档叙述里出现 ADD_STEPS=true 不得触发行为。""" + text = """ + # 示例：ADD_STEPS=true 是启用标记 + 文档中提到 MEMORIZE_TRAIN=true 只在 fake 模式生效 + + """ + b = fake_agent.parse_behavior(text) + assert not b.add_steps + assert not b.memorize_train + + +# ---- candidates ---- +def test_candidate_builders_exist() -> None: + for s in ("success", "no_effect", "overfit"): + c = fake_agent.build_candidate(s) + assert isinstance(c, str) and len(c) > 0 + + +def test_success_candidate_has_add_steps_true() -> None: + b = fake_agent.parse_behavior(fake_agent.build_candidate("success")) + assert b.add_steps + assert not b.memorize_train + + +def test_overfit_candidate_has_memorize_true() -> None: + b = fake_agent.parse_behavior(fake_agent.build_candidate("overfit")) + assert not b.add_steps + assert b.memorize_train + + +def test_no_effect_candidate_is_baseline() -> None: + b = fake_agent.parse_behavior(fake_agent.build_candidate("no_effect")) + assert not b.add_steps + assert not b.memorize_train + + +def test_champion_prompt_baseline_behavior() -> None: + """prompts/system.md 的 baseline 必须解析为全 false。""" + from pathlib import Path + + champ = Path(__file__).resolve().parent.parent / "prompts" / "system.md" + b = fake_agent.parse_behavior(champ.read_text(encoding="utf-8")) + assert not b.add_steps + assert not b.memorize_train + + +# ---- 真值表：根据 README 决策矩阵 ---- +def test_baseline_train_wrong_val_protected_right_val_other_wrong() -> None: + """FAKE_CONTROLS 全 false：train 错 / val protected 对 / val 其他 错。""" + prompt = """ + + """ + assert "无法回答" in fake_agent.gen_final_response_for_case(prompt, case=_TRAIN_CASE) + assert "无法回答" not in fake_agent.gen_final_response_for_case( + prompt, + case=_VAL_PROTECTED_CASE, + ) + assert "无法回答" in fake_agent.gen_final_response_for_case( + prompt, + case=_VAL_OTHER_CASE, + ) + + +def test_add_steps_all_correct() -> None: + """ADD_STEPS=true：所有 case 都对（输出 expected_response 原文）。""" + prompt = fake_agent.build_candidate("success") + for case in (_TRAIN_CASE, _VAL_PROTECTED_CASE, _VAL_OTHER_CASE): + out = fake_agent.gen_final_response_for_case(prompt, case=case) + assert "无法回答" not in out + expected = case["conversation"][0]["final_response"]["parts"][0]["text"] + assert out == expected + + +def test_memorize_train_only_train_correct() -> None: + """MEMORIZE_TRAIN=true：train 对 / val (含 protected) 全错。""" + prompt = fake_agent.build_candidate("overfit") + train_out = fake_agent.gen_final_response_for_case(prompt, case=_TRAIN_CASE) + assert "无法回答" not in train_out + assert train_out == _TRAIN_CASE["conversation"][0]["final_response"]["parts"][0]["text"] + + val_prot = fake_agent.gen_final_response_for_case(prompt, case=_VAL_PROTECTED_CASE) + assert "无法回答" in val_prot + + val_other = fake_agent.gen_final_response_for_case(prompt, case=_VAL_OTHER_CASE) + assert "无法回答" in val_other + + +def test_gen_actual_conversation_produces_trace_cases() -> None: + """gen_actual_conversation 输出结构正确：每个 case 含 eval_mode=trace 和 actual_conversation。""" + evalset = {"eval_set_id": "x", "eval_cases": [_TRAIN_CASE, _VAL_PROTECTED_CASE]} + out = fake_agent.gen_actual_conversation( + fake_agent.build_candidate("success"), + evalset, + ) + assert len(out) == 2 + for new_case in out: + assert new_case["eval_mode"] == "trace" + assert len(new_case["actual_conversation"]) == 1 + assert "final_response" in new_case["actual_conversation"][0] + + +# ---- 暗道禁止 ---- +def test_no_sha_switch_no_dark_path() -> None: + """静态扫描 fake_agent 源码，禁止任何哈希相关分支。""" + src = (Path(fake_agent.__file__)).read_text(encoding="utf-8") + forbidden = ["hashlib", "sha256", ".__hash__"] + for kw in forbidden: + assert kw not in src, f"fake_agent.py 不允许使用 {kw}" + + +def test_no_identity_keyword_branching() -> None: + """静态扫描，禁止 'champion' / 'challenger' / 'scenario ==' 形式的身份分支。""" + src = (Path(fake_agent.__file__)).read_text(encoding="utf-8").lower() + assert "champion" not in src + assert "challenger" not in src + # 不允许出现基于 scenario 字符串的分支 + assert "scenario ==" not in src + assert "scenario_param" not in src diff --git a/examples/optimization/eval_optimize_loop/tests/test_gates.py b/examples/optimization/eval_optimize_loop/tests/test_gates.py new file mode 100644 index 000000000..690717849 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_gates.py @@ -0,0 +1,205 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""G1-G7 gate 单元测试。纯规则，无需 SDK。""" + +from __future__ import annotations + +import gates + + +def _case( + eval_id="c1", + split="val", + slice_name="math", + risk_level="low", + protected=False, + champion_status="PASSED", + challenger_status="PASSED", + champion_score=1.0, + challenger_score=1.0, +): + return gates.CaseDelta( + eval_id=eval_id, + split=split, + slice_name=slice_name, + risk_level=risk_level, + protected=protected, + champion_status=champion_status, + challenger_status=challenger_status, + champion_score=champion_score, + challenger_score=challenger_score, + ) + + +def test_g1_accept_when_val_lift_meets_threshold() -> None: + cases = [_case(champion_score=0.5, challenger_score=0.8)] + d = gates.evaluate(cases) + assert d.accepted + assert "G1" not in d.violated + + +def test_g1_reject_when_val_lift_insufficient() -> None: + cases = [_case(champion_score=0.5, challenger_score=0.51)] + d = gates.evaluate(cases, config=gates.GateConfig(min_val_lift=0.02)) + assert not d.accepted + assert "G1" in d.violated + + +def test_g2_overfit_train_up_val_down_rejected() -> None: + train = [ + _case("t1", split="train", champion_score=0.3, challenger_score=0.9), + _case("t2", split="train", champion_score=0.3, challenger_score=0.9), + ] + val = [ + _case("v1", champion_score=0.7, challenger_score=0.4), + ] + d = gates.evaluate(train + val) + assert "G2" in d.violated + assert not d.accepted + # 理由中必须提到过拟合 + assert any("过拟合" in r for r in d.reasons) + + +def test_g2_not_triggered_when_both_improve() -> None: + train = [_case("t", split="train", champion_score=0.3, challenger_score=0.9)] + val = [_case("v", champion_score=0.5, challenger_score=0.8)] + d = gates.evaluate(train + val) + assert "G2" not in d.violated + + +def test_g3_new_hard_fail_rejected() -> None: + cases = [ + _case( + eval_id="h", + risk_level="high", + champion_status="PASSED", + challenger_status="FAILED", + champion_score=1.0, + challenger_score=0.0, + ), + # 必须有 val lift，否则会先被 G1 卡掉，看不出 G3 + _case(eval_id="v", champion_score=0.0, challenger_score=1.0), + ] + d = gates.evaluate(cases) + assert "G3" in d.violated + + +def test_g3_high_risk_but_already_failing_not_counted() -> None: + cases = [ + _case( + eval_id="h", + risk_level="high", + champion_status="FAILED", + challenger_status="FAILED", + champion_score=0.0, + challenger_score=0.0, + ), + _case(eval_id="v", champion_score=0.0, challenger_score=1.0), + ] + d = gates.evaluate(cases) + assert "G3" not in d.violated + + +def test_g4_protected_regression_rejected() -> None: + cases = [ + _case( + eval_id="p", + protected=True, + champion_score=1.0, + challenger_score=0.5, + ), + _case(eval_id="v", champion_score=0.0, challenger_score=1.0), + ] + d = gates.evaluate(cases) + assert "G4" in d.violated + + +def test_g5_slice_regression_rejected() -> None: + cases = [ + _case("a1", slice_name="alpha", champion_score=1.0, challenger_score=1.0), + _case( + "a2", + slice_name="beta", + champion_score=1.0, + challenger_score=0.0, + ), + # 整体 val lift 必须够，否则会被 G1 拦下 + _case("a3", slice_name="gamma", champion_score=0.0, challenger_score=1.0), + _case("a4", slice_name="gamma", champion_score=0.0, challenger_score=1.0), + ] + d = gates.evaluate(cases, config=gates.GateConfig(slice_tolerance=0.05)) + assert "G5" in d.violated + + +def test_g6_cost_unavailable_rejected() -> None: + cases = [_case(champion_score=0.0, challenger_score=1.0)] + d = gates.evaluate(cases, cost_status="unavailable") + assert "G6" in d.violated + + +def test_g6_cost_over_budget_rejected() -> None: + cases = [_case(champion_score=0.0, challenger_score=1.0)] + d = gates.evaluate( + cases, + cost_status="measured", + total_tokens=10_000_000, + config=gates.GateConfig(budget_tokens=100_000), + ) + assert "G6" in d.violated + + +def test_g6_usd_cost_over_budget_rejected() -> None: + cases = [_case(champion_score=0.0, challenger_score=1.0)] + decision = gates.evaluate( + cases, + cost_status="measured", + total_tokens=100, + total_cost=1.5, + config=gates.GateConfig(budget_usd=1.0), + ) + assert "G6" in decision.violated + + +def test_g6_measured_with_missing_value_rejected() -> None: + cases = [_case(champion_score=0.0, challenger_score=1.0)] + decision = gates.evaluate( + cases, + cost_status="measured", + total_tokens=None, + total_cost=None, + ) + assert "G6" in decision.violated + + +def test_g6_cost_measured_zero_passes() -> None: + cases = [_case(champion_score=0.0, challenger_score=1.0)] + d = gates.evaluate(cases, cost_status="measured", total_tokens=0) + assert "G6" not in d.violated + + +def test_g7_epsilon_guard() -> None: + """val_delta 接近 0 时，即使 G1-G6 全过，G7 也拦下。""" + cases = [ + _case(champion_score=1.0, challenger_score=1.0001), + ] + d = gates.evaluate( + cases, + cost_status="measured", + config=gates.GateConfig(min_val_lift=0.0, epsilon=0.001), + ) + assert "G7" in d.violated + + +def test_all_pass_accept() -> None: + cases = [ + _case(eval_id="t1", split="train", champion_score=0.4, challenger_score=0.9), + _case(eval_id="v1", champion_score=0.4, challenger_score=0.9), + ] + d = gates.evaluate(cases) + assert d.accepted + assert d.violated == [] diff --git a/examples/optimization/eval_optimize_loop/tests/test_native_optimize.py b/examples/optimization/eval_optimize_loop/tests/test_native_optimize.py new file mode 100644 index 000000000..e6701950e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_native_optimize.py @@ -0,0 +1,231 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""End-to-end tests for the native AgentOptimizer integration.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.asyncio + +_CANDIDATE = """# Native Optimizer Candidate + +Return the labeled reference answer for every registered evaluation query. + +TEST_NATIVE_GOOD=true +""" + + +def _answers(loop_root: Path) -> dict[str, str]: + answers: dict[str, str] = {} + for name in ("train.evalset.json", "val.evalset.json"): + payload = json.loads((loop_root / "data" / name).read_text(encoding="utf-8")) + for case in payload["eval_cases"]: + invocation = case["conversation"][0] + query = invocation["user_content"]["parts"][0]["text"] + answer = invocation["final_response"]["parts"][0]["text"] + answers[query] = answer + return answers + + +def _install_optimizer_stub( + monkeypatch, + *, + total_tokens: int, + total_cost: float, + reflection_calls: int, +) -> list[dict]: + from trpc_agent_sdk.evaluation import AgentOptimizer + + calls: list[dict] = [] + + async def optimize_stub(**kwargs): + calls.append(kwargs) + assert kwargs["update_source"] is False + baseline = await kwargs["target_prompt"].read_all() + assert "TEST_NATIVE_GOOD" not in baseline["system"] + + output_dir = Path(kwargs["output_dir"]) + (output_dir / "rounds").mkdir(parents=True, exist_ok=True) + (output_dir / "best_prompts").mkdir(parents=True, exist_ok=True) + (output_dir / "baseline_prompts").mkdir(parents=True, exist_ok=True) + (output_dir / "best_prompts" / "system.md").write_text(_CANDIDATE, encoding="utf-8") + (output_dir / "baseline_prompts" / "system.md").write_text(baseline["system"], encoding="utf-8") + round_record = { + "round_index": 1, + "input_prompt_sha256": "baseline", + "candidate_prompt": _CANDIDATE, + "accepted": True, + "validation_score": 1.0, + } + (output_dir / "rounds" / "round_001.json").write_text( + json.dumps(round_record, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + (output_dir / "result.json").write_text( + json.dumps({"status": "SUCCEEDED", "best_prompts": {"system": _CANDIDATE}}), + encoding="utf-8", + ) + (output_dir / "config.snapshot.json").write_text("{}\n", encoding="utf-8") + (output_dir / "run.log").write_text("stub optimizer succeeded\n", encoding="utf-8") + (output_dir / "summary.txt").write_text("stub optimizer succeeded\n", encoding="utf-8") + + return SimpleNamespace( + algorithm="test_native_stub", + status="SUCCEEDED", + finish_reason="COMPLETED", + stop_reason="completed", + error_message="", + best_prompts={"system": _CANDIDATE}, + total_rounds=1, + rounds=[round_record], + baseline_pass_rate=0.0, + best_pass_rate=1.0, + pass_rate_improvement=1.0, + total_token_usage={ + "prompt": total_tokens, + "completion": 0, + "total": total_tokens, + }, + total_llm_cost=total_cost, + total_reflection_lm_calls=reflection_calls, + ) + + monkeypatch.setattr(AgentOptimizer, "optimize", staticmethod(optimize_stub)) + return calls + + +def _call_agent(loop_root: Path): + answers = _answers(loop_root) + prompt_path = loop_root / "prompts" / "system.md" + + async def call_agent(query: str) -> str: + if query not in answers: + raise KeyError(query) + if "TEST_NATIVE_GOOD" in prompt_path.read_text(encoding="utf-8"): + return answers[query] + return "答案：0" + + # This deterministic callback makes no provider call; zero is measured. + call_agent.cost_status = "measured" + call_agent.total_tokens = 0 + call_agent.total_cost = 0.0 + return call_agent + + +async def test_native_optimize_candidate_reaches_regression_and_gate( + monkeypatch, + loop_root: Path, +) -> None: + calls = _install_optimizer_stub( + monkeypatch, + total_tokens=123, + total_cost=0.0123, + reflection_calls=1, + ) + + import pipeline + + rc = await pipeline.amain( + ["--mode", "optimize"], + call_agent=_call_agent(loop_root), + ) + assert rc == 0 + assert len(calls) == 1 + + optimize_call = calls[0] + assert optimize_call["update_source"] is False + assert Path(optimize_call["train_dataset_path"]).name == "train.evalset.json" + assert Path(optimize_call["validation_dataset_path"]).name == "val.evalset.json" + + payload = json.loads((loop_root / "optimization_report.json").read_text(encoding="utf-8")) + assert payload["candidate_source"] == "agent_optimizer" + assert payload["decision"]["accepted"] is True + assert payload["results"]["train"]["delta"] > 0 + assert payload["results"]["val"]["delta"] > 0 + assert payload["optimizer"]["rounds"][0]["candidate_prompt"] == _CANDIDATE + assert Path(payload["optimizer"]["rounds"][0]["artifact_path"]).is_file() + assert payload["cost"] == { + "status": "measured", + "total_tokens": 123, + "total_usd": 0.0123, + } + assert all(Path(path).exists() for path in payload["optimizer"]["artifacts"].values()) + + audit_path = Path(payload["optimizer"]["artifacts"]["call_agent_audit"]) + audit = json.loads(audit_path.read_text(encoding="utf-8")) + assert audit + assert all("query" in entry and "eval_contexts" in entry for entry in audit) + assert all(context["eval_id"] != entry["query"] for entry in audit for context in entry["eval_contexts"]) + + +async def test_cost_unavailable_rejects_apply_and_preserves_champion( + monkeypatch, + loop_root: Path, +) -> None: + _install_optimizer_stub( + monkeypatch, + total_tokens=0, + total_cost=0.0, + reflection_calls=1, + ) + champion = loop_root / "prompts" / "system.md" + before = champion.read_bytes() + + import pipeline + + rc = await pipeline.amain( + ["--mode", "optimize", "--apply"], + call_agent=_call_agent(loop_root), + ) + assert rc == 2 + assert champion.read_bytes() == before + + payload = json.loads((loop_root / "optimization_report.json").read_text(encoding="utf-8")) + assert payload["decision"]["accepted"] is False + assert "G6" in payload["decision"]["violated"] + assert payload["audit"]["applied"] is False + assert payload["cost"] == { + "status": "unavailable", + "total_tokens": None, + "total_usd": None, + } + + +async def test_missing_model_config_writes_auditable_reject( + monkeypatch, + loop_root: Path, +) -> None: + for key in ( + "TRPC_AGENT_API_KEY", + "TRPC_AGENT_BASE_URL", + "TRPC_AGENT_MODEL_NAME", + ): + monkeypatch.delenv(key, raising=False) + + import pipeline + + rc = await pipeline.amain(["--mode", "optimize"]) + assert rc == 1 + payload = json.loads( + (loop_root / "optimization_report.json").read_text(encoding="utf-8") + ) + assert payload["decision"]["violated"] == ["OPTIMIZER_FAILURE", "G6"] + assert payload["cost"] == { + "status": "unavailable", + "total_tokens": None, + "total_usd": None, + } + assert "TRPC_AGENT_API_KEY" in payload["optimizer"]["error"] + assert Path( + payload["optimizer"]["artifacts"]["optimizer_error"] + ).is_file() diff --git a/examples/optimization/eval_optimize_loop/tests/test_report_fields.py b/examples/optimization/eval_optimize_loop/tests/test_report_fields.py new file mode 100644 index 000000000..a623cd667 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_report_fields.py @@ -0,0 +1,148 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Validate optimization report schema and persisted evidence.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.asyncio + + +REQUIRED_FROZEN_KEYS = { + "champion_sha256", + "challenger_sha256", + "train_sha256", + "val_sha256", + "metric_config_sha256", + "run_config_sha256", + "optimizer_config_sha256", + "seed", + "started_at", + "mode", + "candidate_source", + "gate_config", + "model_info", + "evaluator_info", + "optimizer_info", +} + +REQUIRED_PER_CASE_KEYS = { + "eval_id", + "split", + "slice", + "risk_level", + "protected", + "champion_status", + "challenger_status", + "champion_score", + "challenger_score", + "delta", + "category", + "transition", + "failure_kind", + "failure_reason", + "evidence", +} + +REQUIRED_AUDIT_KEYS = { + "applied", + "duration_seconds", + "candidate_source", + "artifact_dir", + "before_apply_sha256", + "after_apply_sha256", + "repro_cmd", + "artifacts", +} + + +@pytest.fixture(autouse=True) +def _no_api_key(monkeypatch): + for k in ("OPENAI_API_KEY", "TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + monkeypatch.delenv(k, raising=False) + + +async def test_report_has_all_required_fields(loop_root: Path) -> None: + import pipeline + + await pipeline.amain(["--mode", "fake", "--scenario", "no_effect"]) + report = json.loads((loop_root / "optimization_report.json").read_text(encoding="utf-8")) + + assert report["version"] == "2.0" + assert REQUIRED_FROZEN_KEYS <= set(report["frozen"].keys()) + assert {"train", "val"} <= set(report["results"].keys()) + for k in ("train", "val"): + assert {"champion_avg", "challenger_avg", "delta"} <= set(report["results"][k].keys()) + assert "train_delta" in report and "val_delta" in report + assert "cost_status" in report + assert "cost" in report + assert {"accepted", "violated", "reasons"} <= set(report["decision"].keys()) + assert REQUIRED_AUDIT_KEYS <= set(report["audit"].keys()) + + assert len(report["per_case"]) == 6 + for c in report["per_case"]: + assert REQUIRED_PER_CASE_KEYS <= set(c.keys()) + assert c["evidence"]["actual_text"] is not None + assert c["evidence"]["expected_text"] is not None + assert c["evidence"]["trace_ref"] + assert "metric_results" in c["evidence"] + + assert report["frozen"]["mode"] == "fake" + assert report["frozen"]["candidate_source"] == "candidate_file" + + +async def test_train_val_delta_consistent(loop_root: Path) -> None: + import pipeline + + await pipeline.amain(["--mode", "fake", "--scenario", "no_effect"]) + report = json.loads((loop_root / "optimization_report.json").read_text(encoding="utf-8")) + + # report.results.train.delta == report.train_delta + assert abs(report["results"]["train"]["delta"] - report["train_delta"]) < 1e-9 + assert abs(report["results"]["val"]["delta"] - report["val_delta"]) < 1e-9 + + +async def test_markdown_presented_and_aligned(loop_root: Path) -> None: + import pipeline + + await pipeline.amain(["--mode", "fake", "--scenario", "no_effect"]) + md = (loop_root / "optimization_report.md").read_text(encoding="utf-8") + # 必须含核心章节 + assert "决策" in md + assert "Gate 决策" in md + assert "逐 case 明细" in md + assert "审计" in md + assert "REJECT" in md or "ACCEPT" in md + + +async def test_artifacts_files_exist(loop_root: Path) -> None: + import pipeline + + await pipeline.amain(["--mode", "fake", "--scenario", "no_effect"]) + report = json.loads((loop_root / "optimization_report.json").read_text(encoding="utf-8")) + artifacts = report["audit"]["artifacts"] + for k, p in artifacts.items(): + path = Path(p) + assert path.exists(), f"{k} artifact 文件不存在：{p}" + + +async def test_checked_in_example_report_matches_v2_schema(loop_root: Path) -> None: + payload = json.loads( + (loop_root / "optimization_report.example.json").read_text(encoding="utf-8") + ) + assert payload["version"] == "2.0" + assert REQUIRED_FROZEN_KEYS <= set(payload["frozen"]) + assert REQUIRED_AUDIT_KEYS <= set(payload["audit"]) + assert REQUIRED_PER_CASE_KEYS <= set(payload["per_case"][0]) + assert payload["optimizer"]["rounds"][0]["artifact_path"] + categories = {case["category"] for case in payload["per_case"]} + assert {"tool_call_error", "param_error", "rubric_fail"} <= categories diff --git a/examples/optimization/eval_optimize_loop/tests/test_sha256_normalize.py b/examples/optimization/eval_optimize_loop/tests/test_sha256_normalize.py new file mode 100644 index 000000000..657433864 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_sha256_normalize.py @@ -0,0 +1,50 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""sha256 规范化测试：Windows CRLF 与 Unix LF 内容应产生相同 hash。""" + +from __future__ import annotations + +import re +from pathlib import Path + +from runner import new_run_id, sha256_file, sha256_text + + +def test_sha256_text_crlf_matches_lf() -> None: + lf_text = "步骤：1\n答案：1 个\n" + crlf_text = "步骤：1\r\n答案：1 个\r\n" + assert sha256_text(lf_text) == sha256_text(crlf_text) + + +def test_sha256_text_lf_unchanged() -> None: + text = "hello\nworld\n" + # 规范化后与原文本一致 → 与裸 hashlib 一致 + import hashlib + + expected = hashlib.sha256(text.encode("utf-8")).hexdigest() + assert sha256_text(text) == expected + + +def test_sha256_file_crlf_matches_lf(tmp_path: Path) -> None: + lf_file = tmp_path / "lf.md" + crlf_file = tmp_path / "crlf.md" + lf_file.write_bytes("步骤：1\n答案：1 个\n".encode("utf-8")) + crlf_file.write_bytes("步骤：1\r\n答案：1 个\r\n".encode("utf-8")) + assert sha256_file(lf_file) == sha256_file(crlf_file) + + +def test_run_id_uses_utc_microseconds_and_random_suffix() -> None: + run_ids = [new_run_id() for _ in range(200)] + assert len(set(run_ids)) == len(run_ids) + assert all( + re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{6}Z-[0-9a-f]{8}", + run_id, + ) + for run_id in run_ids + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_truth_table.py b/examples/optimization/eval_optimize_loop/tests/test_truth_table.py new file mode 100644 index 000000000..6a05a590f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_truth_table.py @@ -0,0 +1,140 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""端到端真值表测试：直接调用 pipeline + runner + SDK evaluator， +对三个 fake 场景断言 case-level 分数、train/val delta、decision。 + +该测试是 Issue #91 Milestone A 的验收测试： +test_fake_agent.py 单独通过不能视为验收完成。 +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture(autouse=True) +def _no_api_key(monkeypatch): + """强制无 API key，验证 fake 模式不需 key 也能跑。""" + for k in ("OPENAI_API_KEY", "TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + monkeypatch.delenv(k, raising=False) + + +def _per_case_scores(report: dict) -> dict[str, dict[str, float]]: + """把 per_case 列表整理成 {eval_id: {champ, chall, delta}}。""" + out = {} + for c in report["per_case"]: + out[c["eval_id"]] = { + "champion": c["champion_score"], + "challenger": c["challenger_score"], + "delta": c["delta"], + "split": c["split"], + "protected": c["protected"], + } + return out + + +async def _run(scenario: str) -> dict: + import pipeline + + rc = await pipeline.amain(["--mode", "fake", "--scenario", scenario]) + report_path = pipeline._HERE / "optimization_report.json" + report = json.loads(report_path.read_text(encoding="utf-8")) + report["_exit_code"] = rc + return report + + +# ---- 期望的真值表（与 README 一致） ---- +# Champion: train=[0,0,0] val=[0,1,0] (protected case 初始正确) +# success: train=[1,1,1] val=[1,1,1] ACCEPT +# no_effect: 同 Champion REJECT G1 +# overfit: train=[1,1,1] val=[0,0,0] REJECT G2 + 过拟合 + +# eval_id 来自 data/train.evalset.json + data/val.evalset.json +_TRAIN_IDS = ["train_add_steps", "train_noop", "train_overfit_mem"] +_VAL_IDS = ["val_add_steps_b", "val_protected_highrisk", "val_noop_b"] + + +async def test_success_candidate_truth_table() -> None: + """success 场景：train/val 全对，ACCEPT。""" + report = await _run("success") + + scores = _per_case_scores(report) + # Champion baseline + for tid in _TRAIN_IDS: + assert scores[tid]["champion"] == 0.0, f"{tid} champion should be 0" + assert scores["val_protected_highrisk"]["champion"] == 1.0, "protected case 初始正确" + for vid in ("val_add_steps_b", "val_noop_b"): + assert scores[vid]["champion"] == 0.0, f"{vid} champion should be 0" + + # Challenger 全对 + for tid in _TRAIN_IDS: + assert scores[tid]["challenger"] == 1.0, f"{tid} challenger should be 1" + for vid in _VAL_IDS: + assert scores[vid]["challenger"] == 1.0, f"{vid} challenger should be 1" + + # delta + assert report["train_delta"] > 0 + assert report["val_delta"] > 0 + assert report["val_delta"] >= 0.02, "val_delta >= min_val_lift" + + # decision + assert report["decision"]["accepted"] is True + assert report["decision"]["violated"] == [] + + +async def test_no_effect_candidate_truth_table() -> None: + """no_effect 场景：candidate 行为 = Champion，REJECT G1。""" + report = await _run("no_effect") + + scores = _per_case_scores(report) + # Champion 与 Challenger 分数完全相同 + for eid in _TRAIN_IDS + _VAL_IDS: + assert scores[eid]["champion"] == scores[eid]["challenger"], f"{eid} champ=chall" + # delta == 0 + assert abs(report["train_delta"]) < 1e-9 + assert abs(report["val_delta"]) < 1e-9 + + # decision: REJECT 且违反 G1 + assert report["decision"]["accepted"] is False + assert "G1" in report["decision"]["violated"] + + +async def test_overfit_candidate_truth_table() -> None: + """overfit 场景：train 涨 val 跌，REJECT G2 且理由含'过拟合'。""" + report = await _run("overfit") + + scores = _per_case_scores(report) + # Champion: train=[0,0,0], val=[0,1,0] + for tid in _TRAIN_IDS: + assert scores[tid]["champion"] == 0.0 + assert scores["val_protected_highrisk"]["champion"] == 1.0 + for vid in ("val_add_steps_b", "val_noop_b"): + assert scores[vid]["champion"] == 0.0 + + # Challenger: train=[1,1,1], val=[0,0,0] (含 protected 也跌) + for tid in _TRAIN_IDS: + assert scores[tid]["challenger"] == 1.0, f"{tid} challenger should be 1 (memorized)" + for vid in _VAL_IDS: + assert scores[vid]["challenger"] == 0.0, f"{vid} challenger should be 0 (forgot)" + + # delta: train > 0, val < 0 + assert report["train_delta"] > 0 + assert report["val_delta"] < 0, f"val_delta={report['val_delta']} should be negative" + + # decision: REJECT 且违反 G2（过拟合） + assert report["decision"]["accepted"] is False + assert "G2" in report["decision"]["violated"], report["decision"] + assert any("过拟合" in r for r in report["decision"]["reasons"]), report["decision"] + # G3（high-risk 新增 fail）与 G4（protected 退化）也应触发 + assert "G3" in report["decision"]["violated"] + assert "G4" in report["decision"]["violated"] From cc1f7cc3c0819bd6892100c82685a799836b9bfb Mon Sep 17 00:00:00 2001 From: ygrowly Date: Wed, 29 Jul 2026 22:15:17 +0800 Subject: [PATCH 2/6] fix: harden native optimize path and clean up PR artifacts - Fix Windows drive-colon parsing in _agent_evaluator eval set loader - Load optimizer credentials from gitignored .env; lazy-import live_agent - Remove internal notes and local tool config from the branch - Rewrite DESIGN.md as the 300-500 word design note required by #91 --- .claude/settings.local.json | 27 - .gitignore | 2 + Evaluation + Optimization.md | 32 - eval_optimize_analysis.md | 671 ------------------ .../eval_optimize_loop/.gitignore | 3 + .../optimization/eval_optimize_loop/DESIGN.md | 208 +----- .../eval_optimize_loop/optimizer.json | 3 +- .../eval_optimize_loop/pipeline.py | 46 +- trpc_agent_sdk/evaluation/_agent_evaluator.py | 14 +- 9 files changed, 74 insertions(+), 932 deletions(-) delete mode 100644 .claude/settings.local.json delete mode 100644 Evaluation + Optimization.md delete mode 100644 eval_optimize_analysis.md diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index 43bc8d046..000000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(python -m pip install:*)", - "Bash(where:*)", - "Bash(python:*)", - "Bash(\"E:/code/study/trpc-agent-python/.venv/Scripts/python.exe\" -m pip --version:*)", - "Bash(\"E:/code/study/trpc-agent-python/.venv/Scripts/python.exe\" -m pip install -r E:/code/study/trpc-agent-python/requirements-test.txt)", - "Bash(\"E:/code/study/trpc-agent-python/.venv/Scripts/python.exe\" -c:*)", - "WebSearch", - "Bash(\"E:/code/study/trpc-agent-python/.venv/Scripts/python.exe\" -m pytest tests -q --tb=no -p no:cacheprovider)", - "Bash(tee:*)", - "Bash(powershell:*)", - "Bash(\"E:/code/study/trpc-agent-python/.venv/Scripts/python.exe\" -m pytest:*)", - "Bash(Tee-Object:*)", - "Bash(pytest_eval.log)", - "Bash(\"E:\\code\\study\\trpc-agent-python\\.venv\\Scripts\\python.exe\" -m pytest \"E:\\code\\study\\trpc-agent-python\\examples\\optimization\\eval_optimize_loop\\tests\" -q --tb=line)", - "Bash(E:codestudytrpc-agent-pythonpytest_out.txt)", - "Bash(Get-Content E:codestudytrpc-agent-pythonpytest_out.txt -Tail 100)", - "Bash(\"E:\\code\\study\\trpc-agent-python\\pytest_out.txt\")", - "Bash(\"E:\\code\\study\\trpc-agent-python\\.venv\\Scripts\\python.exe\" -m pytest:*)", - "Bash(\"E:\\code\\study\\trpc-agent-python\\.venv\\Scripts\\python.exe\" -c \"import sys; sys.path.insert(0, r''E:\\code\\study\\trpc-agent-python\\examples\\optimization\\eval_optimize_loop''); from runner import sha256_text, sha256_file; print(sha256_text(''a\\nb\\n''))\")" - ], - "deny": [], - "ask": [] - } -} diff --git a/.gitignore b/.gitignore index 58eb6b48a..0cacf109e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ pyrightconfig.json # spec-workflow tool artifacts .spec-workflow + +.claude/settings.local.json diff --git a/Evaluation + Optimization.md b/Evaluation + Optimization.md deleted file mode 100644 index 546ab211c..000000000 --- a/Evaluation + Optimization.md +++ /dev/null @@ -1,32 +0,0 @@ -背景和价值 -tRPC-Agent 已提供 AgentEvaluator 和 AgentOptimizer 两类关键能力：前者负责把 Agent 行为固化成可回归的评测信号，后者基于评测结果自动搜索更优 prompt、skill 描述或 sub-agent 指令。真实业务中，评测和优化不能割裂：如果评测集质量差，优化器会过拟合；如果优化过程不可审计，改出来的 prompt 即使分数变高也很难进入生产。 -该题要求构建一个“评测 - 失败归因 - prompt 优化 - 回归验证 - 产物审计”的自动闭环。它不是简单跑一次 AgentOptimizer，而是要判断优化是否真的提升、是否牺牲其他指标、是否出现过拟合、是否值得回写源 prompt。 -任务描述 -设计并实现一个可复现的 Evaluation + Optimization pipeline。输入 baseline prompt、训练评测集、验证评测集和优化配置，系统自动运行 baseline 评测、定位失败 case、执行若干轮优化、对候选 prompt 做验证集回归，并输出结构化优化报告和是否接受候选的决策。 -具体要求 -pipeline 至少需要包含以下阶段： -1.Baseline 评测：使用 AgentEvaluator 对训练集和验证集分别打分，记录每条 case 的 metric 分、pass/fail、失败原因和关键轨迹。 -2.失败归因：按失败类型聚类，例如最终回复不匹配、工具调用错误、参数错误、LLM rubric 不达标、知识召回不足、格式不符合要求。 -3.优化执行：使用 AgentOptimizer 或等价扩展机制优化至少一个 TargetPrompt，支持 system prompt、skill prompt、router prompt 中的一种或多种。 -4.候选验证：优化后必须重新跑验证集，并和 baseline 做逐 case 对比，区分新增通过、新增失败、分数提升、分数下降。 -5.接受策略：实现可配置 gate，例如验证集总分提升 ≥ 指定阈值、不能新增 hard fail、关键 case 不能退化、成本不能超过预算。 -6.审计落盘：保存每轮候选 prompt、评测结果、接受/拒绝理由、运行成本、耗时、随机种子或复现实验配置。 -输入输出要求： -●输入包含 train.evalset.json、val.evalset.json、optimizer.json 和 prompt 源文件。 -●输出 optimization_report.json，包含 baseline、candidate、delta、gate decision、失败归因统计。 -●输出 optimization_report.md，用人能读懂的方式说明优化是否值得接受。 -●支持 fake judge / fake model / trace mode，保证没有真实 API Key 时也能跑通核心流程。 -交付物 -●新增示例目录，例如 examples/optimization/eval_optimize_loop/。 -●pipeline 入口脚本、样例 evalset、样例 prompt、优化配置和 README。 -●至少 6 条评测 case：3 条训练、3 条验证，其中需要包含可优化成功、优化无效、优化后退化三类情况。 -●optimization_report.json 示例输出。 -●一份 300 – 500 字方案设计说明，解释失败归因方法、接受策略、防过拟合策略和产物审计方式。 -验收标准 -1.公开提供的 6 条样例 case 必须全部可运行，并生成完整优化报告。 -2.在隐藏样本上，优化接受/拒绝决策准确率 ≥ 80%。 -3.对“验证集退化但训练集提升”的过拟合场景，必须能拒绝候选 prompt。 -4.失败归因分类准确率 ≥ 75%，且每个失败 case 至少能给出一个可解释原因。 -5.fake model / trace mode 下完整 pipeline 耗时 ≤ 3 分钟。 -6.报告必须包含 baseline 分数、candidate 分数、逐 case delta、gate 决策、拒绝或接受理由。 - diff --git a/eval_optimize_analysis.md b/eval_optimize_analysis.md deleted file mode 100644 index 3d02b9423..000000000 --- a/eval_optimize_analysis.md +++ /dev/null @@ -1,671 +0,0 @@ -# Evaluation + Optimization 分析报告 - -> 本报告基于 `trpc_agent_sdk/evaluation/`、`tests/evaluation/`、`examples/optimization/` 的源码与 `pytest_full.log` 的实测数据，回答四个问题： -> 1. AgentEvaluator 与 AgentOptimizer 的相关文件路径及职责 -> 2. 一条从 evalset 输入到评测结果、再到优化器输出的真实调用链 -> 3. evaluation 测试失败的完整 traceback -> 4. #91 新增示例目录应放在哪里 -> -> 本报告**不修改代码、不修复 Windows 平台问题**，仅做架构梳理与决策建议。 - ---- - -## 0. 任务规格回顾（来自 `Evaluation + Optimization.md`） - -构建"评测 → 失败归因 → prompt 优化 → 回归验证 → 产物审计"的自动闭环。输入 `train.evalset.json` / `val.evalset.json` / `optimizer.json` / 源 prompt，输出 `optimization_report.json` / `.md`，要求： - -- 6 条 case：3 训练 + 3 验证，覆盖**可优化成功 / 优化无效 / 优化后退化**三类场景 -- fake judge / fake model / trace mode 下 ≤3 分钟跑通 -- 隐藏样本上接受/拒绝决策准确率 ≥80% -- 对"验证集退化但训练集提升"的过拟合场景必须拒绝 -- 失败归因分类准确率 ≥75% -- 报告必须包含 baseline 分数、candidate 分数、逐 case delta、gate 决策、拒绝/接受理由 - ---- - -## 1. 相关文件路径及职责 - -### 1.1 核心模块（`trpc_agent_sdk/evaluation/`） - -| 文件 | 职责 | -|---|---| -| `__init__.py` | 对外总入口，导出 `AgentEvaluator` / `AgentOptimizer` / `TargetPrompt` / `EvalSet` / `EvalCase` / `EvaluateResult` / `OptimizeResult` 等 ~150 个符号 | -| `_agent_evaluator.py` | **评测入口**。`AgentEvaluator.evaluate()` / `get_executer()` 静态方法；`_EvalExecuter._run()`（L140-231）发现 `.evalset.json` → 逐文件加载 → 调 `evaluate_eval_set()` → 汇总打印 → 若有失败 `raise _EvaluationCasesFailed`（L78-93，`AssertionError` 子类，兼容 CI `except AssertionError`）| -| `_agent_optimizer.py` | **优化器门面**。`AgentOptimizer.optimize()` 类方法（L132-296）：预检算法名 → 加载 `OptimizeConfigFile` → 校验输入（`_validate_inputs` L544-614）→ 从 `OPTIMIZER_REGISTRY` 实例化算法 → 调 `optimizer.run(reporter=...)` → 写 `result.json` / `summary.txt` / `rounds/` / `best_prompts/` / `baseline_prompts/` / `config.snapshot.json` / `run.log`。`_mask_sigint`（L72-109）保证 Ctrl+C 期间产物落盘 | -| `_optimize_evaluator_call.py` | **桥接层**。`run_evaluator()`（L83-136）= 优化器调评测器的标准入口：`AgentEvaluator.get_executer()` → `await executer.evaluate()` → 捕获 `_EvaluationCasesFailed`（业务信号，吞掉）→ 其它异常透传 → `summarize_outcome(result)` 输出 `EvaluationOutcome`（`pass_rate` / `tiebreaker` / `metric_breakdown` / `failed_case_ids`）| -| `_base_optimizer.py` | 抽象基类 `BaseOptimizer`。定义 `__init__` 接收 config / call_agent / target_prompt / train / val 路径；声明抽象 `run(reporter=)`；提供 `resolve_required_thresholds`（L80-106）与 `metrics_meet_thresholds`（L108-123）给 framework-level stop policy 复用 | -| `_optimize_gepa_reflective.py` | **GEPA 反思算法实现**（当前 `OPTIMIZER_REGISTRY` 唯一注册项）。`GepaReflectiveOptimizer.run()`（L431-）→ 读 baseline prompts → 加载 train/val evalset → 构造 `_AgentGEPAAdapter` + `_OptimizeModelCallable` → 在 worker thread 调 `gepa.optimize(...)`（L425-429，`asyncio.to_thread`）；`_build_stop_callbacks`（L294-381）把 `max_metric_calls` / `no_improvement` / `timeout` / `score_threshold` / `max_candidate_proposals` / `max_tracked_candidates` / `optimize.stop` 文件 / framework `required_metrics` 都翻译成 gepa `StopperProtocol` | -| `_optimize_gepa_adapter.py` | **GEPA 协议适配器**。`_AgentGEPAAdapter` 实现 `gepa.core.adapter.GEPAAdapter`：`evaluate(candidate, subsample)` 调 `run_evaluator`；`make_reflective_dataset()`（L13-）把失败 case 渲染成 turn-sliced markdown 喂给 reflection LM（含 PASS/FAIL 行、rubric 子项分、synthesized failure reason） | -| `_optimize_gepa_callback.py` | `_AgentGEPACallback`：gepa 事件 → `RoundRecord` 实时缓冲，是 `_build_optimize_result` 的 round 数据首选来源 | -| `_optimize_model_callable.py` | `_OptimizeModelCallable`：把 `OptimizeModelOptions` 包装成 gepa 可调用的 reflection LM | -| `_optimize_config.py` | 配置 schema：`OptimizeConfigFile`（顶层）/ `EvalConfig`（评测）/ `FrameworkStopConfig`（`stop.required_metrics`）/ `GepaReflectiveAlgo`（算法超参，含 `max_metric_calls` / `max_iterations_without_improvement` / `reflection_minibatch_size` 等）| -| `_optimize_registry.py` | `OPTIMIZER_REGISTRY = OptimizerRegistry()`；`_optimize_registrations.py` 触发 gepa_reflective 注册 | -| `_optimize_reporter.py` | `OptimizeReporter` Rich 面板 + ASCII fallback；`RoundView` / `RunHeader` 数据类 | -| `_optimize_result.py` | 结果 schema：`OptimizeResult`（顶层）+ `RoundRecord`（每轮）+ `StopReason` / `RunStatus` / `FinishReason` Literal 类型 | -| `_target_prompt.py` | **多字段 prompt 注册表**。`TargetPrompt.add_path(name, path)` / `.add_callback(name, read=, write=)`；`read_all()` / `write_all()` 原子（path-backed 用 tmp+`os.replace`，部分失败时回滚）| -| `_eval_set.py` | `EvalSet` Pydantic 模型：`eval_set_id` / `app_name?` / `name?` / `description?` / `eval_cases: list[EvalCase]` | -| `_eval_case.py` | `EvalCase` / `Invocation` / `ConversationScenario` / `StaticConversation`；`EvalModeTrace = "trace"`（不跑 agent，直接评测预录对话）| -| `_eval_config.py` | `EvalConfig`：`criteria` (旧式) 或 `metrics` (新式) + `num_runs` | -| `_eval_metrics.py` | `EvalMetric` / `EvalStatus(PASSED\|FAILED)` / `PrebuiltMetrics` 枚举（`final_response_avg_score` 等）| -| `_eval_result.py` | `EvalCaseResult` / `EvalSetAggregateResult` / `EvaluateResult` / `EvalMetricResult`（含 score/threshold/eval_status/details）| -| `_eval_service_base.py` | `BaseEvalService` / `InferenceRequest` / `EvaluateRequest` / `InferenceConfig` / `EvaluateConfig` | -| `_local_eval_service.py` | `LocalEvalService`：本地驱动 agent 推理 + 评测 | -| `_remote_eval_service.py` | `RemoteEvalService` + `CallAgent = Callable[[str], Awaitable[str]]`：黑盒模式，业务回调驱动 agent | -| `_final_response_evaluator.py` | `FinalResponseEvaluator`：精确 / 包含 / 正则 / JSON 匹配 | -| `_trajectory_evaluator.py` | `TrajectoryEvaluator`：工具调用轨迹评分（需 session traces）| -| `_rouge_evaluator.py` | `RougeEvaluator`：ROUGE 分 | -| `_llm_criterion.py` / `_llm_evaluator.py` / `_llm_judge.py` | LLM-as-judge 三件套：rubric 打分 / 知识召回 / 最终回复质量，支持 multi-model judges / weighted aggregators | -| `_eval_callbacks.py` | `Callbacks`（生命周期钩子）+ `CallbacksRunner` | -| `_local_eval_sets_manager.py` / `_in_memory_eval_sets_manager.py` / `_local_eval_set_results_manager.py` | evalset 持久化与历史结果管理 | - -### 1.2 测试目录（`tests/evaluation/`，53 个文件） - -按职责分组： - -- **AgentEvaluator 主流程**：`test_agent_evaluator.py`、`test_agent_evaluator_call_agent.py` -- **AgentOptimizer 门面**：`test_agent_optimizer.py` -- **桥接层**：`test_optimize_evaluator_call.py` -- **GEPA 四件套**：`test_optimize_gepa_adapter.py`、`test_optimize_gepa_callback.py`、`test_optimize_gepa_e2e.py`、`test_optimize_gepa_reflective.py` -- **端到端**：`test_optimize_quickstart_example.py`（以 `examples/optimization/quickstart/` 为素材） -- **组件级**：`test_optimize_config.py`、`test_optimize_result.py`、`test_optimize_reporter.py`、`test_optimize_registry.py`、`test_optimize_model_callable.py`、`test_optimize_model_options.py`、`test_optimize_metric_info.py` -- **TargetPrompt**：`test_target_prompt.py` -- **数据模型**：`test_eval_case.py`、`test_eval_set.py`、`test_eval_config.py`、`test_eval_metrics.py`、`test_eval_pass.py`、`test_eval_result.py` -- **回调**：`test_eval_callbacks.py`、`test_eval_callbacks_ext.py` -- **基础设施**：`test_eval_session_service.py`、`test_eval_service_base.py`、`test_local_eval_sets_manager.py`、`test_in_memory_eval_sets_manager.py`、`test_remote_eval_service.py` -- **Evaluator 实现**：`test_final_response_evaluator.py`、`test_trajectory_evaluator.py`、`test_rouge_evaluator.py`、`test_llm_judge*.py`、`test_llm_criterion.py`、`test_llm_evaluator_registry.py` - -### 1.3 现有 examples 目录（`examples/optimization/`） - -``` -examples/optimization/ -├── quickstart/ # 入门示例（小学算术题，2 个 prompt 文件） -├── advanced_strategies/ # 进阶策略（pareto / current_best / merge） -├── blackbox_cli/ # CLI 黑盒 agent 接入 -├── ci_integration/ # pytest + AgentOptimizer 闭环 -├── http_service/ # HTTP 服务形态 -├── multi_agent_pipeline/ # 多 agent 编排 -├── multi_metric_with_judges/ # 多 metric + 独立 judge model -├── remote_prompt_store/ # 远端 prompt 源（Redis / HTTP） -└── slo_runtime_control/ # SLO / budget / kill switch -``` - -每个子目录都有 `README.md` + `optimizer.json` + 可独立运行的入口脚本 + `train.evalset.json` + `val.evalset.json`。 - ---- - -## 2. 真实调用链：evalset → 评测结果 → 优化器输出 - -以 `examples/optimization/quickstart/run_optimization.py` 为例，从 `asyncio.run(main())` 到产物落盘的完整链路： - -### 阶段 A：业务组装（`run_optimization.py`） - -```python -TargetPrompt().add_path("system_prompt", SYSTEM_PROMPT_PATH) - .add_path("skill", SKILL_PATH) -await AgentOptimizer.optimize( - config_path="optimizer.json", - call_agent=call_agent, # 业务回调 - target_prompt=target, - train_dataset_path="train.evalset.json", - validation_dataset_path="val.evalset.json", - output_dir="runs/", - update_source=False, -) -``` - -### 阶段 B：`AgentOptimizer.optimize` 门面（`_agent_optimizer.py:132-296`） - -1. `_precheck_algorithm_name(config_path)` L200 → 读 raw JSON，查 `OPTIMIZER_REGISTRY`，未注册则 fail-fast -2. `load_optimize_config(config_path)` L201 → pydantic 校验，输出 `OptimizeConfigFile` -3. `_validate_inputs(...)` L202-209 → async check / train≠val / disallowed metrics in call_agent mode / use_merge≥2 fields -4. `os.makedirs(output_dir)` L210 -5. `OPTIMIZER_REGISTRY.get("gepa_reflective")` L213 → `GepaReflectiveOptimizer` -6. 实例化 optimizer L214-224，注入 config / call_agent / target_prompt / train / val / output_dir -7. `create_reporter(verbose=1, stream=sys.stdout)` L226 -8. `baseline_snapshot = await target_prompt.read_all()` L227（**关键：快照基线 prompt，供 finally 回滚**） -9. `_build_run_header(...)` L228-235 → 读 train/val evalset 数 case 个数 -10. `reporter.run_started(header)` L236 -11. `try: result = await optimizer.run(reporter=reporter)` L251 -12. `if update_source and result.status == "SUCCEEDED": await target_prompt.write_all(result.best_prompts)` L256-264 -13. `finally:` L272 — 若 cleanup 未完成，回滚到 `baseline_snapshot`；调 `_persist_artifacts` 写产物；`reporter.run_finished/failed` - -### 阶段 C：`GepaReflectiveOptimizer.run`（`_optimize_gepa_reflective.py:431-`） - -1. `baseline_prompts = await self.target_prompt.read_all()` L441 -2. `_load_evalset_cases(train_path)` + `_load_evalset_cases(val_path)` L445-446 — `Path.read_text` + `EvalSet.model_validate_json` → `list[EvalCase]` -3. 构造 `_AgentGEPAAdapter(target_prompt, eval_config, call_agent, callbacks, num_runs, case_parallelism, reflection_history_top_k)` L457-465 -4. 构造 `_OptimizeModelCallable(algo.reflection_lm)` L466 — 包装 reflection LM -5. 调 `_run_with_adapter(...)` L469-481，`finally: adapter.close()` - -### 阶段 D：`_run_with_adapter` → `gepa.optimize(...)` - -- `_build_stop_callbacks(algo, stop_config, metric_thresholds, output_dir)` L294-381 把配置翻译成 gepa stopper 列表 -- `await self._call_gepa_optimize(**kwargs)` L425-429 → `asyncio.to_thread(gepa_optimize, **kwargs)` 把同步 GEPA 主循环扔到 worker thread，不阻塞 surrounding event loop -- GEPA 主循环每轮： - 1. `module_selector` 选要改写的 prompt 字段（`round_robin` 在 system_prompt ↔ skill 间交替） - 2. 从 trainset 抽 `reflection_minibatch_size` 条 case - 3. `adapter.evaluate(parent, subsample)` → **下到阶段 E** - 4. `adapter.make_reflective_dataset(failed_cases)` → 渲染 markdown 反思素材 - 5. reflection LM 生成新候选 prompt → `target_prompt.write_all(candidate)` 写回磁盘 - 6. `adapter.evaluate(candidate, full_valset)` → **下到阶段 E** - 7. Pareto 前沿比较，决定接受 / 拒绝 - 8. `_AgentGEPACallback` 实时缓冲 `RoundRecord` - 9. stop callbacks 投票，命中则退出 - -### 阶段 E：`_AgentGEPAAdapter.evaluate` → `run_evaluator`（`_optimize_evaluator_call.py:83-136`） - -```python -executer = AgentEvaluator.get_executer( - eval_dataset_path, # 临时文件 or val.evalset.json - call_agent=call_agent, # 业务回调 - callbacks=callbacks, - num_runs=num_runs, - print_detailed_results=False, # 优化器静默评测 - print_summary_report=False, - eval_metrics_file_path_or_dir=eval_metrics_path, - case_parallelism=case_parallelism, -) -try: - await executer.evaluate() -except _EvaluationCasesFailed: - pass # 业务信号，吞掉 -result = executer.get_result() -return summarize_outcome(result) # → EvaluationOutcome -``` - -### 阶段 F：`_EvalExecuter._run`（`_agent_evaluator.py:140-231`） - -1. `_resolve_shared_config(eval_metrics_file_path_or_dir)` L156 -2. 遍历 `.evalset.json` 文件 L158-165 -3. 对每个文件：`_load_eval_set_from_file` L187（支持 `file.json:case_id` 选择子集）→ `EvalSet.model_validate_json` -4. `evaluate_eval_set(eval_set, call_agent=call_agent, eval_config=eval_config, ...)` L188-202 -5. 收集 failures，构造 `EvalSetAggregateResult` -6. `_RESULT_HANDLER.print_evaluation_report` 打印 -7. 若 `all_failures`：`raise _EvaluationCasesFailed(json.dumps(...))` L231 — **此时 `self._result` 已被赋值**，优化器能拿到 - -### 阶段 G：`AgentEvaluator.evaluate_eval_set`（`_agent_evaluator.py:470-586`） - -1. 校验：`call_agent` / `agent_module` / `runner` 互斥；trace_only 模式可省 agent -2. `_get_eval_results_by_eval_id(...)` L540-552 — **下到阶段 H** -3. 后处理：`_get_eval_metric_results_with_invocation` L561 翻转 grouping -4. `_RESULT_HANDLER.process_metrics_and_get_failures` L563-570 提取失败 case -5. `build_summary` + `build_evaluation_result_lines` 输出表格 -6. 返回 `(failed_summary, details_lines, result_lines, eval_results_by_eval_id)` - -### 阶段 H：`_get_eval_results_by_eval_id`（`_agent_evaluator.py:815-915`） - -1. `InMemoryEvalSetsManager` 创建并填充 evalset -2. 选择 `RemoteEvalService`（call_agent 模式）或 `LocalEvalService` -3. `InferenceConfig(parallelism=case_parallelism)` 构造 N 份 `InferenceRequest`（N=num_runs） -4. `async for inference_result in eval_service.perform_inference(...)` L893 — 调 `call_agent(query)` 拿 agent 回复 -5. `EvaluateConfig(eval_metrics=eval_metrics)` + `EvaluateRequest(inference_results=inference_results)` L901-906 -6. `async for eval_result in eval_service.evaluate(evaluate_request)` L908 — 跑每个 metric 的 evaluator（`FinalResponseEvaluator` / `LLMRubricResponseEvaluator` 等） -7. 按 eval_id 聚合，返回 `dict[str, list[EvalCaseResult]]` - -### 阶段 I：结果回传与产物落盘 - -- 阶段 H 结果 → 阶段 G `eval_results_by_eval_id` → 阶段 F `EvaluateResult(results_by_eval_set_id)` → 阶段 E `summarize_outcome(result)` 输出 `EvaluationOutcome(pass_rate, tiebreaker, metric_breakdown, failed_case_ids)` -- GEPA 用 `EvaluationOutcome.pass_rate` 比较 Pareto 前沿 -- 主循环结束后，阶段 C `_build_optimize_result`（`_optimize_gepa_reflective.py:119-258`）从 `gepa_result` + `callback_rounds` 组装 `OptimizeResult` -- 阶段 B `_persist_artifacts`（L374-437）写： - - `output_dir/result.json` — 完整 `OptimizeResult.model_dump_json` - - `output_dir/summary.txt` — 人类可读摘要 - - `output_dir/rounds/round_NNN.json` — 每轮 `RoundRecord` - - `output_dir/baseline_prompts/.md` - - `output_dir/best_prompts/.md` - - `output_dir/config.snapshot.json` - - `output_dir/run.log` - -### 调用链时序图（一图速览） - -``` -run_optimization.py - │ - ▼ -AgentOptimizer.optimize(config_path, call_agent, target_prompt, train_path, val_path, output_dir) - │ - ├─ load_optimize_config ──► OptimizeConfigFile - ├─ _validate_inputs - ├─ target_prompt.read_all ──► baseline_snapshot - │ - ├─ GepaReflectiveOptimizer(config, call_agent, target_prompt, ...).run(reporter) - │ │ - │ ├─ _load_evalset_cases(train_path) + _load_evalset_cases(val_path) - │ ├─ _AgentGEPAAdapter(target_prompt, eval_config, call_agent, ...) - │ ├─ _OptimizeModelCallable(algo.reflection_lm) - │ │ - │ └─ asyncio.to_thread(gepa.optimize, ...) ◄── GEPA 主循环（每轮） - │ │ - │ ├─ module_selector 选字段 - │ ├─ adapter.evaluate(parent, train_subsample) - │ │ │ - │ │ └─ run_evaluator ──► AgentEvaluator.get_executer - │ │ │ - │ │ └─ _EvalExecuter._run - │ │ │ - │ │ ├─ _load_eval_set_from_file ──► EvalSet - │ │ ├─ evaluate_eval_set - │ │ │ │ - │ │ │ └─ _get_eval_results_by_eval_id - │ │ │ ├─ perform_inference (call_agent) - │ │ │ └─ eval_service.evaluate (metric evaluators) - │ │ │ └─► EvalCaseResult[] - │ │ │ - │ │ └─ (failure) raise _EvaluationCasesFailed - │ │ - │ ├─ make_reflective_dataset(failed_cases) ──► markdown - │ ├─ reflection_lm(markdown) ──► candidate prompt - │ ├─ target_prompt.write_all(candidate) ◄── 落盘新候选 - │ ├─ adapter.evaluate(candidate, full_valset) - │ ├─ Pareto 比较 → accept / reject - │ └─ stop_callbacks 投票 - │ - ├─ (optional) target_prompt.write_all(result.best_prompts) ◄── update_source=True - │ - └─ _persist_artifacts - ├─ result.json / summary.txt - ├─ rounds/round_NNN.json - ├─ baseline_prompts/.md - ├─ best_prompts/.md - ├─ config.snapshot.json - └─ run.log -``` - ---- - -## 3. evaluation 测试失败的完整 traceback - -`tests/evaluation` 共 **2 条**测试失败（来自 `pytest_full.log` L728-744 与 L1047+；完整 suite 9290 通过 / 118 失败，evaluation 模块占 2 条）。 - -### 失败 1：`test_eval_executer_raises_evaluation_cases_failed_on_case_failure` - -``` -tests\evaluation\test_optimize_evaluator_call.py:524: in test_eval_executer_raises_evaluation_cases_failed_on_case_failure - await executer.evaluate() -trpc_agent_sdk\evaluation\_agent_evaluator.py:244: in evaluate - await self._ensure_run() -trpc_agent_sdk\evaluation\_agent_evaluator.py:236: in _ensure_run - await self._task -trpc_agent_sdk\evaluation\_agent_evaluator.py:187: in _run - eval_set = AgentEvaluator._load_eval_set_from_file(test_file, eval_config) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -trpc_agent_sdk\evaluation\_agent_evaluator.py:673: in _load_eval_set_from_file - raise FileNotFoundError(f"Eval set file not found: {actual_file_path}") -E FileNotFoundError: Eval set file not found: C -``` - -**根因**：Windows 路径解析 bug。测试用 `tmp_path / "tiny.evalset.json"` 生成形如 `C:\Users\...\tiny.evalset.json` 的绝对路径。`_load_eval_set_from_file` 在 `_agent_evaluator.py:667` 检查 `if ":" in eval_set_file:` 时假定 `:` 是 ADK 风格的 case 选择分隔符（`file.json:case_id`），用 `split(":", 1)` 取 parts[0] 作为文件路径。Windows 盘符 `C:` 被错误切分，`actual_file_path` 变成 `"C"`，文件查找失败。 - -**修复方向**（不在本次任务范围）：用 `os.path.splitdrive()` 先剥离盘符再查 `:`，或限制分隔符只匹配 `.json:` 之后的字符。 - -### 失败 2：`test_quickstart_real_gepa_loop_reuses_single_event_loop_across_rounds` - -``` -tests\evaluation\test_optimize_quickstart_example.py:475: in test_quickstart_real_gepa_loop_reuses_single_event_loop_across_rounds - assert len(seen_loop_ids) >= 2, ( -E AssertionError: Expected real gepa main loop to call call_agent more than once; saw 0 call(s). -E assert 0 >= 2 -E + where 0 = len([]) -``` - -**根因**：这个测试 monkeypatch `_OptimizeModelCallable.__call__` 用 fake reflection LM，设了 `max_metric_calls=6` / `max_iterations_without_improvement=1`，期待 gepa 主循环至少跑 2 轮（baseline + round 1）。但 `seen_loop_ids` 为空 → call_agent 一次都没被调到。可能原因： - -- 该测试依赖 `examples/optimization/quickstart/optimizer.json` 中的 `${TRPC_AGENT_MODEL_NAME}` 等环境变量，未设置时 gepa 在 baseline 评估阶段就抛异常退出，没有进入第一轮反思 -- 也可能是 `asyncio.to_thread(gepa_optimize, ...)` 在 worker thread 内调用 `call_agent`（async），而 monkeypatch 的 `stub_call_agent` 没有跨线程传播；或 gepa 的 asyncio event loop 配置在 Windows ProactorEventLoop 下行为不同 - -### 测试运行说明 - -本次报告未能成功跑出 `pytest tests/evaluation -q --tb=short` 的新日志：用 PowerShell `Tee-Object` 时不实时刷新；改用 bash `> pytest_eval.log 2>&1` 重定向后，进程运行 16+ 分钟仍未结束（远超完整 suite 的 3.5 分钟），疑似 Windows 下 gepa / asyncio 阻塞或缺 API key 卡住。所有失败数据已从既有的 `pytest_full.log`（完整 suite 运行结果）提取。 - ---- - -## 4. #91 新增示例目录建议 - -任务规格明确要求 `examples/optimization/eval_optimize_loop/`。基于现有 9 个 example 的命名约定与目录结构，**强烈建议直接按规格放在**： - -``` -examples/optimization/eval_optimize_loop/ -├── README.md # 300–500 字方案设计说明 + 使用指引 -├── pipeline.py # 入口脚本（baseline 评测 → 失败归因 → 优化 → 回归 → 决策 → 落盘） -├── optimizer.json # GEPA + metric 配置（fake model + trace mode 默认开） -├── train.evalset.json # 3 条训练 case -├── val.evalset.json # 3 条验证 case（覆盖可优化成功 / 优化无效 / 优化退化的三类场景） -├── agent/ -│ ├── __init__.py -│ ├── agent.py # create_agent() 工厂，重读 prompt -│ ├── config.py # 环境变量读取，支持 FAKE_MODEL=1 短路 -│ └── prompts/ -│ ├── system.md # 被优化的 system prompt -│ └── skill.md # 被优化的 skill prompt -├── attribution/ # 失败归因模块（规格第 2 阶段） -│ └── cluster.py # 按失败类型聚类（reply_mismatch / tool_call_error / param_error / rubric_fail / knowledge_fail / format_fail） -├── gate/ # 接受策略（规格第 5 阶段） -│ └── accept_policy.py # 可配置 gate：val_improvement ≥ threshold / no_new_hard_fail / critical_case_no_regression / cost ≤ budget -├── audit/ # 审计落盘（规格第 6 阶段） -│ └── report.py # 生成 optimization_report.json + .md -├── fake/ # fake judge / fake model / trace mode（规格要求） -│ ├── fake_model.py -│ ├── fake_judge.py -│ └── traces/ # 预录对话，trace mode 用 -├── runs/ # 输出根目录（每次运行写到时间戳子目录） -└── optimization_report.json.example # 示例输出 -``` - -### 选此位置的理由 - -1. **目录命名与规格 1:1 对齐**：任务规格交付物明确写 `examples/optimization/eval_optimize_loop/`，与现有 9 个 example 同层级，便于索引 -2. **现有目录覆盖场景互补**：现有 9 个 example 都只展示"优化器本身怎么用"，没有任何一个演示"评测 → 归因 → 优化 → 回归 → 决策 → 审计"端到端闭环。`ci_integration/` 最接近但只覆盖 PR 守门 + 夜间优化两段，缺失败归因和 gate decision -3. **依赖现有模块最小**：`pipeline.py` 直接复用 `AgentEvaluator.evaluate` 跑 baseline + 回归，`AgentOptimizer.optimize` 跑优化轮，新增的 `attribution/` `gate/` `audit/` 是薄包装层 -4. **fake mode 可对照 quickstart 测试**：trace mode + fake judge 在 `optimizer.json` 里通过 `"judge_model": {"model_name": "fake-judge"}` 触发，`fake/fake_model.py` 提供 `async def fake_call_agent(query)` 直接返回 `val.evalset.json` 中预录的 `final_response`，满足"无 API Key 跑通 ≤3min"的验收标准 - -### 与现有模块的集成点 - -- `pipeline.py` 调 `AgentEvaluator.get_executer(val_path, call_agent=fake_call_agent, eval_metrics_path="optimizer.json", print_summary_report=False)` 跑 baseline，拿 `EvaluateResult` -- `attribution/cluster.py` 遍历 `EvaluateResult.results_by_eval_set_id[...].eval_results_by_eval_id` 的每个 `EvalCaseResult`，按 `eval_metric_results[*].metric_name` 与 `details.reason` 分类： - - `final_response_avg_score` fail → `"reply_mismatch"` 或 `"format_fail"` - - `tool_trajectory_avg_score` fail → `"tool_call_error"` 或 `"param_error"` - - `llm_rubric_response` fail → `"rubric_fail"` - - `llm_rubric_knowledge_recall` fail → `"knowledge_fail"` -- 调 `AgentOptimizer.optimize(config_path="optimizer.json", call_agent=fake_call_agent, target_prompt=target, train_path, val_path, output_dir)` 跑优化 -- 拿到 `OptimizeResult` 后，`audit/report.py` 调 `AgentEvaluator.get_executer(val_path, call_agent=...)` 用 `result.best_prompts`（先 `target_prompt.write_all(result.best_prompts)`）再跑一次 val 评测做候选验证 -- `gate/accept_policy.py` 比较 baseline 与 candidate 的 `EvaluationOutcome`，输出 `{"accepted": bool, "reason": str}`，写入 `optimization_report.json` - -### 6 条 case 的设计要点（满足规格三类场景） - -| case_id | 集合 | 场景类型 | 设计方式 | -|---|---|---|---| -| `t_opt_success` | train | 可优化成功 | baseline prompt 漏掉"输出答案前必须给出推理步骤"，导致 final_response_avg_score fail；reflection LM 补上即可通过 | -| `t_opt_noop` | train | 优化无效 | case 本身模糊（缺少关键数字），任何 prompt 都无法让 agent 给出匹配答案；reflection LM 反复改写无法提升 | -| `t_opt_regress` | train | 优化后退化的诱因 | 训练集包含一个特殊格式偏好（例如要求 "答：" 而非 "答案："），改写 prompt 满足此 case 会牺牲验证集 | -| `v_opt_success` | val | 验证可优化成功 | 与 `t_opt_success` 同类型，验证 baseline→candidate 提升 | -| `v_opt_noop` | val | 验证优化无效 | 同 `t_opt_noop`，确认 train 与 val 一致地无提升 | -| `v_opt_regress` | val | 验证退化 | 与 `t_opt_regress` 的格式偏好冲突，candidate 通过 train 但在 val 退化；gate policy 必须拒绝 | - -接受策略 gate 至少配置： -- `val_total_score_improvement >= 0.02`（防止噪声波动被误判为提升） -- `no_new_hard_fail = True`（candidate 不能让任何 baseline PASS 的 case 变 FAIL） -- `critical_case_ids = ["v_opt_regress"]` 不能退化（直接命中规格验收标准 3） -- `cost_budget_usd = 5.0`（防止 fake mode 失灵时 LLM 失控） - ---- - -## 附录：关键文件行号速查表 - -| 关注点 | 文件:行号 | -|---|---| -| AgentEvaluator 主入口 | `trpc_agent_sdk/evaluation/_agent_evaluator.py:255` | -| AgentEvaluator.evaluate 静态方法 | `_agent_evaluator.py:308` | -| AgentEvaluator.get_executer | `_agent_evaluator.py:363` | -| `_EvalExecuter._run` | `_agent_evaluator.py:140` | -| `_EvaluationCasesFailed` 抛出点 | `_agent_evaluator.py:231` | -| Windows 盘符 bug 现场 | `_agent_evaluator.py:667`（`if ":" in eval_set_file`）| -| `evaluate_eval_set` | `_agent_evaluator.py:470` | -| `_get_eval_results_by_eval_id` | `_agent_evaluator.py:815` | -| AgentOptimizer 主入口 | `trpc_agent_sdk/evaluation/_agent_optimizer.py:112` | -| `AgentOptimizer.optimize` | `_agent_optimizer.py:132` | -| `_validate_inputs` | `_agent_optimizer.py:544` | -| `_persist_artifacts` | `_agent_optimizer.py:374` | -| `_mask_sigint` 上下文管理器 | `_agent_optimizer.py:72` | -| `run_evaluator` 桥接函数 | `trpc_agent_sdk/evaluation/_optimize_evaluator_call.py:83` | -| `summarize_outcome` | `_optimize_evaluator_call.py:44` | -| `EvaluationOutcome` 数据类 | `_optimize_evaluator_call.py:23` | -| `GepaReflectiveOptimizer.run` | `trpc_agent_sdk/evaluation/_optimize_gepa_reflective.py:431` | -| `_build_stop_callbacks` | `_optimize_gepa_reflective.py:294` | -| `_build_optimize_result` | `_optimize_gepa_reflective.py:119` | -| `_AgentGEPAAdapter` | `trpc_agent_sdk/evaluation/_optimize_gepa_adapter.py` | -| `OptimizeResult` schema | `trpc_agent_sdk/evaluation/_optimize_result.py:167` | -| `RoundRecord` schema | `_optimize_result.py:43` | -| `TargetPrompt` | `trpc_agent_sdk/evaluation/_target_prompt.py:63` | -| `EvalSet` schema | `trpc_agent_sdk/evaluation/_eval_set.py:33` | -| quickstart 入口 | `examples/optimization/quickstart/run_optimization.py` | -| quickstart agent 工厂 | `examples/optimization/quickstart/agent/agent.py` | -| quickstart optimizer.json | `examples/optimization/quickstart/optimizer.json` | -| quickstart train.evalset.json | `examples/optimization/quickstart/train.evalset.json`（5 条小学算术题）| -| quickstart val.evalset.json | `examples/optimization/quickstart/val.evalset.json`（3 条小学算术题）| - ---- - -## 5. 现有能力盘点（Issue #91 视角） - -> 本节回答："SDK 已经提供了哪些能力可以直接复用，不必重造？"——是对第 1、2 节的能力视角重组，按 Issue #91 要求的"评测 / 归因 / 优化 / 验证 / 决策 / 审计"六个阶段归类。 - -### 5.1 评测阶段（AgentEvaluator） - -| 能力 | 关键文件 / 行号 | 是否已就绪 | -|---|---|---| -| 评测入口（有返回值版本） | `_agent_evaluator.py:363` `AgentEvaluator.get_executer(...)` → `_EvalExecuter` → `await executer.evaluate()` → `executer.get_result(): EvaluateResult` | ✅ | -| 评测入口（断言版本，CI 友好） | `_agent_evaluator.py:308` `AgentEvaluator.evaluate(...)`，无返回值，全失败抛 `_EvaluationCasesFailed`（L78-93，`AssertionError` 子类） | ✅ | -| EvalSet JSON schema | `_eval_set.py:33` `EvalSet`；`_eval_case.py:170-239` `EvalCase` / `Invocation` / `IntermediateData` | ✅ | -| trace mode（跳过 agent 直接打分预录对话） | `_eval_case.py:152` `EvalModeTrace = "trace"`；`_agent_evaluator.py:300-530` `_is_trace_only` 分支 | ✅ | -| 7 个内置 metric | `_eval_metrics.py:45-66` `PrebuiltMetrics`：`final_response_avg_score` / `tool_trajectory_avg_score` / `response_match_score` / `response_evaluation_score` / `llm_final_response` / `llm_rubric_response` / `llm_rubric_knowledge_recall` | ✅ | -| Pass/Fail 判定 | `_eval_metrics.py:38-42` `EvalStatus`；每个 evaluator 内部 `_get_eval_status(score) = PASSED if score >= threshold else FAILED` | ✅ | -| Case 级失败原因字段 | `_eval_result.py:185` `EvalCaseResult.error_message`；`:108-116` `EvalMetricResultDetails.reason`；`:77-81` `PerInvocationResult.reason/rubric_scores`；`:53` `NamedScoreResult.reason` | ✅ | -| 工具调用轨迹字段 | `_eval_case.py:121` `Invocation.intermediate_data.tool_uses/tool_responses`；`:242-289` `get_all_tool_calls` / `get_all_tool_responses` | ✅ | - -### 5.2 归因阶段（部分已就绪） - -| 能力 | 现状 | 缺口 | -|---|---|---| -| 字段级归因（GEPA 已做） | `RoundRecord.optimized_field_names`（`_optimize_result.py:88-91`）；`RoundRecord.per_field_diagnosis`（`:96-99`）来自反思 LM | — | -| 反思 LM 自动看失败 case | GEPA 算法内置（`_optimize_gepa_adapter.py:657-721` `make_reflective_dataset`），`reflection_minibatch_size` 控制批大小 | — | -| **case 级失败类型聚类（6 类）** | 字段都在，但**没有任何示例代码**做"按失败类型聚类"（回复不匹配 / 工具调用错 / 参数错 / rubric 不达标 / 召回不足 / 格式不符） | ❌ 需在 `eval_optimize_loop/` 新增 | - -### 5.3 优化阶段（AgentOptimizer） - -| 能力 | 关键文件 / 行号 | 是否已就绪 | -|---|---|---| -| 优化入口 | `_agent_optimizer.py:132-296` `AgentOptimizer.optimize(*, config_path, call_agent, target_prompt, train_dataset_path, validation_dataset_path, output_dir, update_source=False, ...)` | ✅ | -| GEPA 反思算法（当前唯一注册） | `_optimize_gepa_reflective.py:408` `GepaReflectiveOptimizer`；adapter 在 `_optimize_gepa_adapter.py:505`；callback 在 `_optimize_gepa_callback.py:81` | ✅ | -| 配置 schema | `_optimize_config.py:226-243` `OptimizeConfigFile`；`GepaReflectiveAlgo` 含 `max_metric_calls` / `max_iterations_without_improvement` / `timeout_seconds` / `score_threshold` / `max_candidate_proposals` / `max_tracked_candidates` 等 6 类停止条件 | ✅ | -| 候选选择策略 | `_optimize_config.py` `candidate_selection_strategy ∈ {pareto, current_best, epsilon_greedy, top_k_pareto}`；`frontier_type ∈ {instance, objective, hybrid, cartesian}` | ✅ | -| 多字段 prompt 优化 | `module_selector ∈ {round_robin, all, random}`；`use_merge=true` 触发字段合并（要求注册 ≥2 字段，`_agent_optimizer.py:602-614`） | ✅ | -| 框架层接受/早停 | `_optimize_gepa_reflective.py:89-116, 294-381` `_RequiredMetricsAboveThresholdStopper` + 6 类 stopper；`optimize.stop.required_metrics` ∈ `"all" / list / []` | ✅ | - -### 5.4 验证阶段（部分已就绪） - -| 能力 | 现状 | 缺口 | -|---|---|---| -| train / val 物理隔离校验 | `_agent_optimizer.py:544-614` `_validate_inputs` 强制 train≠val | ✅ | -| 每轮全量 val 评估 | GEPA 算法内置（`_optimize_gepa_adapter.py:594-655`） | ✅ | -| 独立 val 评测（脱离 optimizer） | `AgentEvaluator.get_executer(val_path, ...)` 即可，参考 `examples/optimization/ci_integration/tests/test_agent_quality.py:48-62` | ✅ | -| **逐 case 跨 run 回归 diff** | `examples/optimization/advanced_strategies/compare.py:52-96` 只对比汇总指标，**不做 case-level diff** | ❌ 需新增 | - -### 5.5 接受决策阶段（部分已就绪） - -| 能力 | 现状 | 缺口 | -|---|---|---| -| Pareto 前沿 / 早停 / required_metrics 门禁 | 见 5.3 | ✅ | -| **业务级可配置 gate**（"总分提升≥N"、"hard fail 数不增"、"关键 case 不退化"、"成本≤预算"） | **完全不存在**，SDK 的 stop policy 只控制"早停"，不等价于业务"是否回写" | ❌ 需新增 | - -### 5.6 审计阶段（部分已就绪） - -| 能力 | 现状 | 缺口 | -|---|---|---| -| `result.json` / `summary.txt` / `rounds/round_NNN.json` | `_agent_optimizer.py:374-491` `_persist_artifacts` | ✅ | -| `baseline_prompts/` + `best_prompts/` 双快照 | 同上 | ✅ | -| 时间戳子目录隔离 | 所有 example 入口都做了，例如 `quickstart/run_optimization.py:147-148` | ✅ | -| `config.snapshot.json` 复现配置 | 同上 | ✅ | -| **`optimization_report.{json,md}`（含 baseline / candidate / 逐 case delta / gate decision / 失败归因 / 成本 / 耗时 / 随机种子）** | **完全不存在** | ❌ 需新增 | -| **操作员级审计日志（谁/何时/改了哪个 prompt）** | **完全不存在**；`update_source=True` 直接覆盖源文件 | ❌ 需新增 | - -### 5.7 fake judge / fake model / trace mode - -> Issue 验收标准 #5 要求"fake model / trace mode 下完整 pipeline 耗时 ≤ 3 分钟"。 - -| 路径 | 现状 | 实现策略 | -|---|---|---| -| **trace mode（原生）** | ✅ `_eval_case.py:152` + `_agent_evaluator.py:300-530`；示例 `examples/evaluation/trace_mode/` | **首选**：在 `evalset` 中给 `eval_mode="trace"` + `actual_conversation`，评测阶段完全跳过 agent 推理，Windows 上稳定 ≤3 min | -| **fake call_agent（黑盒）** | ✅ `_agent_optimizer.py:547-551`、`_optimize_evaluator_call.py:83-136`，`CallAgent = async (query: str) -> str` 协议，业务自定义 | 次选：在 `fake/fake_model.py` 实现 async 函数，按规则 / 字典返回字符串 | -| **fake judge（自定义 scorer）** | ✅ `_llm_evaluator.py:103-178` `LLM_EVALUATOR_REGISTRY` 支持注册自定义 `response_scorer` / `models_aggregator`，scorer 内可不调 LLM | 在 `fake/fake_judge.py` 注册假 scorer | -| **fake reflection LM** | ✅ `_optimize_model_callable.py:210-309` 可被 monkeypatch；`OptimizeModelOptions.base_url`（`_optimize_model_options.py:24`）可指向本地 fake server；`ModelRegistry` 也支持自注册 provider | 在 `fake/fake_reflection.py` 包一个符合 gepa `LanguageModel` 协议的同步 callable | -| **框架内置 fake 实现** | ❌ 不存在 | **必须新增** `fake/` 子目录，落地上述四个 fake 路径 | - ---- - -## 6. 需要新增能力清单（仅在 `examples/optimization/eval_optimize_loop/`） - -下表把第 5 节标 ❌ 的缺口集中起来，给出建议落点。 - -| # | 新增能力 | 落点（相对 `eval_optimize_loop/`） | 输入 | 输出 | -|---|---|---|---|---| -| 1 | 失败归因分类器 | `attribution/cluster.py` | `EvalCaseResult[]` + `EvalCase.conversation[*].intermediate_data` | `{category: str, case_ids: list[str], sample_reason: str}[]`，category ∈ {reply_mismatch, tool_call_error, param_error, rubric_fail, knowledge_fail, format_fail} | -| 2 | 逐 case 跨 run regression diff | `regression/diff.py` | baseline `EvaluateResult` + candidate `EvaluateResult` | `{newly_passed, newly_failed, score_up, score_down: list[{eval_id, baseline_score, candidate_score, delta}]}` | -| 3 | 可配置接受 gate | `gate/accept_policy.py` | gate config（JSON）+ regression diff + cost + critical_case_ids | `{accepted: bool, reasons: list[str], violated_rules: list[str]}` | -| 4 | 双格式优化报告 | `audit/report_builder.py` | baseline / candidate 评测结果 + diff + 归因 + gate 决策 + cost + duration + seed | `optimization_report.json` + `optimization_report.md` | -| 5 | 审计落盘增强 | 同 #4，加 `audit` 节 | seed / total_cost / duration / accepted / reasons / repro_config_hash | 写入 `optimization_report.json` 顶层 | -| 6 | fake 实现四件套 | `fake/fake_model.py` / `fake_judge.py` / `fake_reflection.py` / `traces/*.json` | — | 满足"无 API Key 跑通" | -| 7 | 6 条三类样例 case | `data/train.evalset.json`（3 条）+ `data/val.evalset.json`（3 条） | — | 覆盖"可优化成功 / 优化无效 / 优化后退化"三类各 1/3 | -| 8 | pipeline 入口 + README | `run_pipeline.py` + `README.md` | `optimizer.json` + `data/*.json` + `prompts/*.md` | 端到端产物 + 300–500 字设计说明 | - ---- - -## 7. 建议集成点（不修改 SDK 的前提下） - -> 这一节回答："新增的 8 个模块怎么和 SDK 现有 API 对接？" - -### 7.1 跑 baseline 与 candidate 评测 - -```python -# 拿 EvaluateResult 对象做 diff，必须用 get_executer，不要用 evaluate（无返回值） -executer = AgentEvaluator.get_executer( - eval_dataset_path=str(VAL_PATH), - call_agent=fake_call_agent, # 或 trace mode 下传 None - eval_metrics_file_path_or_dir="optimizer.json", # 复用同一份 metric 配置 - num_runs=1, - print_detailed_results=False, - print_summary_report=False, -) -await executer.evaluate() -baseline_result = executer.get_result() # → EvaluateResult -``` - -关键位置：`_agent_evaluator.py:363`（`get_executer`）、`_agent_evaluator.py:140`（`_EvalExecuter._run`）、`_eval_result.py:310`（`EvaluateResult`）。 - -### 7.2 跑优化（保留自动回滚） - -```python -result: OptimizeResult = await AgentOptimizer.optimize( - config_path="optimizer.json", - call_agent=fake_call_agent, - target_prompt=target, # TargetPrompt().add_path("system_prompt", ...).add_path("skill", ...) - train_dataset_path="data/train.evalset.json", - validation_dataset_path="data/val.evalset.json", - output_dir="runs//optimizer", - update_source=False, # ★ 让 SDK 自动回滚 baseline，避免污染下一次评测 - verbose=1, -) -best_prompts: dict[str, str] = result.best_prompts # 最优候选 prompt -``` - -关键位置：`_agent_optimizer.py:132-296`、`_optimize_result.py:167`（`OptimizeResult`）、`_target_prompt.py:63`（`TargetPrompt`，**注意：仓库中没有 `TargetSkill` 类，skill 是 `TargetPrompt` 的字段名之一**）。 - -### 7.3 写回 candidate 重跑验证集 - -```python -baseline_snapshot = await target.read_all() -try: - await target.write_all(best_prompts) # 临时把 candidate 写到源文件 - candidate_executer = AgentEvaluator.get_executer( - eval_dataset_path=str(VAL_PATH), - call_agent=fake_call_agent, - eval_metrics_file_path_or_dir="optimizer.json", - num_runs=1, - print_detailed_results=False, - print_summary_report=False, - ) - await candidate_executer.evaluate() - candidate_result = candidate_executer.get_result() -finally: - await target.write_all(baseline_snapshot) # ★ 必须复位，否则污染后续 run -``` - -关键位置：`_target_prompt.py:135-171` `write_all`（path 字段用 tmp+`os.replace` 原子写，部分失败自动回滚）、`_target_prompt.py:128-133` `read_all`。 - -### 7.4 失败归因输入来源 - -从 baseline 的 `EvaluateResult` 取： - -- `result.results_by_eval_set_id[set_id].eval_results_by_eval_id[eval_id].error_message`（`_eval_result.py:185`） -- `result....eval_metric_results[*].details.reason`（`_eval_result.py:108-116`） -- 对应的 `EvalCase.conversation[*].intermediate_data.tool_uses` / `tool_responses`（`_eval_case.py:121`）—— 用于区分"工具调用错"vs"参数错" - -trace mode 下这些字段都已就绪；非 trace 模式需要让 fake call_agent 在 `IntermediateData` 里回填假 tool_uses。 - -### 7.5 fake 路径选择优先级 - -1. **首选 trace mode**：评测阶段完全跳过 agent 推理，Windows 上稳定 ≤3 分钟。需要让 train/val evalset 都带 `eval_mode="trace"` + `actual_conversation`。 -2. **次选 fake call_agent**：当 case 不易预录对话时，在 `fake/fake_model.py` 实现 async 函数（按规则 / 字典返回字符串），由 pipeline 传入 `AgentEvaluator` / `AgentOptimizer`。 -3. **fake judge**：在 `fake/fake_judge.py` 用 `LLM_EVALUATOR_REGISTRY.register_response_scorer(...)`（`_llm_evaluator.py:103-178`）注册假 scorer，让 LLM metric 不调真实 LLM。 -4. **fake reflection LM**：在 `fake/fake_reflection.py` 包一个符合 gepa `LanguageModel` 协议的同步 callable，通过 `optimizer.json` 的 `reflection_lm.model_name = "fake-reflection"` 触发，或在 `run_pipeline.py` 里 monkeypatch `_OptimizeModelCallable.__call__`（`_optimize_model_callable.py:245-248`）。 - -### 7.6 接受 gate 必须独立于 SDK 决策 - -SDK 的 `stop.required_metrics` 只控制"早停"，不等于业务"是否接受 candidate"。新 pipeline 的 gate 必须放在 `AgentOptimizer.optimize` 返回**之后**再判一次： - -```python -result: OptimizeResult = await AgentOptimizer.optimize(...) -# 即使 result.status == "SUCCEEDED"，也要再判业务 gate -gate_decision = accept_policy.evaluate( - baseline_result=baseline_val_result, - candidate_result=candidate_val_result, - regression_diff=diff, - cost_usd=result.total_cost, - config=gate_config, -) -if not gate_decision.accepted: - # 即使 SDK 把 best_prompts 落到 best_prompts/ 也不回写源文件 - pass -``` - -这是"不能新增 hard fail"、"关键 case 不能退化"、"成本 ≤ 预算"等业务规则的**唯一干净落点**——SDK 内部的 Pareto / threshold 早停都不覆盖这些。 - -### 7.7 复现配置与随机种子 - -- `optimizer.json` 中的 `algorithm.seed`（`_optimize_config.py` `GepaReflectiveAlgo.seed`）是 GEPA 随机种子，写进 `audit.seed` -- `config.snapshot.json` 已由 SDK 自动落（`_agent_optimizer.py:425-432`），做 SHA-256 写进 `audit.repro_config_hash` -- 环境变量差异（`TRPC_AGENT_MODEL_NAME` 等）脱敏后写进 `audit.env_diff_redacted` - -### 7.8 集成点时序图（一图速览） - -``` -run_pipeline.py - │ - ├─ ① baseline 评测 - │ └─ AgentEvaluator.get_executer(val_path, call_agent=fake, ...) → EvaluateResult - │ - ├─ ② 失败归因 - │ └─ attribution/cluster.py(EvaluateResult) → 归因类别列表 - │ - ├─ ③ 优化 - │ └─ AgentOptimizer.optimize(config_path, call_agent=fake, target_prompt, train, val, output_dir) - │ │ - │ └─ 内部: GEPA 主循环（每轮：反思 LM → 写候选 → 跑 train/val → Pareto 比较） - │ └─ 返回 OptimizeResult.best_prompts - │ - ├─ ④ candidate 评测（写回 + 跑 val + 复位） - │ ├─ target_prompt.write_all(best_prompts) - │ ├─ AgentEvaluator.get_executer(val_path, call_agent=fake, ...) → EvaluateResult - │ └─ finally: target_prompt.write_all(baseline_snapshot) ◄── 必须复位 - │ - ├─ ⑤ 逐 case regression diff - │ └─ regression/diff.py(baseline_result, candidate_result) → diff - │ - ├─ ⑥ 接受 gate（独立于 SDK 决策） - │ └─ gate/accept_policy.py(diff, cost, config) → {accepted, reasons[]} - │ - └─ ⑦ 双格式报告 + 审计 - └─ audit/report_builder.py(全部上述产物) → optimization_report.{json,md} -``` - ---- - -## 8. 关键决策与陷阱速查 - -1. **没有 `TargetSkill` 类**——全仓库只有 `TargetPrompt`（`_target_prompt.py:63`），skill 是 `TargetPrompt.add_path("skill", ...)` 的字段名。 -2. **框架不内置 fake 实现**——所有 fake 路径都是扩展点（call_agent 协议 / `LLM_EVALUATOR_REGISTRY` / `ModelRegistry` / monkeypatch），必须由 `eval_optimize_loop/fake/` 落地。 -3. **trace mode 是 Windows ≤3 min 的最优解**——评测阶段完全跳过 agent，避免 `asyncio.to_thread` + Windows ProactorEventLoop 的潜在阻塞（参考第 3 节失败 #2）。 -4. **`AgentEvaluator.evaluate` 无返回值，`get_executer` 才有返回值**——做 regression diff 必须用后者。 -5. **`update_source=False` 是安全默认**——SDK 会自动把 baseline 回滚到源文件；pipeline 内不要再额外写回。 -6. **接受 gate 必须独立于 SDK**——SDK 的 `stop.required_metrics` / Pareto 早停只控制迭代终止，不等于业务接受。 -7. **Windows 盘符 bug 待避**（`_agent_evaluator.py:667`，参考第 3 节失败 #1）——evalset 路径优先用相对路径或 `Path.as_posix()`，绕开 `:` 分隔符误判。 -8. **6 条 case 的"退化"场景构造**——让 train 含特殊格式偏好（例如要求 "答：" 而非 "答案："），val 的 expected 与该偏好冲突；candidate 为通过 train 会牺牲 val，gate 必须拒绝。这是覆盖 Issue 验收标准 #3 的关键设计。 diff --git a/examples/optimization/eval_optimize_loop/.gitignore b/examples/optimization/eval_optimize_loop/.gitignore index 36e3337a6..60f8d1221 100644 --- a/examples/optimization/eval_optimize_loop/.gitignore +++ b/examples/optimization/eval_optimize_loop/.gitignore @@ -3,6 +3,9 @@ runs/ optimization_report.json optimization_report.md +# Local model credentials (never commit) +.env + # Python __pycache__/ *.pyc diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index eb5f33fbc..1d3ffdb32 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -1,191 +1,33 @@ -# 方案设计 +# 方案设计（Issue #91：Evaluation + Optimization 闭环） -管线把当前源提示词定义为 Champion，把 `AgentOptimizer.optimize(update_source=False)` 产生的最佳提示词或显式候选定义为 Challenger。优化器只使用训练集发现问题；独立验证集只参加回归和门禁，避免把验证失败细节反馈给优化器。fake 模式由提示词内公开控制项生成可评测轨迹，仅验证流程，不作为隐藏样本能力证明。 +## Champion–Challenger 与数据隔离 -每次运行以 UTC 微秒时间和随机后缀生成唯一目录，冻结提示词、数据集、评测配置、优化配置、Gate、模型和随机种子的哈希或标识。Champion 与 Challenger 均由 `AgentEvaluator` 评分；逐 case 保存实际/预期回复、指标原因、工具调用与响应、参数差异及 trace 引用。归因依据这些证据输出回复、格式、工具、参数、rubric、知识、基础设施或证据不足类别，不依赖场景标签。 +管线把当前源 prompt 定义为 Champion，把 `AgentOptimizer.optimize(update_source=False)` +产出的最佳 prompt（或 fake 模式下的显式候选）定义为 Challenger。训练集只供优化器发现 +问题；验证集是独立裁判，只参与回归与门禁，失败细节不回喂优化器。fake 模式由 prompt 内 +公开的 `FAKE_CONTROLS` 控制项生成可评测轨迹，仅验证流程本身，不证明泛化能力。 -Gate 同时检查验证提升、训练涨验证跌的过拟合、新增高风险失败、protected case 与 slice 退化、成本证据和微小波动。成本未知时保存空值并由 G6 拒绝自动应用。评测中的临时替换、异常恢复和最终写回都经 `TargetPrompt`；只有显式 `--apply` 且全部 Gate 通过才更新 Champion。JSON 与 Markdown 报告记录决策、每轮候选、耗时、成本、产物路径和复现命令，优化失败也会落盘可审计的 REJECT 报告。 +## 失败归因方法 -你在可信的本地仓库 trpc-agent-python 中工作，目标是完成 GitHub Issue #91： +归因完全基于 evaluator 证据，不读场景标签：先用 `error_message` / `NOT_EVALUATED` +区分 infrastructure_failure；再按失败 metric 名与 reason 识别 knowledge_fail / +rubric_fail；然后对比期望与实际工具轨迹区分 tool_call_error（缺调用或调错工具）与 +param_error（同名工具参数不一致，附参数 diff）；最后对比 actual/expected 文本，核心 +数值一致判 format_fail，否则 reply_mismatch；证据不足时输出 insufficient_evidence， +不臆造类别。每个失败 case 至少给出一条可解释原因和原始证据。 -“构建一个可复现的 Evaluation + Optimization pipeline： -评测 → 失败归因 → Prompt 优化 → 回归验证 → 产物审计。” +## 接受策略与防过拟合 -请连续完成以下工作；只有在确实无法从仓库源码判断时才提问。完成后不要 git commit、不要 git push、不要修改或删除当前任务范围之外的文件。 +Gate 为纯规则、逐条单测：G1 验证集提升 ≥ min_val_lift；G2 train 升而 val 降视为过拟 +合，直接拒绝；G3 不新增 high-risk hard fail；G4 protected case 不退化；G5 slice 平均 +退化不超 tolerance；G6 成本证据完整且不超预算，成本未知一律禁止自动接受；G7 epsilon +内的微小波动不算有效提升。防过拟合由 train/val 物理隔离 + G2 + G4/G5 共同保证。 -# 总体边界 +## 产物审计 -1. 只新增或修改： - examples/optimization/eval_optimize_loop/ -2. 不修改 trpc_agent_sdk/、tests/ 现有 SDK 测试、项目依赖、CI、git 配置。 -3. 不做 Web UI、数据库、远端 Prompt Store、多 Agent 编排、MCP 服务。 -4. 不复制其他 PR 的代码；可以复用当前仓库公开 API 和已有 example 的编码风格。 -5. 不运行全仓 pytest；Windows 存在与本任务无关的 POSIX 平台失败。 -6. 使用相对 evalset 路径，避免 Windows 盘符解析问题。 -7. 所有写 Prompt 的操作必须通过 TargetPrompt 的公开 API，并正确 await read_all/write_all。 -8. 默认绝不改 prompts/system.md。只有 gate=ACCEPT 且 CLI 显式传入 --apply 时才写回。 -9. 不使用 monkeypatch SDK 私有成员，不新增 jsonschema 或第三方依赖。 - -# 要保留的设计主线 - -当前 Prompt 是 Champion；新 Prompt 是 Challenger。 -候选不能因为训练集分高就自动成为 Champion，必须经过独立验证与门禁。 -任何证据缺失都不能自动接受候选。 - -训练集 = 用于优化器发现问题； -验证集 = 独立裁判，不能把其详细失败内容喂回优化器。 - -# 必须交付的目录内容 - -在 examples/optimization/eval_optimize_loop/ 内实现： - -- README.md：中文优先，80 行左右，说明目标、运行命令、三种场景、产物。 -- DESIGN.md：简洁说明 Champion/Challenger、数据隔离、gate、fake/live 两种模式。 -- pipeline.py：唯一 CLI 入口。 -- runner.py：运行 Champion 与 Challenger 的评测、冻结输入、确保恢复 Prompt。 -- attribution.py：把失败结果归类为： - reply_mismatch / format_fail / tool_call_error / param_error / - rubric_fail / knowledge_fail / none -- gates.py：纯规则、可独立单测。 -- report.py：输出 optimization_report.json 和 optimization_report.md。 -- data/train.evalset.json：3 条。 -- data/val.evalset.json：3 条。 -- prompts/system.md。 -- 必要的 fake 实现与 tests/。 -- runs/ 输出目录应 gitignore。 - -核心 Python 文件保持精简；不要为了形式拆成大量 package。 - -# 两阶段实现 - -## Milestone A：确定性 fake 模式，必须完整可跑 - -实现 --mode fake，完全不需要 API Key。 - -fake agent 的行为必须由当前 Prompt 中公开、可读的标记决定，例如： -[[FORMAT_JSON]] -[[USE_CORRECT_TOOL]] -[[MEMORIZE_TRAIN]] - -禁止根据 prompt hash 偷偷切换预录答案。 -也禁止伪造总分；必须返回可评测的回答或工具轨迹，再由评测/规则计算结果。 - -必须内置三个可复现情景： - -1. success - Candidate 修复格式或工具问题，train 和 val 都提升，最终 ACCEPT。 - -2. no_effect - Candidate 没有产生有效改善，最终 REJECT。 - -3. overfit - Candidate 只记住 train 样本，train 提升而 val 退化， - 或新增高风险失败，最终 REJECT；理由必须明确提到“过拟合”。 - -## Milestone B：原生 AgentOptimizer 接入 - -实现 --mode optimize： - -- 构造 TargetPrompt； -- 调用： - await AgentOptimizer.optimize( - config_path=..., - call_agent=..., - target_prompt=..., - train_dataset_path=..., - validation_dataset_path=..., - output_dir=..., - update_source=False, - ) -- 从结果中获取 best_prompts 作为 Challenger； -- 使用与 fake 模式相同的 runner / gates / report； -- 即使本地没有真实模型 API Key，也要做到配置缺失时报错清楚； -- fake 模式必须始终能完整跑通。 - -# Gate 规则 - -Gate 必须同时看 train 与 val，输出 ACCEPT 或 REJECT 以及逐条理由。 - -必须包含： - -G1. validation 最小有效提升，例如 min_val_lift=0.02。 -G2. train 提升而 validation 下降，视为 overfit，必拒绝。 -G3. 不新增 hard fail。 -G4. protected case 不退化。 -G5. 关键 slice 平均分不超过 tolerance 地下降。 -G6. 成本证据完整： - - trace/fake 模式可写 cost_status="measured", cost=0， - 因为确实没有模型调用； - - 无法采集真实模型成本时写 cost_status="unavailable"， - 并拒绝自动 ACCEPT，不能把未知成本写成 0。 -G7. 明显微小的分数变化不算有效提升。 - -不要写互相重复的规则。 -每条规则都必须有独立单元测试。 - -# 数据与报告 - -只使用 train / val 两个 evalset，各 3 条，eval_id 不能重叠。 - -报告 optimization_report.json 和 optimization_report.md 必须含： - -- frozen：prompt、dataset、optimizer config、运行模式等哈希/版本信息； -- champion 与 challenger 的来源和 sha256； -- train / val 的 baseline、candidate、delta； -- 每个 case 的状态、分数、delta、slice、risk_level、失败分类； -- decision、违反的 gate、自然语言理由； -- audit：duration_seconds、cost_status、cost、applied、artifact 路径、可复现命令； -- candidate_source：candidate_file 或 agent_optimizer。 - -Markdown 和 JSON 表达同一套核心信息。 - -# Prompt 安全写回 - -候选评测时可临时写入 Challenger，但必须： - -- 先保存 Champion snapshot； -- 用 try/finally 恢复； -- 测试 dry-run 后 prompts/system.md 的 sha256 不变； -- --apply + ACCEPT 才真正写回； -- --apply + REJECT 必须报出原因且源文件不变； -- 备份和写回也使用 TargetPrompt 的机制，不要 Path.write_text 另起一套逻辑。 - -# 测试要求 - -新增 tests 至少覆盖： - -1. train / val eval_id 不重叠； -2. 六类失败归因； -3. G1-G7； -4. success / no_effect / overfit 三个端到端场景； -5. dry-run 后 Champion 不变； -6. --apply 成功与拒绝； -7. 无 API Key 的 fake mode； -8. 报告字段完整； -9. 目标目录测试总时长小于 180 秒。 - -先写测试，再实现对应最小代码。 - -# 验证命令 - -只运行： - -python -m pytest examples/optimization/eval_optimize_loop/tests -q -python examples/optimization/eval_optimize_loop/pipeline.py --mode fake --scenario success -python examples/optimization/eval_optimize_loop/pipeline.py --mode fake --scenario no_effect -python examples/optimization/eval_optimize_loop/pipeline.py --mode fake --scenario overfit -python -m compileall -q examples/optimization/eval_optimize_loop -git diff --check -git status -sb - -# 最终回复格式 - -完成后按以下格式汇报： - -1. 已修改/新增文件； -2. 每个阶段如何对应 Issue #91； -3. 三个 fake 场景的实际输出与 decision； -4. pytest 结果、耗时、其他验证结果； -5. 未验证的 live optimize 风险； -6. git diff 摘要； -7. 不要提交或推送。 \ No newline at end of file +每次运行生成唯一 `runs/-<随机后缀>/` 目录，`frozen.json` 冻结 prompt、数据 +集、评测/优化/gate 配置与随机种子的 sha256；落盘 Champion/Challenger 快照、完整 +evaluator 结果、优化轮次、调用审计、耗时与成本。`optimization_report.{json,md}` 记录 +baseline / candidate 分数、逐 case delta、归因统计、gate 决策与理由、复现命令。默认 +dry-run；仅 ACCEPT 且显式 `--apply` 时经 `TargetPrompt` 原子写回，优化失败也会落盘可 +审计的 REJECT 报告。 diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json index 3470e1820..238e9e1d0 100644 --- a/examples/optimization/eval_optimize_loop/optimizer.json +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -1,5 +1,4 @@ { - "_comment": "--mode optimize 使用：真实调用 AgentOptimizer.optimize(update_source=false)。reflection_lm 变量由 TRPC_AGENT_* 环境提供；fake 回归不会读取这些凭据。", "evaluate": { "metrics": [ { @@ -26,7 +25,7 @@ "name": "gepa_reflective", "seed": 42, "reflection_lm": { - "provider_name": "${TRPC_AGENT_PROVIDER_NAME:openai}", + "provider_name": "openai", "model_name": "${TRPC_AGENT_MODEL_NAME}", "base_url": "${TRPC_AGENT_BASE_URL}", "api_key": "${TRPC_AGENT_API_KEY}", diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index 1b990f1df..cd190b550 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -12,6 +12,7 @@ import argparse import asyncio import json +import os import sys from dataclasses import asdict, dataclass from datetime import datetime, timezone @@ -27,10 +28,17 @@ import fake_agent # type: ignore[unresolved-import] import gates # type: ignore[unresolved-import] -import live_agent # type: ignore[unresolved-import] import report # type: ignore[unresolved-import] import runner # type: ignore[unresolved-import] +try: + # 本地凭据从同目录 .env 读取（已被 gitignore）；已有环境变量优先。 + from dotenv import load_dotenv + + load_dotenv(_HERE / ".env") +except ImportError: + pass + DEFAULT_RUN_JSON: dict[str, Any] = { "champion_prompt": "prompts/system.md", "train_evalset": "data/train.evalset.json", @@ -218,16 +226,23 @@ async def _run_optimize_for_candidate( from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt target = TargetPrompt().add_path("system", str(champion_prompt_path)) - result = await AgentOptimizer.optimize( - config_path=str(optimizer_config_path), - call_agent=call_agent, - target_prompt=target, - train_dataset_path=str(train_evalset_path), - validation_dataset_path=str(val_evalset_path), - output_dir=str(output_dir), - update_source=False, - verbose=0, - ) + # Windows 盘符含冒号，会被 evalset 的 "file.json:case_id" 语法误切； + # 与 runner._run_evaluator 一致，切到示例根目录并传相对路径。 + previous_cwd = Path.cwd() + try: + os.chdir(_HERE) + result = await AgentOptimizer.optimize( + config_path=str(optimizer_config_path), + call_agent=call_agent, + target_prompt=target, + train_dataset_path=os.path.relpath(train_evalset_path, _HERE), + validation_dataset_path=os.path.relpath(val_evalset_path, _HERE), + output_dir=str(output_dir), + update_source=False, + verbose=0, + ) + finally: + os.chdir(previous_cwd) status = _enum_value(result.status) if status != "SUCCEEDED" or not result.best_prompts: raise RuntimeError( @@ -316,7 +331,11 @@ def _failure_frozen( async def _amain(args: argparse.Namespace, *, call_agent=None) -> int: - config_path = _resolve(_HERE, args.config) if args.config else None + if args.config: + config_path: Optional[Path] = _resolve(_HERE, args.config) + else: + default_run_json = _HERE / "run.json" + config_path = default_run_json if default_run_json.exists() else None config = _load_config(config_path) champion_path = _resolve(_HERE, config["champion_prompt"]) train_path = _resolve(_HERE, config["train_evalset"]) @@ -339,6 +358,9 @@ async def _amain(args: argparse.Namespace, *, call_agent=None) -> int: try: base_call_agent = call_agent if base_call_agent is None: + # 懒加载：fake 模式无需引入真实模型 SDK 依赖链。 + import live_agent # type: ignore[unresolved-import] + model_info = live_agent.model_info_from_env() base_call_agent = live_agent.build_call_agent(champion_path) else: diff --git a/trpc_agent_sdk/evaluation/_agent_evaluator.py b/trpc_agent_sdk/evaluation/_agent_evaluator.py index a2e47009d..8aa6750a0 100644 --- a/trpc_agent_sdk/evaluation/_agent_evaluator.py +++ b/trpc_agent_sdk/evaluation/_agent_evaluator.py @@ -660,14 +660,18 @@ def _load_eval_set_from_file( FileNotFoundError: If file doesn't exist ValueError: If file format is invalid or eval case not found """ - # Check if file_path contains a case selector (ADK style: "file.json:case_id") + # Check if file_path contains a case selector (ADK style: "file.json:case_id"). + # A Windows drive letter also contains ":", so only treat the colon as a + # selector when the full string is not an existing file and the part + # before the last colon is. selected_case_id = None actual_file_path = eval_set_file - if ":" in eval_set_file: - parts = eval_set_file.split(":", 1) - actual_file_path = parts[0] - selected_case_id = parts[1] + if ":" in eval_set_file and not os.path.exists(eval_set_file): + file_part, _, case_part = eval_set_file.rpartition(":") + if file_part and os.path.exists(file_part): + actual_file_path = file_part + selected_case_id = case_part if not os.path.exists(actual_file_path): raise FileNotFoundError(f"Eval set file not found: {actual_file_path}") From 5bee3f638e567967c0ce03507e1e654b9b4a3e31 Mon Sep 17 00:00:00 2001 From: ygrowly Date: Thu, 30 Jul 2026 15:05:52 +0800 Subject: [PATCH 3/6] test: cover eval set case-selector parsing incl. Windows drive-letter paths --- tests/evaluation/test_agent_evaluator.py | 59 ++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/evaluation/test_agent_evaluator.py b/tests/evaluation/test_agent_evaluator.py index 8609ccc5d..2e2a604c4 100644 --- a/tests/evaluation/test_agent_evaluator.py +++ b/tests/evaluation/test_agent_evaluator.py @@ -10,12 +10,18 @@ import trpc_agent_sdk.runners # noqa: F401 from trpc_agent_sdk.evaluation import EvalStatus +from trpc_agent_sdk.evaluation import EvalCase from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import EvalConfig from trpc_agent_sdk.evaluation import EvalMetricResult +from trpc_agent_sdk.evaluation import EvalSet from trpc_agent_sdk.evaluation import EvalSetAggregateResult from trpc_agent_sdk.evaluation import EvaluateResult from trpc_agent_sdk.evaluation import AgentEvaluator +from trpc_agent_sdk.evaluation import Invocation from trpc_agent_sdk.evaluation import PassNC +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import Part class TestPassNC: @@ -97,3 +103,56 @@ def test_pass_at_k_delegates(self): def test_pass_hat_k_delegates(self): """Test AgentEvaluator.pass_hat_k delegates to _eval_pass.""" assert AgentEvaluator.pass_hat_k(10, 5, 2) == 0.25 + + +class TestLoadEvalSetFromFile: + """Test suite for AgentEvaluator._load_eval_set_from_file. + + Covers the case-selector colon parsing, including absolute paths that + contain a drive-letter colon on Windows (e.g. "C:\\...\\set.json"). + """ + + @staticmethod + def _write_eval_set(tmp_path, case_ids): + """Write a minimal eval set file and return its absolute path.""" + cases = [ + EvalCase( + eval_id=case_id, + conversation=[ + Invocation( + invocation_id="i", + user_content=Content(parts=[Part(text="hi")]), + ), + ], + ) + for case_id in case_ids + ] + eval_set = EvalSet(eval_set_id="set1", eval_cases=cases) + file_path = tmp_path / "set.evalset.json" + file_path.write_text(eval_set.model_dump_json(), encoding="utf-8") + return str(file_path) + + def test_load_absolute_path_without_selector(self, tmp_path): + """An existing absolute path must load as-is, even if it contains ':'.""" + file_path = self._write_eval_set(tmp_path, ["case_a", "case_b"]) + eval_set = AgentEvaluator._load_eval_set_from_file(file_path, EvalConfig(criteria={})) + assert [c.eval_id for c in eval_set.eval_cases] == ["case_a", "case_b"] + + def test_load_with_case_selector(self, tmp_path): + """"file.json:case_id" selects a single case from the set.""" + file_path = self._write_eval_set(tmp_path, ["case_a", "case_b"]) + eval_set = AgentEvaluator._load_eval_set_from_file(f"{file_path}:case_b", EvalConfig(criteria={})) + assert [c.eval_id for c in eval_set.eval_cases] == ["case_b"] + assert eval_set.eval_set_id == "set1_case_b" + + def test_load_with_unknown_case_selector_raises(self, tmp_path): + """Selecting a case id that does not exist raises ValueError.""" + file_path = self._write_eval_set(tmp_path, ["case_a"]) + with pytest.raises(ValueError, match="not found"): + AgentEvaluator._load_eval_set_from_file(f"{file_path}:missing", EvalConfig(criteria={})) + + def test_load_missing_file_raises(self, tmp_path): + """A non-existing path (with or without ':') raises FileNotFoundError.""" + missing = str(tmp_path / "nope.evalset.json") + with pytest.raises(FileNotFoundError): + AgentEvaluator._load_eval_set_from_file(missing, EvalConfig(criteria={})) From d6085ff99881a4e13c347500424163eb96874242 Mon Sep 17 00:00:00 2001 From: ygrowly Date: Thu, 30 Jul 2026 15:19:09 +0800 Subject: [PATCH 4/6] fix: address AI review - narrow AssertionError handling and quote repro cmd - _run_evaluator: re-raise assertions that abort before results exist instead of masking them as empty NOT_EVALUATED outcomes - _build_repro_cmd: shlex.quote each argument so paths with spaces stay intact - Add tests covering both swallowed-assertion paths and repro cmd quoting --- .../eval_optimize_loop/pipeline.py | 4 +- .../optimization/eval_optimize_loop/runner.py | 11 ++- .../tests/test_evaluator_error_handling.py | 81 +++++++++++++++++++ 3 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/tests/test_evaluator_error_handling.py diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index cd190b550..a688aee2e 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -13,6 +13,7 @@ import asyncio import json import os +import shlex import sys from dataclasses import asdict, dataclass from datetime import datetime, timezone @@ -102,7 +103,8 @@ def _build_repro_cmd(args: argparse.Namespace) -> str: continue flag = f"--{key.replace('_', '-')}" argv.append(flag if value is True else f"{flag}={value}") - return " ".join(argv) + # 路径可能含空格等特殊字符，引用后保证复现命令可直接粘贴执行。 + return " ".join(shlex.quote(arg) for arg in argv) def _content_text(content: dict[str, Any]) -> str: diff --git a/examples/optimization/eval_optimize_loop/runner.py b/examples/optimization/eval_optimize_loop/runner.py index 6a1ff0019..69ac59758 100644 --- a/examples/optimization/eval_optimize_loop/runner.py +++ b/examples/optimization/eval_optimize_loop/runner.py @@ -275,10 +275,15 @@ async def _run_evaluator( ) try: await executer.evaluate() - except AssertionError: + except AssertionError as exc: # Case failures are represented as AssertionError by the public facade; - # the structured result remains available from the executer. - pass + # the structured result remains available from the executer. Any other + # assertion (config/contract errors, SDK bugs) aborts before a result + # exists and must not be masked as an empty NOT_EVALUATED outcome. + if executer.get_result() is None: + raise RuntimeError( + f"Evaluator aborted before producing results ({type(exc).__name__}): {exc}" + ) from exc result = executer.get_result() finally: os.chdir(previous_cwd) diff --git a/examples/optimization/eval_optimize_loop/tests/test_evaluator_error_handling.py b/examples/optimization/eval_optimize_loop/tests/test_evaluator_error_handling.py new file mode 100644 index 000000000..0107bbe8e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_evaluator_error_handling.py @@ -0,0 +1,81 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""验证 _run_evaluator 不吞非 case-failure 断言，以及复现命令的 shell 引用。""" + +from __future__ import annotations + +import argparse +import shlex +from pathlib import Path + +import pytest + +import pipeline +import runner +from trpc_agent_sdk.evaluation import EvaluateResult + + +class _StubExecuter: + """evaluate() 抛 AssertionError；get_result() 返回预置结果。""" + + def __init__(self, result): + self._result = result + + async def evaluate(self): + raise AssertionError("boom") + + def get_result(self): + return self._result + + +@pytest.mark.asyncio +async def test_assertion_without_result_is_not_swallowed(tmp_path: Path, monkeypatch) -> None: + """配置/契约类断言（结果尚未生成）必须重新抛出，不能伪装成空结果。""" + + stub = _StubExecuter(result=None) + monkeypatch.setattr( + runner.AgentEvaluator, "get_executer", staticmethod(lambda *a, **k: stub) + ) + evalset = tmp_path / "set.evalset.json" + evalset.write_text("{}", encoding="utf-8") + + with pytest.raises(RuntimeError, match="aborted before producing results"): + await runner._run_evaluator(evalset) + + +@pytest.mark.asyncio +async def test_case_failure_assertion_keeps_structured_result(tmp_path: Path, monkeypatch) -> None: + """case 失败断言仍返回 executer 里的结构化结果。""" + + expected = EvaluateResult(results_by_eval_set_id={}) + stub = _StubExecuter(result=expected) + monkeypatch.setattr( + runner.AgentEvaluator, "get_executer", staticmethod(lambda *a, **k: stub) + ) + evalset = tmp_path / "set.evalset.json" + evalset.write_text("{}", encoding="utf-8") + + result = await runner._run_evaluator(evalset) + assert result is expected + + +def test_repro_cmd_quotes_paths_with_spaces() -> None: + """含空格的路径必须被引用，复现命令可安全粘贴执行。""" + + args = argparse.Namespace( + mode="fake", + scenario="success", + candidate_file="C:/tmp/my candidate.md", + apply=True, + optimizer_config=None, + ) + cmd = pipeline._build_repro_cmd(args) + parsed = shlex.split(cmd) + assert "--candidate-file=C:/tmp/my candidate.md" in parsed + assert "--apply" in parsed + assert not any(part == "candidate.md" for part in parsed) From d99414f98cc94c06d16b59e538ef7a4d831ed296 Mon Sep 17 00:00:00 2001 From: ygrowly Date: Thu, 30 Jul 2026 23:17:18 +0800 Subject: [PATCH 5/6] test: exercise colon guard on POSIX; drop chdir workarounds - Add POSIX-only case loading an existing path that contains ':' so the drive-letter guard is covered on Linux CI as well - Pass absolute paths to evaluator/optimizer now that the SDK guards the case selector, removing process-wide chdir and metric-config copying --- .../eval_optimize_loop/pipeline.py | 30 ++++------ .../optimization/eval_optimize_loop/runner.py | 59 ++++++++----------- tests/evaluation/test_agent_evaluator.py | 31 ++++++++-- 3 files changed, 65 insertions(+), 55 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index a688aee2e..90a43ec54 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -12,7 +12,6 @@ import argparse import asyncio import json -import os import shlex import sys from dataclasses import asdict, dataclass @@ -228,23 +227,18 @@ async def _run_optimize_for_candidate( from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt target = TargetPrompt().add_path("system", str(champion_prompt_path)) - # Windows 盘符含冒号，会被 evalset 的 "file.json:case_id" 语法误切； - # 与 runner._run_evaluator 一致，切到示例根目录并传相对路径。 - previous_cwd = Path.cwd() - try: - os.chdir(_HERE) - result = await AgentOptimizer.optimize( - config_path=str(optimizer_config_path), - call_agent=call_agent, - target_prompt=target, - train_dataset_path=os.path.relpath(train_evalset_path, _HERE), - validation_dataset_path=os.path.relpath(val_evalset_path, _HERE), - output_dir=str(output_dir), - update_source=False, - verbose=0, - ) - finally: - os.chdir(previous_cwd) + # SDK 已修复盘符冒号误切（_load_eval_set_from_file 用存在性守卫）， + # 直接传绝对路径，避免进程级 chdir 带来的全局状态耦合。 + result = await AgentOptimizer.optimize( + config_path=str(optimizer_config_path), + call_agent=call_agent, + target_prompt=target, + train_dataset_path=str(train_evalset_path), + validation_dataset_path=str(val_evalset_path), + output_dir=str(output_dir), + update_source=False, + verbose=0, + ) status = _enum_value(result.status) if status != "SUCCEEDED" or not result.best_prompts: raise RuntimeError( diff --git a/examples/optimization/eval_optimize_loop/runner.py b/examples/optimization/eval_optimize_loop/runner.py index 69ac59758..af1ab138c 100644 --- a/examples/optimization/eval_optimize_loop/runner.py +++ b/examples/optimization/eval_optimize_loop/runner.py @@ -11,7 +11,6 @@ import hashlib import json -import os import shutil import time import uuid @@ -254,39 +253,33 @@ async def _run_evaluator( call_agent: Optional[CallAgent] = None, metric_config_path: Optional[Path] = None, ) -> EvaluateResult: - """Run the public evaluator API while avoiding the Windows drive-colon parser.""" - - previous_cwd = Path.cwd() + """Run the public evaluator API with absolute paths. + + The SDK's ``_load_eval_set_from_file`` guards the "file.json:case_id" + selector with existence checks, so Windows drive-letter paths are safe + to pass directly and no process-wide chdir workaround is needed. + """ + + metric_arg = str(metric_config_path) if metric_config_path is not None else None + executer = AgentEvaluator.get_executer( + str(evalset_path), + call_agent=call_agent, + eval_metrics_file_path_or_dir=metric_arg, + print_detailed_results=False, + print_summary_report=False, + ) try: - os.chdir(evalset_path.parent) - metric_arg: Optional[str] = None - if metric_config_path is not None: - if metric_config_path.parent.resolve() != evalset_path.parent.resolve(): - local_config = evalset_path.parent / metric_config_path.name - if not local_config.exists(): - shutil.copyfile(metric_config_path, local_config) - metric_arg = metric_config_path.name - executer = AgentEvaluator.get_executer( - evalset_path.name, - call_agent=call_agent, - eval_metrics_file_path_or_dir=metric_arg, - print_detailed_results=False, - print_summary_report=False, - ) - try: - await executer.evaluate() - except AssertionError as exc: - # Case failures are represented as AssertionError by the public facade; - # the structured result remains available from the executer. Any other - # assertion (config/contract errors, SDK bugs) aborts before a result - # exists and must not be masked as an empty NOT_EVALUATED outcome. - if executer.get_result() is None: - raise RuntimeError( - f"Evaluator aborted before producing results ({type(exc).__name__}): {exc}" - ) from exc - result = executer.get_result() - finally: - os.chdir(previous_cwd) + await executer.evaluate() + except AssertionError as exc: + # Case failures are represented as AssertionError by the public facade; + # the structured result remains available from the executer. Any other + # assertion (config/contract errors, SDK bugs) aborts before a result + # exists and must not be masked as an empty NOT_EVALUATED outcome. + if executer.get_result() is None: + raise RuntimeError( + f"Evaluator aborted before producing results ({type(exc).__name__}): {exc}" + ) from exc + result = executer.get_result() return result or EvaluateResult() diff --git a/tests/evaluation/test_agent_evaluator.py b/tests/evaluation/test_agent_evaluator.py index 2e2a604c4..ccbad24f2 100644 --- a/tests/evaluation/test_agent_evaluator.py +++ b/tests/evaluation/test_agent_evaluator.py @@ -5,6 +5,8 @@ # tRPC-Agent-Python is licensed under Apache-2.0. """Unit tests for agent evaluator (agent_evaluator).""" +import os + import pytest import trpc_agent_sdk.runners # noqa: F401 @@ -108,8 +110,10 @@ def test_pass_hat_k_delegates(self): class TestLoadEvalSetFromFile: """Test suite for AgentEvaluator._load_eval_set_from_file. - Covers the case-selector colon parsing, including absolute paths that - contain a drive-letter colon on Windows (e.g. "C:\\...\\set.json"). + Covers the case-selector colon parsing. The drive-letter guard + ("an existing path containing ':' must load as-is") is exercised on + Windows by the absolute ``tmp_path`` ("C:\\...") and on POSIX by a + file whose name literally contains a colon. """ @staticmethod @@ -133,13 +137,32 @@ def _write_eval_set(tmp_path, case_ids): return str(file_path) def test_load_absolute_path_without_selector(self, tmp_path): - """An existing absolute path must load as-is, even if it contains ':'.""" + """An existing absolute path must load as-is. + + On Windows ``tmp_path`` starts with a drive letter, so the ':' guard + is hit here; POSIX coverage of the same guard lives in + ``test_load_existing_path_containing_colon``. + """ file_path = self._write_eval_set(tmp_path, ["case_a", "case_b"]) eval_set = AgentEvaluator._load_eval_set_from_file(file_path, EvalConfig(criteria={})) assert [c.eval_id for c in eval_set.eval_cases] == ["case_a", "case_b"] + @pytest.mark.skipif(os.name == "nt", reason="':' is not a legal filename character on Windows") + def test_load_existing_path_containing_colon(self, tmp_path): + """A colon inside an existing full path must not be split as a selector.""" + source = self._write_eval_set(tmp_path, ["case_a", "case_b"]) + colon_path = tmp_path / "drive:like.evalset.json" + with open(source, "r", encoding="utf-8") as f: + colon_path.write_text(f.read(), encoding="utf-8") + eval_set = AgentEvaluator._load_eval_set_from_file(str(colon_path), EvalConfig(criteria={})) + assert [c.eval_id for c in eval_set.eval_cases] == ["case_a", "case_b"] + def test_load_with_case_selector(self, tmp_path): - """"file.json:case_id" selects a single case from the set.""" + """"file.json:case_id" selects a single case from the set. + + The full string never exists on disk while the part before the last + colon does, so the rpartition branch runs on every platform. + """ file_path = self._write_eval_set(tmp_path, ["case_a", "case_b"]) eval_set = AgentEvaluator._load_eval_set_from_file(f"{file_path}:case_b", EvalConfig(criteria={})) assert [c.eval_id for c in eval_set.eval_cases] == ["case_b"] From 32214089528e70e42486dce181c08023b92c4f9c Mon Sep 17 00:00:00 2001 From: ygrowly Date: Fri, 31 Jul 2026 10:19:31 +0800 Subject: [PATCH 6/6] fix: harden audit callback, event text collection and gate config bounds - Audit callback: normalize whitespace when matching queries and degrade to an unmatched-marked entry instead of raising KeyError mid-optimize - live_agent: extract response_text_from_event using getattr so parts without thought/text attributes cannot break regression runs - GateConfig: reject non-positive budget_tokens/budget_usd to keep G6 meaningful - Add tests for trimmed-query matching, unknown-query degradation, thought-less parts and budget validation --- .../optimization/eval_optimize_loop/gates.py | 14 ++- .../eval_optimize_loop/live_agent.py | 23 ++++- .../eval_optimize_loop/pipeline.py | 21 +++- .../tests/test_audit_robustness.py | 99 +++++++++++++++++++ 4 files changed, 144 insertions(+), 13 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/tests/test_audit_robustness.py diff --git a/examples/optimization/eval_optimize_loop/gates.py b/examples/optimization/eval_optimize_loop/gates.py index 9d359bb82..1790499ed 100644 --- a/examples/optimization/eval_optimize_loop/gates.py +++ b/examples/optimization/eval_optimize_loop/gates.py @@ -73,6 +73,14 @@ class GateConfig: budget_usd: Optional[float] = None epsilon: float = 0.001 + def __post_init__(self) -> None: + # fake/trace 模式的 measured 成本恒为 0 token；若允许 budget_tokens<=0， + # G6 会把任意合法成本误判为超预算。 + if self.budget_tokens <= 0: + raise ValueError(f"budget_tokens 必须为正整数，当前值：{self.budget_tokens}") + if self.budget_usd is not None and self.budget_usd <= 0: + raise ValueError(f"budget_usd 配置时必须为正数，当前值：{self.budget_usd}") + @dataclass class Decision: @@ -149,10 +157,8 @@ def evaluate( bad_slices = {k: v for k, v in slice_deltas.items() if v < -cfg.slice_tolerance} if bad_slices: violated.append("G5") - reasons.append( - f"slice 退化超过 tolerance={cfg.slice_tolerance:.4f}：" - f"{ {k: round(v, 4) for k, v in bad_slices.items()} }。" - ) + rounded = {k: round(v, 4) for k, v in bad_slices.items()} + reasons.append(f"slice 退化超过 tolerance={cfg.slice_tolerance:.4f}：{rounded}。") # G6: cost evidence if cost_status == "unavailable": diff --git a/examples/optimization/eval_optimize_loop/live_agent.py b/examples/optimization/eval_optimize_loop/live_agent.py index f4630d04e..fb035cede 100644 --- a/examples/optimization/eval_optimize_loop/live_agent.py +++ b/examples/optimization/eval_optimize_loop/live_agent.py @@ -48,6 +48,23 @@ def validate_model_env() -> None: ) +def response_text_from_event(event: Any) -> str: + """Collect non-thought text from a final-response event. + + Uses ``getattr`` so parts from any provider adapter are safe even when + they carry no ``thought``/``text`` attribute at all. + """ + + if not event.is_final_response() or not event.content: + return "" + texts = [] + for part in getattr(event.content, "parts", None) or []: + text = getattr(part, "text", None) + if text and not getattr(part, "thought", None): + texts.append(text) + return "".join(texts) + + def build_call_agent(prompt_path: Path) -> CallAgent: """Build a callback that reloads the current TargetPrompt on every call.""" @@ -87,11 +104,7 @@ async def call_agent(query: str) -> str: session_id=session_id, new_message=message, ): - if not event.is_final_response() or not event.content: - continue - for part in event.content.parts or []: - if not part.thought and part.text: - response += part.text + response += response_text_from_event(event) return response.strip() # The SDK runner does not expose a provider-independent USD amount here. diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py index 90a43ec54..a37210146 100644 --- a/examples/optimization/eval_optimize_loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -110,6 +110,16 @@ def _content_text(content: dict[str, Any]) -> str: return "\n".join(str(part.get("text", "")) for part in (content.get("parts") or []) if part.get("text")) +def _normalized_query(query: str) -> str: + """归一化用于审计匹配的 query：去首尾空白并折叠连续空白。 + + 优化器可能对 query 做 trim/换行重排等轻量改写，完全严格的文本匹配 + 会让审计直接杀死整个 optimize 流程，得不偿失。 + """ + + return " ".join(query.split()) + + def _case_contexts(*paths: Path) -> dict[str, list[dict[str, Any]]]: """Index evaluation metadata by query without treating query as eval_id. @@ -129,7 +139,7 @@ def _case_contexts(*paths: Path) -> dict[str, list[dict[str, Any]]]: query = _content_text(invocation.get("user_content") or {}) if not query: continue - contexts.setdefault(query, []).append( + contexts.setdefault(_normalized_query(query), []).append( { "eval_id": case["eval_id"], "split": ((case.get("session_input") or {}).get("state") or {}).get("split"), @@ -146,9 +156,10 @@ def _audited_call_agent( audit: list[dict[str, Any]], ): async def call_agent(query: str) -> str: - matching_contexts = contexts.get(query) - if not matching_contexts: - raise KeyError(f"call_agent 收到未登记的评测 query：{query!r}") + # 未登记的 query 降级为空上下文继续执行并在审计中标记， + # 而不是抛错中断整个优化流程：审计是观察手段，不应成为故障点。 + matching_contexts = contexts.get(_normalized_query(query)) or [] + context_match = "matched" if matching_contexts else "unmatched" try: response = await base_call_agent(query) except BaseException as error: @@ -156,6 +167,7 @@ async def call_agent(query: str) -> str: { "query": query, "eval_contexts": matching_contexts, + "context_match": context_match, "status": "error", "error": f"{type(error).__name__}: {error}", } @@ -165,6 +177,7 @@ async def call_agent(query: str) -> str: { "query": query, "eval_contexts": matching_contexts, + "context_match": context_match, "status": "ok", "actual_response": response, } diff --git a/examples/optimization/eval_optimize_loop/tests/test_audit_robustness.py b/examples/optimization/eval_optimize_loop/tests/test_audit_robustness.py new file mode 100644 index 000000000..151662fff --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_audit_robustness.py @@ -0,0 +1,99 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""验证审计回调对改写 query 的容错、事件文本拼接的健壮性与 GateConfig 边界。""" + +from __future__ import annotations + +import pytest + +import gates +import live_agent +import pipeline + + +async def _echo(query: str) -> str: + return f"reply:{query}" + + +@pytest.mark.asyncio +async def test_audited_call_agent_matches_trimmed_query() -> None: + """优化器对 query 做 trim/换行重排后仍应命中登记的上下文。""" + + contexts = {pipeline._normalized_query("小明有 3 个苹果"): [{"eval_id": "case_1"}]} + audit: list[dict] = [] + call_agent = pipeline._audited_call_agent(_echo, contexts, audit) + + await call_agent(" 小明有 3 个苹果 \n") + + assert audit[0]["context_match"] == "matched" + assert audit[0]["eval_contexts"] == [{"eval_id": "case_1"}] + assert audit[0]["status"] == "ok" + + +@pytest.mark.asyncio +async def test_audited_call_agent_degrades_on_unknown_query() -> None: + """未登记的 query 不应抛错中断优化流程，而是空上下文继续并标记。""" + + audit: list[dict] = [] + call_agent = pipeline._audited_call_agent(_echo, {}, audit) + + response = await call_agent("完全陌生的模板包裹 query") + + assert response == "reply:完全陌生的模板包裹 query" + assert audit[0]["context_match"] == "unmatched" + assert audit[0]["eval_contexts"] == [] + + +class _PartNoThought: + """模拟不带 thought 属性的 provider Part。""" + + def __init__(self, text: str): + self.text = text + + +class _PartWithThought: + def __init__(self, text: str, thought: bool): + self.text = text + self.thought = thought + + +class _Content: + def __init__(self, parts): + self.parts = parts + + +class _Event: + def __init__(self, parts, final: bool = True): + self.content = _Content(parts) + self._final = final + + def is_final_response(self) -> bool: + return self._final + + +def test_response_text_tolerates_parts_without_thought_attr() -> None: + """Part 完全没有 thought 属性时不应抛 AttributeError。""" + + event = _Event([_PartNoThought("你好"), _PartNoThought("世界")]) + assert live_agent.response_text_from_event(event) == "你好世界" + + +def test_response_text_skips_thought_parts_and_non_final_events() -> None: + event = _Event([_PartWithThought("内心推理", True), _PartWithThought("答案：10", False)]) + assert live_agent.response_text_from_event(event) == "答案：10" + assert live_agent.response_text_from_event(_Event([_PartNoThought("忽略")], final=False)) == "" + + +def test_gate_config_rejects_nonpositive_budget() -> None: + """budget_tokens<=0 会让 G6 把任意 measured 成本误判为超预算，必须拒绝。""" + + with pytest.raises(ValueError, match="budget_tokens"): + gates.GateConfig(budget_tokens=0) + with pytest.raises(ValueError, match="budget_usd"): + gates.GateConfig(budget_usd=-1.0) + assert gates.GateConfig().budget_tokens == 100_000