diff --git a/.gitignore b/.gitignore index 58eb6b48a..0cacf109e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ pyrightconfig.json # spec-workflow tool artifacts .spec-workflow + +.claude/settings.local.json diff --git a/examples/optimization/eval_optimize_loop/.gitignore b/examples/optimization/eval_optimize_loop/.gitignore new file mode 100644 index 000000000..60f8d1221 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/.gitignore @@ -0,0 +1,12 @@ +# Runtime artifact output +runs/ +optimization_report.json +optimization_report.md + +# Local model credentials (never commit) +.env + +# Python +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 000000000..1d3ffdb32 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,33 @@ +# 方案设计（Issue #91：Evaluation + Optimization 闭环） + +## Champion–Challenger 与数据隔离 + +管线把当前源 prompt 定义为 Champion，把 `AgentOptimizer.optimize(update_source=False)` +产出的最佳 prompt（或 fake 模式下的显式候选）定义为 Challenger。训练集只供优化器发现 +问题；验证集是独立裁判，只参与回归与门禁，失败细节不回喂优化器。fake 模式由 prompt 内 +公开的 `FAKE_CONTROLS` 控制项生成可评测轨迹，仅验证流程本身，不证明泛化能力。 + +## 失败归因方法 + +归因完全基于 evaluator 证据，不读场景标签：先用 `error_message` / `NOT_EVALUATED` +区分 infrastructure_failure；再按失败 metric 名与 reason 识别 knowledge_fail / +rubric_fail；然后对比期望与实际工具轨迹区分 tool_call_error（缺调用或调错工具）与 +param_error（同名工具参数不一致，附参数 diff）；最后对比 actual/expected 文本，核心 +数值一致判 format_fail，否则 reply_mismatch；证据不足时输出 insufficient_evidence， +不臆造类别。每个失败 case 至少给出一条可解释原因和原始证据。 + +## 接受策略与防过拟合 + +Gate 为纯规则、逐条单测：G1 验证集提升 ≥ min_val_lift；G2 train 升而 val 降视为过拟 +合，直接拒绝；G3 不新增 high-risk hard fail；G4 protected case 不退化；G5 slice 平均 +退化不超 tolerance；G6 成本证据完整且不超预算，成本未知一律禁止自动接受；G7 epsilon +内的微小波动不算有效提升。防过拟合由 train/val 物理隔离 + G2 + G4/G5 共同保证。 + +## 产物审计 + +每次运行生成唯一 `runs/-<随机后缀>/` 目录，`frozen.json` 冻结 prompt、数据 +集、评测/优化/gate 配置与随机种子的 sha256；落盘 Champion/Challenger 快照、完整 +evaluator 结果、优化轮次、调用审计、耗时与成本。`optimization_report.{json,md}` 记录 +baseline / candidate 分数、逐 case delta、归因统计、gate 决策与理由、复现命令。默认 +dry-run；仅 ACCEPT 且显式 `--apply` 时经 `TargetPrompt` 原子写回，优化失败也会落盘可 +审计的 REJECT 报告。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 000000000..a3c08ea84 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,78 @@ +# Evaluation + Optimization 闭环示例 + +本示例把 `AgentEvaluator` 与 `AgentOptimizer` 串成可复现的 +Champion–Challenger 流程：冻结输入、评测基线、归因失败、生成候选、 +重新评测、执行 Gate，并保存 JSON/Markdown 报告。默认 dry-run；只有 +显式传入 `--apply` 且全部 Gate 通过时才更新源 Prompt。 + +## 无 API Key 的管线回归 + +```powershell +python .\examples\optimization\eval_optimize_loop\pipeline.py --mode fake --scenario success +python .\examples\optimization\eval_optimize_loop\pipeline.py --mode fake --scenario no_effect +python .\examples\optimization\eval_optimize_loop\pipeline.py --mode fake --scenario overfit +``` + +| 场景 | 明文 `FAKE_CONTROLS` | 预期结果 | +|---|---|---| +| `success` | `ADD_STEPS=true` | train/val 均提升，ACCEPT | +| `no_effect` | 两项均为 `false` | 无有效提升，REJECT G1 | +| `overfit` | `MEMORIZE_TRAIN=true` | train 提升、val 退化，REJECT G2 | + +Fake 模式根据 Prompt 内的公开控制项生成 trace，再交给真实 +`AgentEvaluator` 评分；它只用于确定性管线回归，不证明隐藏样本泛化能力， +也不替代原生优化器。 + +## 原生 optimize + +```powershell +$env:TRPC_AGENT_API_KEY="..." +$env:TRPC_AGENT_BASE_URL="https://..." +$env:TRPC_AGENT_MODEL_NAME="..." +python .\examples\optimization\eval_optimize_loop\pipeline.py --mode optimize --optimizer-config optimizer.json +``` + +该路径使用真实模型回调并实际调用 +`AgentOptimizer.optimize(..., update_source=False)`。Optimizer 仅看到训练集； +返回的 `best_prompts["system"]` 作为 Challenger，再用同一 train/val 数据和 +Gate 做回归。缺少配置、优化异常或未产出 Candidate 时仍会生成 +`OPTIMIZER_FAILURE + G6` 的 REJECT 报告。模型端未返回可信 token/费用时， +报告写 `cost_status="unavailable"`、数值为 `null`，并禁止自动写回。 + +## Gate 与写回 + +G1 验证集最小提升；G2 训练涨而验证跌；G3 不新增高风险失败；G4 protected +case 不退化；G5 slice 退化不超阈值；G6 成本证据完整且不超预算；G7 +过滤 epsilon 内的微小变化。阈值见 `run.json`。 + +```powershell +# 只有 ACCEPT 才会通过 TargetPrompt 原子写回 +python .\examples\optimization\eval_optimize_loop\pipeline.py --mode fake --scenario success --apply +``` + +REJECT 加 `--apply` 返回退出码 2，Champion 保持不变。评测临时切换和测试恢复 +也统一使用 `TargetPrompt.read_all/write_all`。 + +## 数据、证据与产物 + +- `data/train.evalset.json`、`data/val.evalset.json`：各 3 条，eval_id 互斥。 +- `data/attribution_holdout.json`：公开标注证据集，不是官方隐藏集。 +- `optimization_report.json/.md`：最新决策及逐 case 证据。 +- `optimization_report.example.json`：已脱敏的报告结构示例。 +- `runs/-<随机后缀>/`：冻结清单、Prompt 快照、完整 evaluator + 结果、优化轮次、调用审计与报告；该目录被 Git 忽略。 + +逐 case 报告保存 actual/expected response、metric reason、tool use/response、 +参数差异和 trace 引用，并区分 agent-quality、infrastructure failure 与 +`insufficient_evidence`。官方隐藏集未提供，因此不声称满足隐藏样本准确率； +公开归因 holdout 的实际统计由测试计算。 + +## 验证 + +```powershell +python -m pytest .\examples\optimization\eval_optimize_loop\tests -q +python -m compileall -q .\examples\optimization\eval_optimize_loop +``` + +测试包含三种 fake 场景、mock native optimizer 完整闭环、真实 evaluator +工具/参数证据、成本不可用拒绝 apply、Prompt 恢复、Gate 和报告 schema。 diff --git a/examples/optimization/eval_optimize_loop/attribution.py b/examples/optimization/eval_optimize_loop/attribution.py new file mode 100644 index 000000000..88764ddc8 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/attribution.py @@ -0,0 +1,293 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Evidence-based failure attribution for evaluation cases.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Optional + +CATEGORIES = ( + "reply_mismatch", + "format_fail", + "tool_call_error", + "param_error", + "rubric_fail", + "knowledge_fail", + "infrastructure_failure", + "insufficient_evidence", + "none", +) + + +@dataclass(frozen=True) +class AttributionInput: + status: str + metric_name_failed: Optional[str] = None + metric_names_failed: tuple[str, ...] = () + metric_reasons: tuple[str, ...] = () + expected_text: Optional[str] = None + actual_text: Optional[str] = None + expected_tool_uses: tuple[dict[str, Any], ...] = () + actual_tool_uses: tuple[dict[str, Any], ...] = () + expected_tool_responses: tuple[dict[str, Any], ...] = () + actual_tool_responses: tuple[dict[str, Any], ...] = () + error_message: Optional[str] = None + # Backward-compatible fields used by older callers/tests. + scenario_tag: Optional[str] = None + score: float = 1.0 + has_tool_calls_actual: bool = False + has_tool_calls_expected: bool = False + has_tool_param_mismatch: bool = False + has_knowledge_response: bool = True + + +@dataclass(frozen=True) +class AttributionResult: + category: str + reason: str + evidence: dict[str, Any] = field(default_factory=dict) + + +def _tool_name(tool: dict[str, Any]) -> str: + return str(tool.get("name") or "") + + +def _tool_args(tool: dict[str, Any]) -> dict[str, Any]: + args = tool.get("args") + return args if isinstance(args, dict) else {} + + +def _tool_mismatch(inp: AttributionInput) -> tuple[bool, bool]: + expected = {_tool_name(tool): tool for tool in inp.expected_tool_uses if _tool_name(tool)} + actual = {_tool_name(tool): tool for tool in inp.actual_tool_uses if _tool_name(tool)} + if not expected: + return False, False + missing_or_wrong = bool(set(expected) - set(actual)) + parameter_error = any( + _tool_args(expected[name]) != _tool_args(actual[name]) for name in set(expected) & set(actual) + ) + return missing_or_wrong, parameter_error + + +def _parameter_differences( + expected_tools: tuple[dict[str, Any], ...], + actual_tools: tuple[dict[str, Any], ...], +) -> list[dict[str, Any]]: + expected = {_tool_name(tool): tool for tool in expected_tools if _tool_name(tool)} + actual = {_tool_name(tool): tool for tool in actual_tools if _tool_name(tool)} + return [ + { + "tool_name": name, + "expected_args": _tool_args(expected[name]), + "actual_args": _tool_args(actual[name]), + } + for name in sorted(set(expected) & set(actual)) + if _tool_args(expected[name]) != _tool_args(actual[name]) + ] + + +def _failed_metric_names(inp: AttributionInput) -> tuple[str, ...]: + names = list(inp.metric_names_failed) + if inp.metric_name_failed and inp.metric_name_failed not in names: + names.append(inp.metric_name_failed) + return tuple(str(name) for name in names) + + +def classify_with_reason(inp: AttributionInput) -> AttributionResult: + """Classify one failure without relying on scenario labels.""" + + if inp.status == "PASSED": + return AttributionResult("none", "case 已通过，无失败归因。") + + if inp.error_message or inp.status == "NOT_EVALUATED": + reason = inp.error_message or "评测未完成，缺少可用的 agent 输出。" + return AttributionResult( + "infrastructure_failure", + f"基础设施或评测执行失败：{reason}", + {"error_message": inp.error_message, "status": inp.status}, + ) + + metric_names = _failed_metric_names(inp) + lowered_metrics = tuple(name.lower() for name in metric_names) + reasons_text = " ".join(inp.metric_reasons).lower() + + if any("knowledge" in name for name in lowered_metrics) or any( + marker in reasons_text for marker in ("knowledge", "recall", "知识", "召回") + ): + return AttributionResult( + "knowledge_fail", + "知识召回证据未达到评测要求。", + {"failed_metrics": metric_names, "metric_reasons": inp.metric_reasons}, + ) + + if any("rubric" in name for name in lowered_metrics): + return AttributionResult( + "rubric_fail", + "LLM rubric 指标未达到阈值。", + {"failed_metrics": metric_names, "metric_reasons": inp.metric_reasons}, + ) + + expected_tools = inp.expected_tool_uses + actual_tools = inp.actual_tool_uses + if inp.has_tool_calls_expected and not expected_tools: + expected_tools = ({"name": ""},) + if inp.has_tool_calls_actual and not actual_tools: + actual_tools = ({"name": ""},) + normalized = AttributionInput( + **{ + **inp.__dict__, + "expected_tool_uses": expected_tools, + "actual_tool_uses": actual_tools, + } + ) + missing_or_wrong, parameter_error = _tool_mismatch(normalized) + if inp.has_tool_param_mismatch: + parameter_error = True + if parameter_error: + return AttributionResult( + "param_error", + "工具名称匹配，但调用参数与期望轨迹不一致。", + { + "expected_tool_uses": expected_tools, + "actual_tool_uses": actual_tools, + "parameter_differences": _parameter_differences(expected_tools, actual_tools), + }, + ) + if missing_or_wrong or (expected_tools and not actual_tools): + return AttributionResult( + "tool_call_error", + "缺少期望工具调用，或调用了错误的工具。", + {"expected_tool_uses": expected_tools, "actual_tool_uses": actual_tools}, + ) + + if inp.expected_text is not None and inp.actual_text is not None: + if _answer_numbers_match(inp.expected_text, inp.actual_text): + return AttributionResult( + "format_fail", + "答案核心数值一致，但回复格式或必要结构不符合要求。", + {"expected_text": inp.expected_text, "actual_text": inp.actual_text}, + ) + return AttributionResult( + "reply_mismatch", + "最终回复与参考答案不匹配。", + {"expected_text": inp.expected_text, "actual_text": inp.actual_text}, + ) + + return AttributionResult( + "insufficient_evidence", + "证据不足：缺少可核对的 actual/expected response 或工具轨迹，" "不能仅凭笼统 metric reason 可靠归因。", + { + "failed_metrics": metric_names, + "metric_reasons": inp.metric_reasons, + "status": inp.status, + }, + ) + + +def classify(inp: AttributionInput) -> str: + """Backward-compatible category-only API.""" + + return classify_with_reason(inp).category + + +def from_case_record(case: Any) -> AttributionResult: + failed_metrics = tuple( + str(metric.get("metric_name", "")) for metric in case.metric_results if metric.get("eval_status") != "PASSED" + ) + metric_reasons = tuple(str(reason) for reason in case.failure_reasons if reason) + return classify_with_reason( + AttributionInput( + status=case.challenger_status, + metric_names_failed=failed_metrics, + metric_reasons=metric_reasons, + expected_text=case.expected_text, + actual_text=case.actual_text, + expected_tool_uses=tuple(case.expected_tool_uses), + actual_tool_uses=tuple(case.actual_tool_uses), + expected_tool_responses=tuple(case.expected_tool_responses), + actual_tool_responses=tuple(case.actual_tool_responses), + error_message=case.error_message, + ) + ) + + +def from_eval_case_result( + result: Any, + *, + scenario_tag: Optional[str] = None, + expected_text: Optional[str] = None, +) -> str: + """Attribute directly from the public EvalCaseResult fields.""" + + failed_metrics: list[str] = [] + reasons: list[str] = [] + for metric in result.overall_eval_metric_results or []: + status = getattr(metric.eval_status, "name", str(metric.eval_status)) + if status != "PASSED": + failed_metrics.append(metric.metric_name) + details = getattr(metric, "details", None) + reason = getattr(details, "reason", None) if details else None + if reason: + reasons.append(str(reason)) + + actual_text: Optional[str] = None + expected_tools: list[dict[str, Any]] = [] + actual_tools: list[dict[str, Any]] = [] + for per_invocation in result.eval_metric_result_per_invocation or []: + actual = per_invocation.actual_invocation + expected = per_invocation.expected_invocation + if actual_text is None: + actual_text = _content_to_text(getattr(actual, "final_response", None)) + actual_tools.extend(_intermediate_tools(actual)) + if expected is not None: + expected_tools.extend(_intermediate_tools(expected)) + if expected_text is None: + expected_text = _content_to_text(getattr(expected, "final_response", None)) + + status = getattr(result.final_eval_status, "name", str(result.final_eval_status)) + return classify( + AttributionInput( + status=status, + metric_names_failed=tuple(failed_metrics), + metric_reasons=tuple(reasons), + expected_text=expected_text, + actual_text=actual_text, + expected_tool_uses=tuple(expected_tools), + actual_tool_uses=tuple(actual_tools), + error_message=result.error_message, + ) + ) + + +def _content_to_text(content: Any) -> Optional[str]: + parts = getattr(content, "parts", None) or [] + texts = [str(getattr(part, "text", "")) for part in parts if getattr(part, "text", None)] + return "\n".join(texts) if texts else None + + +def _intermediate_tools(invocation: Any) -> list[dict[str, Any]]: + intermediate = getattr(invocation, "intermediate_data", None) + tools = getattr(intermediate, "tool_uses", None) or [] + return [ + ( + tool.model_dump(mode="json", by_alias=True) + if hasattr(tool, "model_dump") + else {"name": getattr(tool, "name", ""), "args": getattr(tool, "args", {})} + ) + for tool in tools + ] + + +def _answer_numbers_match(expected: str, actual: str) -> bool: + expected_numbers = re.findall(r"-?\d+(?:\.\d+)?", expected) + actual_numbers = re.findall(r"-?\d+(?:\.\d+)?", actual) + if not expected_numbers or not actual_numbers: + return False + return expected_numbers[-1] == actual_numbers[-1] diff --git a/examples/optimization/eval_optimize_loop/data/attribution_holdout.json b/examples/optimization/eval_optimize_loop/data/attribution_holdout.json new file mode 100644 index 000000000..05c28999c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/attribution_holdout.json @@ -0,0 +1,73 @@ +{ + "description": "公开、可复现的人工标注 evaluator-evidence holdout；不是官方隐藏集。", + "cases": [ + { + "eval_id": "holdout_reply", + "expected_category": "reply_mismatch", + "status": "FAILED", + "failed_metrics": ["final_response_avg_score"], + "metric_reasons": ["final response text mismatch"], + "expected_text": "答案：11", + "actual_text": "答案：22" + }, + { + "eval_id": "holdout_format", + "expected_category": "format_fail", + "status": "FAILED", + "failed_metrics": ["final_response_avg_score"], + "metric_reasons": ["required answer structure is missing"], + "expected_text": "步骤：4 + 7 = 11\n答案：11 个", + "actual_text": "11" + }, + { + "eval_id": "holdout_tool", + "expected_category": "tool_call_error", + "status": "FAILED", + "failed_metrics": ["tool_trajectory_avg_score"], + "metric_reasons": ["expected tool call was not found"], + "expected_tool_uses": [{"id": "t1", "name": "search", "args": {"query": "pricing"}}], + "actual_tool_uses": [] + }, + { + "eval_id": "holdout_param", + "expected_category": "param_error", + "status": "FAILED", + "failed_metrics": ["tool_trajectory_avg_score"], + "metric_reasons": ["tool arguments did not match"], + "expected_tool_uses": [{"id": "t1", "name": "search", "args": {"query": "pricing", "top_k": 3}}], + "actual_tool_uses": [{"id": "a1", "name": "search", "args": {"query": "weather", "top_k": 1}}] + }, + { + "eval_id": "holdout_rubric", + "expected_category": "rubric_fail", + "status": "FAILED", + "failed_metrics": ["llm_rubric_response"], + "metric_reasons": ["rubric reasoning_clear failed"], + "expected_text": "清晰解释计算过程", + "actual_text": "答案是 20" + }, + { + "eval_id": "holdout_knowledge", + "expected_category": "knowledge_fail", + "status": "FAILED", + "failed_metrics": ["llm_rubric_knowledge_recall"], + "metric_reasons": ["knowledge recall does not support rubric kr1"], + "expected_tool_uses": [{"id": "t1", "name": "knowledge_search", "args": {"query": "产品"}}], + "actual_tool_uses": [{"id": "a1", "name": "knowledge_search", "args": {"query": "产品"}}], + "actual_tool_responses": [{"id": "a1", "name": "knowledge_search", "response": {"documents": []}}] + }, + { + "eval_id": "holdout_infra", + "expected_category": "infrastructure_failure", + "status": "NOT_EVALUATED", + "error_message": "TimeoutError: model endpoint unavailable" + }, + { + "eval_id": "holdout_insufficient", + "expected_category": "insufficient_evidence", + "status": "FAILED", + "failed_metrics": ["final_response_avg_score"], + "metric_reasons": ["score below threshold"] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/test_config.json b/examples/optimization/eval_optimize_loop/data/test_config.json new file mode 100644 index 000000000..2fd719a93 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/test_config.json @@ -0,0 +1,17 @@ +{ + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "contains", + "case_insensitive": true + } + } + } + } + ], + "num_runs": 1 +} diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json new file mode 100644 index 000000000..de38a250c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -0,0 +1,88 @@ +{ + "eval_set_id": "eval_optimize_loop_train", + "name": "eval_optimize_loop 训练集（3 条）", + "description": "用于优化器发现问题。覆盖三种可优化情景：可优化成功 (T-ADD)、优化无效 (T-NOOP)、优化后退退 (T-OVERFIT-MEM)。", + "eval_cases": [ + { + "eval_id": "train_add_steps", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "train", + "state": { + "split": "train", + "slice": "math", + "risk_level": "low", + "protected": false, + "scenario_tag": "add_steps" + } + }, + "conversation": [ + { + "invocation_id": "t1", + "user_content": { + "parts": [{"text": "小明买了 4 个苹果又买了 7 个苹果，他一共有多少个苹果？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "步骤：4 + 7 = 11\n答案：11 个"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "train_noop", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "train", + "state": { + "split": "train", + "slice": "noop", + "risk_level": "low", + "protected": false, + "scenario_tag": "noop_ambiguous" + } + }, + "conversation": [ + { + "invocation_id": "t2", + "user_content": { + "parts": [{"text": "一个含糊不清的问题，候选 prompt 无论怎么改都答不对。"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：42"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "train_overfit_mem", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "train", + "state": { + "split": "train", + "slice": "math", + "risk_level": "low", + "protected": false, + "scenario_tag": "memorize_train" + } + }, + "conversation": [ + { + "invocation_id": "t3", + "user_content": { + "parts": [{"text": "一件衣服原价 200 元，打 8 折，折后价是多少？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：160 元"}], + "role": "model" + } + } + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json new file mode 100644 index 000000000..b39acbfc1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -0,0 +1,88 @@ +{ + "eval_set_id": "eval_optimize_loop_val", + "name": "eval_optimize_loop 验证集（3 条，独立裁判）", + "description": "eval_id 与 train 互斥。包含一条 protected case、一条 high risk case、一条 noop。", + "eval_cases": [ + { + "eval_id": "val_add_steps_b", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "val", + "state": { + "split": "val", + "slice": "math", + "risk_level": "low", + "protected": false, + "scenario_tag": "add_steps" + } + }, + "conversation": [ + { + "invocation_id": "v1", + "user_content": { + "parts": [{"text": "小轿车一共有 4 个轮子，5 辆小轿车一共有多少个轮子？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "步骤：4 × 5 = 20\n答案：20 个"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "val_protected_highrisk", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "val", + "state": { + "split": "val", + "slice": "risky", + "risk_level": "high", + "protected": true, + "scenario_tag": "protected_must_pass" + } + }, + "conversation": [ + { + "invocation_id": "v2", + "user_content": { + "parts": [{"text": "班里 40 名学生，25% 戴眼镜，戴眼镜的有多少人？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "步骤：40 × 25% = 10\n答案：10 人"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "val_noop_b", + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "val", + "state": { + "split": "val", + "slice": "noop", + "risk_level": "low", + "protected": false, + "scenario_tag": "noop_ambiguous" + } + }, + "conversation": [ + { + "invocation_id": "v3", + "user_content": { + "parts": [{"text": "另一个含糊不清的问题，候选 prompt 无论怎么改都答不对。"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：42"}], + "role": "model" + } + } + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/fake_agent.py b/examples/optimization/eval_optimize_loop/fake_agent.py new file mode 100644 index 000000000..1abc61ea1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake_agent.py @@ -0,0 +1,221 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Milestone A 的 fake agent。 + +行为**完全由 prompt 中公开、明确的 FAKE_CONTROLS 控制块**决定： + + + +只有该块内、且值严格等于 ``true``（大小写敏感）才启用对应行为。 +注释文字、文档叙述、示例代码里出现的同名关键字**不会**误触发。 + +决策矩阵（与 README 真值表一致）： + +| 控制块 | train case | val protected | val 其他 | +|---------------------------------|------------|---------------|----------| +| 全 false（baseline） | 错 | 对 | 错 | +| ADD_STEPS=true | 对 | 对 | 对 | +| MEMORIZE_TRAIN=true | 对 | 错 | 错 | + +- "对" = 输出 evalset 中该 case 的 ``final_response`` 原文（满足 contains 评测） +- "错" = 输出固定错误文本 + +成功 case 的输出**直接来自 evalset 中 case.conversation[0].final_response**， +避免与 contains 评测再次漂移。 + +禁止行为（test_no_sha_switch 强制）： + - 通过任何形式的哈希指纹选择预录答案 + - 通过身份关键字偏置结果 + - 读取 scenario 参数 / eval_id 内容决定答案 +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Optional + +# ---- FAKE_CONTROLS 控制块解析 ---- +# 多行块： 之间按 `KEY=value` 行解析 +_FAKE_CONTROLS_RE = re.compile( + r"", + re.DOTALL | re.IGNORECASE, +) +_KV_RE = re.compile(r"^\s*([A-Z_]+)\s*=\s*(\S+)\s*$") + + +@dataclass(frozen=True) +class FakeBehavior: + """根据 FAKE_CONTROLS 块推断出的行为集合。 + + 只有严格等于字符串 "true" 才视为启用，避免 "false" / "True" / 注释文字误触发。 + """ + + add_steps: bool = False + memorize_train: bool = False + + +def parse_behavior(prompt_text: str) -> FakeBehavior: + """从 prompt 文本中提取 FAKE_CONTROLS 块并解析。 + + - 不存在控制块 → baseline（全 false） + - 存在控制块 → 仅接受 ``KEY=true``（严格小写 true）作为启用 + - 其他值（false / True / 注释叙述）一律视为 false + + >>> parse_behavior("no block here").add_steps + False + >>> parse_behavior("").add_steps + True + >>> parse_behavior("").add_steps + False + """ + add_steps = False + memorize_train = False + for match in _FAKE_CONTROLS_RE.finditer(prompt_text or ""): + body = match.group(1) + for line in body.splitlines(): + kv = _KV_RE.match(line) + if not kv: + continue + key, val = kv.group(1), kv.group(2) + enabled = val == "true" + if key == "ADD_STEPS": + add_steps = enabled or add_steps + elif key == "MEMORIZE_TRAIN": + memorize_train = enabled or memorize_train + return FakeBehavior(add_steps=add_steps, memorize_train=memorize_train) + + +# 错误文本（与所有 expected_response 都不含的关键短语，确保 contains 失败） +_WRONG_TEXT = "抱歉，我无法回答这个问题。" + + +def _expected_text_from_case(case: dict) -> Optional[str]: + """从 evalset 单个 case 中取出 reference final_response 的纯文本。""" + conv = case.get("conversation") or [] + if not conv: + return None + final = conv[0].get("final_response") or {} + parts = final.get("parts") or [] + texts = [p.get("text", "") for p in parts if p.get("text")] + return "\n".join(texts) if texts else None + + +def _is_protected(case: dict) -> bool: + state = (case.get("session_input") or {}).get("state") or {} + return bool(state.get("protected", False)) + + +def _is_train(case: dict) -> bool: + state = (case.get("session_input") or {}).get("state") or {} + return state.get("split") == "train" + + +def gen_final_response_for_case( + prompt_text: str, + *, + case: dict, +) -> str: + """根据当前 prompt 行为 + evalset 中的 case，返回最终回复文本。 + + 决策完全基于 FAKE_CONTROLS 块 + case 的 split/protected 字段； + 不读取 scenario 名、eval_id 内容、prompt hash、自然语言 query。 + """ + behavior = parse_behavior(prompt_text) + expected = _expected_text_from_case(case) or "" + is_train = _is_train(case) + is_protected = _is_protected(case) + + if behavior.add_steps: + return expected + if behavior.memorize_train: + return expected if is_train else _WRONG_TEXT + # baseline + return expected if is_protected else _WRONG_TEXT + + +def gen_actual_conversation(prompt_text: str, evalset_dict: dict) -> list: + """对一个 evalset 中的每个 case 生成 actual_conversation，返回新的 eval_cases list.""" + new_cases = [] + for case in evalset_dict.get("eval_cases", []): + eval_id = case["eval_id"] + conv = case.get("conversation") or case.get("actual_conversation") or [] + if not conv: + continue + first_inv = conv[0] + + actual_text = gen_final_response_for_case(prompt_text, case=case) + actual_invocation = { + "invocation_id": first_inv.get("invocation_id", "act-1"), + "user_content": first_inv["user_content"], + "final_response": { + "parts": [{"text": actual_text}], + "role": "model", + }, + } + + new_case = dict(case) + new_case["eval_mode"] = "trace" + new_case["actual_conversation"] = [actual_invocation] + # trace 模式下保留 conversation 作为 reference + if "conversation" not in new_case: + new_case["conversation"] = conv + new_cases.append(new_case) + return new_cases + + +# scenario → candidate prompt 模板。 +# 所有 candidate 通过显式 FAKE_CONTROLS 块声明行为，不靠自然语言文字。 +SUCCESS_CANDIDATE = """# System Prompt (Candidate - Success) + +你是一个严谨的数学问答助手。 + +## 输出格式 +所有回答必须按"步骤：...\\n答案：..."的格式给出最终答案。 + + +""" + +NO_EFFECT_CANDIDATE = """# System Prompt (Candidate - NoEffect) + +你是一个数学问答助手。本候选不启用任何 fake 改进标记，行为与初始版本一致。 + + +""" + +OVERFIT_CANDIDATE = """# System Prompt (Candidate - Overfit) + +本候选只对训练集记忆答案，验证集（含受保护样本）会答错。 + + +""" + +CANDIDATES = { + "success": SUCCESS_CANDIDATE, + "no_effect": NO_EFFECT_CANDIDATE, + "overfit": OVERFIT_CANDIDATE, +} + + +def build_candidate(scenario: str) -> str: + """根据 scenario 名返回对应 candidate prompt 文本。""" + if scenario not in CANDIDATES: + raise ValueError(f"unknown scenario: {scenario}, expected one of {list(CANDIDATES)}") + return CANDIDATES[scenario] diff --git a/examples/optimization/eval_optimize_loop/gates.py b/examples/optimization/eval_optimize_loop/gates.py new file mode 100644 index 000000000..1790499ed --- /dev/null +++ b/examples/optimization/eval_optimize_loop/gates.py @@ -0,0 +1,186 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Gate：纯规则决策，输入两个 split 的 case-level delta，输出 ACCEPT/REJECT。 + +对应 Issue #91 的 G1-G7 接受策略： + + G1 validation 最小有效提升 val_delta >= min_val_lift + G2 train 涨 val 跌 = 过拟合 not (train_delta > min_val_lift and val_delta < -min_val_lift) + G3 不新增 hard fail risk_level==high 的 case 不能从 PASSED→FAILED + G4 protected case 不退化 protected==true 的 case 不能分数下降 + G5 slice 平均退化不超过 tolerance 每个 slice 平均 delta >= -slice_tol + G6 成本证据完整 cost_status != "unavailable" and total_tokens <= budget + G7 明显微小变化不算有效提升 候选接受时，关键 delta 必须 > epsilon + +不写互相重复的规则：G1 看 val 整体；G5 看 slice；G3/G4 看个体风险；G6 看成本；G7 防 epsilon 内噪声。 +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Iterable, List, Optional + + +@dataclass(frozen=True) +class CaseDelta: + """单 case 的对比信息。 + + Attributes: + eval_id: case id + split: train | val + slice_name: 来自 state.slice + risk_level: low | mid | high + protected: bool + champion_status: PASSED | FAILED + challenger_status: PASSED | FAILED + champion_score: 0~1 + challenger_score: 0~1 + delta: challenger - champion + """ + + eval_id: str + split: str + slice_name: str + risk_level: str = "low" + protected: bool = False + champion_status: str = "FAILED" + challenger_status: str = "FAILED" + champion_score: float = 0.0 + challenger_score: float = 0.0 + + @property + def delta(self) -> float: + return self.challenger_score - self.champion_score + + @property + def new_hard_fail(self) -> bool: + """risk_level==high 且 champion 通过但 challenger 失败。""" + return self.risk_level == "high" and self.champion_status == "PASSED" and self.challenger_status != "PASSED" + + +@dataclass +class GateConfig: + """Gate 阈值。所有字段必须可从 run.json 反序列化。""" + + min_val_lift: float = 0.02 + slice_tolerance: float = 0.05 + budget_tokens: int = 100_000 + budget_usd: Optional[float] = None + epsilon: float = 0.001 + + def __post_init__(self) -> None: + # fake/trace 模式的 measured 成本恒为 0 token；若允许 budget_tokens<=0， + # G6 会把任意合法成本误判为超预算。 + if self.budget_tokens <= 0: + raise ValueError(f"budget_tokens 必须为正整数，当前值：{self.budget_tokens}") + if self.budget_usd is not None and self.budget_usd <= 0: + raise ValueError(f"budget_usd 配置时必须为正数，当前值：{self.budget_usd}") + + +@dataclass +class Decision: + accepted: bool + violated: List[str] = field(default_factory=list) + reasons: List[str] = field(default_factory=list) + + +def _avg(values: Iterable[float]) -> float: + vs = list(values) + return sum(vs) / len(vs) if vs else 0.0 + + +def _slice_avg_deltas(cases: List[CaseDelta]): + """按 slice 聚合 delta，返回 {slice_name: avg_delta}。""" + groups: dict[str, list[float]] = {} + for c in cases: + groups.setdefault(c.slice_name, []).append(c.delta) + return {k: _avg(v) for k, v in groups.items()} + + +def evaluate( + cases: List[CaseDelta], + *, + cost_status: str = "measured", + total_tokens: Optional[int] = 0, + total_cost: Optional[float] = 0.0, + config: Optional[GateConfig] = None, +) -> Decision: + """主入口。 + + Args: + cases: train + val 两个 split 全部 case 的 delta（合并传入） + cost_status: measured / unavailable + total_tokens: 实际消耗 token；fake/trace 模式为 0 + config: gate 阈值，None 用默认 + """ + cfg = config or GateConfig() + violated: List[str] = [] + reasons: List[str] = [] + + train_cases = [c for c in cases if c.split == "train"] + val_cases = [c for c in cases if c.split == "val"] + train_delta = _avg(c.delta for c in train_cases) + val_delta = _avg(c.delta for c in val_cases) + + # G2 必须先于 G1：先识别"训练涨 / val 跌"的过拟合，再判 val 是否涨够。 + # G2: overfit + if train_delta > cfg.min_val_lift and val_delta < -cfg.min_val_lift: + violated.append("G2") + reasons.append( + f"过拟合：train_delta={train_delta:+.4f} 上升而 " f"val_delta={val_delta:+.4f} 下降，候选只记住训练集。" + ) + + # G1: val minimal lift + if val_delta < cfg.min_val_lift: + violated.append("G1") + reasons.append(f"验证集无有效提升：val_delta={val_delta:+.4f} < " f"min_val_lift={cfg.min_val_lift:.4f}。") + + # G3: no new hard fail + new_hard = [c.eval_id for c in cases if c.new_hard_fail] + if new_hard: + violated.append("G3") + reasons.append(f"新增 hard fail（risk_level==high 由 PASSED→FAILED）：{new_hard}。") + + # G4: protected case no regression + protected_reg = [c.eval_id for c in cases if c.protected and c.delta < -cfg.epsilon] + if protected_reg: + violated.append("G4") + reasons.append(f"protected case 退化：{protected_reg}。") + + # G5: slice tolerance + slice_deltas = _slice_avg_deltas(cases) + bad_slices = {k: v for k, v in slice_deltas.items() if v < -cfg.slice_tolerance} + if bad_slices: + violated.append("G5") + rounded = {k: round(v, 4) for k, v in bad_slices.items()} + reasons.append(f"slice 退化超过 tolerance={cfg.slice_tolerance:.4f}：{rounded}。") + + # G6: cost evidence + if cost_status == "unavailable": + violated.append("G6") + reasons.append("成本证据缺失（cost_status=unavailable），不可自动 ACCEPT。") + elif total_tokens is None or total_cost is None: + violated.append("G6") + reasons.append("成本状态标为 measured，但 token/cost 证据为空，不可自动 ACCEPT。") + elif total_tokens > cfg.budget_tokens: + violated.append("G6") + reasons.append(f"超成本预算：total_tokens={total_tokens} > " f"budget_tokens={cfg.budget_tokens}。") + elif cfg.budget_usd is not None and total_cost > cfg.budget_usd: + violated.append("G6") + reasons.append(f"超成本预算：total_cost={total_cost:.6f} > " f"budget_usd={cfg.budget_usd:.6f}。") + + # G7: epsilon guard —— 候选接受时，val_delta 必须严格 > epsilon（非噪声） + # 仅在前面 6 条全过时才检查；它防的是 "val_delta==0.0001" 这种几乎为零的波动 + # 被 G1 当作有效提升（因为 G1 用 >= min_val_lift 默认 0.02 已经隐含保护， + # 但当用户把 min_val_lift 调到 0 时 G7 仍是最后一道闸）。 + if not violated and val_delta <= cfg.epsilon: + violated.append("G7") + reasons.append(f"val_delta={val_delta:+.4f} ≤ epsilon={cfg.epsilon:.4f}，" "属于明显微小变化，不视为有效提升。") + + accepted = not violated + return Decision(accepted=accepted, violated=violated, reasons=reasons) diff --git a/examples/optimization/eval_optimize_loop/live_agent.py b/examples/optimization/eval_optimize_loop/live_agent.py new file mode 100644 index 000000000..fb035cede --- /dev/null +++ b/examples/optimization/eval_optimize_loop/live_agent.py @@ -0,0 +1,115 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Real model-backed call_agent used by native AgentOptimizer mode.""" + +from __future__ import annotations + +import os +import uuid +from pathlib import Path +from typing import Any, Awaitable, Callable + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.models import OpenAIModel +from trpc_agent_sdk.runners import Runner +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.types import Content, GenerateContentConfig, Part + +CallAgent = Callable[[str], Awaitable[str]] +APP_NAME = "eval_optimize_loop" + + +def model_info_from_env() -> dict[str, Any]: + """Return non-secret model identity for the frozen manifest.""" + + return { + "provider": os.environ.get("TRPC_AGENT_PROVIDER_NAME", "openai"), + "model_name": os.environ.get("TRPC_AGENT_MODEL_NAME", ""), + # Do not persist an endpoint that might embed credentials. + "base_url_configured": bool(os.environ.get("TRPC_AGENT_BASE_URL")), + "usage_tracking": False, + } + + +def validate_model_env() -> None: + missing = [ + key for key in ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME") if not os.environ.get(key) + ] + if missing: + raise RuntimeError( + "native optimize 需要模型配置：" + + ", ".join(f"${key}" for key in missing) + + "；无需 API Key 的回归演示请使用 --mode fake。" + ) + + +def response_text_from_event(event: Any) -> str: + """Collect non-thought text from a final-response event. + + Uses ``getattr`` so parts from any provider adapter are safe even when + they carry no ``thought``/``text`` attribute at all. + """ + + if not event.is_final_response() or not event.content: + return "" + texts = [] + for part in getattr(event.content, "parts", None) or []: + text = getattr(part, "text", None) + if text and not getattr(part, "thought", None): + texts.append(text) + return "".join(texts) + + +def build_call_agent(prompt_path: Path) -> CallAgent: + """Build a callback that reloads the current TargetPrompt on every call.""" + + validate_model_env() + api_key = os.environ["TRPC_AGENT_API_KEY"] + base_url = os.environ["TRPC_AGENT_BASE_URL"] + model_name = os.environ["TRPC_AGENT_MODEL_NAME"] + + async def call_agent(query: str) -> str: + instruction = prompt_path.read_text(encoding="utf-8") + model = OpenAIModel(model_name=model_name, api_key=api_key, base_url=base_url) + agent = LlmAgent( + name="eval_optimize_loop_agent", + description="Agent evaluated and optimized by Issue #91 pipeline.", + model=model, + instruction=instruction, + generate_content_config=GenerateContentConfig( + temperature=0.2, + top_p=0.9, + max_output_tokens=2048, + ), + ) + sessions = InMemorySessionService() + runner = Runner(app_name=APP_NAME, agent=agent, session_service=sessions) + session_id = str(uuid.uuid4()) + user_id = "optimizer" + await sessions.create_session( + app_name=APP_NAME, + user_id=user_id, + session_id=session_id, + state={}, + ) + message = Content(role="user", parts=[Part.from_text(text=query)]) + response = "" + async for event in runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=message, + ): + response += response_text_from_event(event) + return response.strip() + + # The SDK runner does not expose a provider-independent USD amount here. + # Treat the whole optimize run as unmeasured so G6 blocks auto-apply. + call_agent.cost_status = "unavailable" + call_agent.total_tokens = None + call_agent.total_cost = None + return call_agent diff --git a/examples/optimization/eval_optimize_loop/optimization_report.example.json b/examples/optimization/eval_optimize_loop/optimization_report.example.json new file mode 100644 index 000000000..d7255f48c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimization_report.example.json @@ -0,0 +1,345 @@ +{ + "version": "2.0", + "candidate_source": "agent_optimizer", + "frozen": { + "run_id": "2026-07-28T00-00-00.000000Z-a1b2c3d4", + "champion_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "challenger_sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "train_sha256": "3333333333333333333333333333333333333333333333333333333333333333", + "val_sha256": "4444444444444444444444444444444444444444444444444444444444444444", + "metric_config_sha256": "5555555555555555555555555555555555555555555555555555555555555555", + "run_config_sha256": "6666666666666666666666666666666666666666666666666666666666666666", + "optimizer_config_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "seed": 42, + "started_at": "2026-07-28T00:00:00.000Z", + "mode": "optimize", + "candidate_source": "agent_optimizer", + "scenario": null, + "gate_config": { + "min_val_lift": 0.02, + "slice_tolerance": 0.05, + "budget_tokens": 100000, + "budget_usd": null, + "epsilon": 0.001 + }, + "model_info": { + "provider": "test-stub", + "model_name": "injected-call-agent" + }, + "evaluator_info": { + "name": "AgentEvaluator", + "metric_config_sha256": "5555555555555555555555555555555555555555555555555555555555555555" + }, + "optimizer_info": { + "algorithm": "gepa_reflective", + "status": "SUCCEEDED", + "total_rounds": 1 + } + }, + "results": { + "train": { + "champion_avg": 0.0, + "challenger_avg": 1.0, + "delta": 1.0 + }, + "val": { + "champion_avg": 0.333333, + "challenger_avg": 1.0, + "delta": 0.666667 + } + }, + "train_delta": 1.0, + "val_delta": 0.666667, + "per_case": [ + { + "eval_id": "val_add_steps_b", + "split": "val", + "slice": "math", + "risk_level": "low", + "protected": false, + "scenario_tag": "add_steps", + "champion_status": "FAILED", + "challenger_status": "PASSED", + "champion_score": 0.0, + "challenger_score": 1.0, + "delta": 1.0, + "transition": "newly_passed", + "failure_kind": "none", + "category": "none", + "failure_reason": "case 已通过，无失败归因。", + "runs": 1, + "evidence": { + "actual_text": "步骤：4 × 5 = 20\n答案：20 个", + "expected_text": "步骤：4 × 5 = 20\n答案：20 个", + "metric_results": [ + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "eval_status": "PASSED", + "reason": null, + "rubric_scores": [] + } + ], + "actual_tool_uses": [], + "expected_tool_uses": [], + "actual_tool_responses": [], + "expected_tool_responses": [], + "error_message": null, + "metric_reasons": [], + "trace_ref": "val_eval.json#eval_id=val_add_steps_b", + "attribution": {} + } + }, + { + "eval_id": "evidence_tool_missing", + "split": "val", + "slice": "tool", + "risk_level": "low", + "protected": false, + "scenario_tag": null, + "champion_status": "FAILED", + "challenger_status": "FAILED", + "champion_score": 0.0, + "challenger_score": 0.0, + "delta": 0.0, + "transition": "unchanged", + "failure_kind": "agent_quality_failure", + "category": "tool_call_error", + "failure_reason": "缺少期望工具调用，或调用了错误的工具。", + "runs": 1, + "evidence": { + "actual_text": "已查询价格。", + "expected_text": "已查询价格。", + "metric_results": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": "expected tool call was not found", + "rubric_scores": [] + } + ], + "actual_tool_uses": [], + "expected_tool_uses": [ + { + "id": "t1", + "name": "search", + "args": { + "query": "pricing" + } + } + ], + "actual_tool_responses": [], + "expected_tool_responses": [], + "error_message": null, + "metric_reasons": [ + "expected tool call was not found" + ], + "trace_ref": "val_eval.json#eval_id=evidence_tool_missing", + "attribution": { + "expected_tool_uses": [ + { + "id": "t1", + "name": "search", + "args": { + "query": "pricing" + } + } + ], + "actual_tool_uses": [] + } + } + }, + { + "eval_id": "evidence_param_wrong", + "split": "val", + "slice": "tool", + "risk_level": "low", + "protected": false, + "scenario_tag": null, + "champion_status": "FAILED", + "challenger_status": "FAILED", + "champion_score": 0.0, + "challenger_score": 0.0, + "delta": 0.0, + "transition": "unchanged", + "failure_kind": "agent_quality_failure", + "category": "param_error", + "failure_reason": "工具名称匹配，但调用参数与期望轨迹不一致。", + "runs": 1, + "evidence": { + "actual_text": "已查询。", + "expected_text": "已查询。", + "metric_results": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": "tool arguments did not match", + "rubric_scores": [] + } + ], + "actual_tool_uses": [ + { + "id": "a1", + "name": "search", + "args": { + "query": "weather" + } + } + ], + "expected_tool_uses": [ + { + "id": "t1", + "name": "search", + "args": { + "query": "pricing" + } + } + ], + "actual_tool_responses": [], + "expected_tool_responses": [], + "error_message": null, + "metric_reasons": [ + "tool arguments did not match" + ], + "trace_ref": "val_eval.json#eval_id=evidence_param_wrong", + "attribution": { + "parameter_differences": [ + { + "tool_name": "search", + "expected_args": { + "query": "pricing" + }, + "actual_args": { + "query": "weather" + } + } + ] + } + } + }, + { + "eval_id": "evidence_rubric_failed", + "split": "val", + "slice": "quality", + "risk_level": "low", + "protected": false, + "scenario_tag": null, + "champion_status": "FAILED", + "challenger_status": "FAILED", + "champion_score": 0.0, + "challenger_score": 0.0, + "delta": 0.0, + "transition": "unchanged", + "failure_kind": "agent_quality_failure", + "category": "rubric_fail", + "failure_reason": "LLM rubric 指标未达到阈值。", + "runs": 1, + "evidence": { + "actual_text": "答案是 20。", + "expected_text": "清晰解释计算过程并给出答案。", + "metric_results": [ + { + "metric_name": "llm_rubric_response", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": "rubric reasoning_clear failed", + "rubric_scores": [ + { + "id": "reasoning_clear", + "score": 0.0, + "reason": "missing explanation" + } + ] + } + ], + "actual_tool_uses": [], + "expected_tool_uses": [], + "actual_tool_responses": [], + "expected_tool_responses": [], + "error_message": null, + "metric_reasons": [ + "rubric reasoning_clear failed" + ], + "trace_ref": "val_eval.json#eval_id=evidence_rubric_failed", + "attribution": { + "failed_metrics": [ + "llm_rubric_response" + ], + "metric_reasons": [ + "rubric reasoning_clear failed" + ] + } + } + } + ], + "transition_counts": { + "newly_passed": 1, + "unchanged": 3 + }, + "fail_category_counts": { + "none": 1, + "tool_call_error": 1, + "param_error": 1, + "rubric_fail": 1 + }, + "decision": { + "accepted": true, + "violated": [], + "reasons": [] + }, + "cost_status": "measured", + "cost": { + "status": "measured", + "total_tokens": 123, + "total_usd": 0.0123 + }, + "optimizer": { + "info": { + "algorithm": "gepa_reflective", + "status": "SUCCEEDED", + "total_rounds": 1 + }, + "rounds": [ + { + "round": 1, + "candidate_prompts": { + "system": "Example candidate prompt" + }, + "accepted": true, + "validation_pass_rate": 1.0, + "artifact_path": "runs//optimizer/rounds/round_001.json" + } + ], + "artifacts": { + "optimizer_result": "runs//optimizer/result.json", + "optimizer_best_prompt": "runs//optimizer/best_prompts/system.md" + } + }, + "audit": { + "run_id": "2026-07-28T00-00-00.000000Z-a1b2c3d4", + "applied": false, + "duration_seconds": 12.3456, + "candidate_source": "agent_optimizer", + "scenario": null, + "cost_status": "measured", + "cost": 0.0123, + "artifact_dir": "runs/", + "artifacts": { + "frozen_json": "runs//frozen.json", + "train_eval_log": "runs//train_eval.json", + "val_eval_log": "runs//val_eval.json" + }, + "before_apply_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "after_apply_sha256": null, + "repro_cmd": "python pipeline.py --mode optimize" + }, + "limitations": { + "hidden_sample_accuracy": "未使用官方隐藏集；公开 fixture 只用于可复现测试。" + } +} diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 000000000..238e9e1d0 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,45 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "contains", + "case_insensitive": true + } + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 2, + "stop": { + "required_metrics": ["final_response_avg_score"] + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "provider_name": "openai", + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "max_tokens": 2048, + "temperature": 0.4 + } + }, + "candidate_selection_strategy": "pareto", + "module_selector": "round_robin", + "reflection_minibatch_size": 3, + "skip_perfect_score": false, + "max_metric_calls": 30, + "max_iterations_without_improvement": 4 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py new file mode 100644 index 000000000..a37210146 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -0,0 +1,579 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Evaluation → attribution → optimization → regression → audit pipeline.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import shlex +import sys +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[2] +for _path in (str(_REPO_ROOT), str(_HERE)): + if _path not in sys.path: + sys.path.insert(0, _path) + + +import fake_agent # type: ignore[unresolved-import] +import gates # type: ignore[unresolved-import] +import report # type: ignore[unresolved-import] +import runner # type: ignore[unresolved-import] + +try: + # 本地凭据从同目录 .env 读取（已被 gitignore）；已有环境变量优先。 + from dotenv import load_dotenv + + load_dotenv(_HERE / ".env") +except ImportError: + pass + +DEFAULT_RUN_JSON: dict[str, Any] = { + "champion_prompt": "prompts/system.md", + "train_evalset": "data/train.evalset.json", + "val_evalset": "data/val.evalset.json", + "metric_config": "data/test_config.json", + "gate": { + "min_val_lift": 0.02, + "slice_tolerance": 0.05, + "budget_tokens": 100_000, + "budget_usd": None, + "epsilon": 0.001, + }, + "seed": 42, +} + + +@dataclass +class OptimizerCandidate: + prompt: str + info: dict[str, Any] + rounds: list[dict[str, Any]] + artifacts: dict[str, str] + cost_status: str + total_tokens: Optional[int] + total_cost: Optional[float] + + +def _load_config(config_path: Optional[Path]) -> dict[str, Any]: + if config_path is None: + return json.loads(json.dumps(DEFAULT_RUN_JSON)) + if not config_path.exists(): + raise FileNotFoundError(f"run config 不存在：{config_path}") + config = json.loads(config_path.read_text(encoding="utf-8")) + merged = json.loads(json.dumps(DEFAULT_RUN_JSON)) + merged.update(config) + merged_gate = dict(DEFAULT_RUN_JSON["gate"]) + merged_gate.update(config.get("gate", {})) + merged["gate"] = merged_gate + return merged + + +def _resolve(base: Path, value: str) -> Path: + path = Path(value) + return path if path.is_absolute() else (base / path).resolve() + + +def _gate_config(config: dict[str, Any]) -> gates.GateConfig: + return gates.GateConfig( + min_val_lift=float(config.get("min_val_lift", 0.02)), + slice_tolerance=float(config.get("slice_tolerance", 0.05)), + budget_tokens=int(config.get("budget_tokens", 100_000)), + budget_usd=(float(config["budget_usd"]) if config.get("budget_usd") is not None else None), + epsilon=float(config.get("epsilon", 0.001)), + ) + + +def _build_repro_cmd(args: argparse.Namespace) -> str: + argv = [sys.executable, str(_HERE / "pipeline.py")] + for key, value in vars(args).items(): + if value is None or value is False: + continue + flag = f"--{key.replace('_', '-')}" + argv.append(flag if value is True else f"{flag}={value}") + # 路径可能含空格等特殊字符，引用后保证复现命令可直接粘贴执行。 + return " ".join(shlex.quote(arg) for arg in argv) + + +def _content_text(content: dict[str, Any]) -> str: + return "\n".join(str(part.get("text", "")) for part in (content.get("parts") or []) if part.get("text")) + + +def _normalized_query(query: str) -> str: + """归一化用于审计匹配的 query：去首尾空白并折叠连续空白。 + + 优化器可能对 query 做 trim/换行重排等轻量改写，完全严格的文本匹配 + 会让审计直接杀死整个 optimize 流程，得不偿失。 + """ + + return " ".join(query.split()) + + +def _case_contexts(*paths: Path) -> dict[str, list[dict[str, Any]]]: + """Index evaluation metadata by query without treating query as eval_id. + + The public optimizer callback receives only ``query``. A query can legally + appear in more than one eval case, so the audit entry preserves every + matching case context instead of inventing an eval id from the query. + """ + + contexts: dict[str, list[dict[str, Any]]] = {} + for path in paths: + evalset = json.loads(path.read_text(encoding="utf-8")) + for case in evalset.get("eval_cases", []): + conversation = case.get("conversation") or [] + if not conversation: + continue + invocation = conversation[0] + query = _content_text(invocation.get("user_content") or {}) + if not query: + continue + contexts.setdefault(_normalized_query(query), []).append( + { + "eval_id": case["eval_id"], + "split": ((case.get("session_input") or {}).get("state") or {}).get("split"), + "expected_response": _content_text(invocation.get("final_response") or {}), + "expected_tool_context": ((invocation.get("intermediate_data") or {}).get("tool_uses") or []), + } + ) + return contexts + + +def _audited_call_agent( + base_call_agent, + contexts: dict[str, list[dict[str, Any]]], + audit: list[dict[str, Any]], +): + async def call_agent(query: str) -> str: + # 未登记的 query 降级为空上下文继续执行并在审计中标记， + # 而不是抛错中断整个优化流程：审计是观察手段，不应成为故障点。 + matching_contexts = contexts.get(_normalized_query(query)) or [] + context_match = "matched" if matching_contexts else "unmatched" + try: + response = await base_call_agent(query) + except BaseException as error: + audit.append( + { + "query": query, + "eval_contexts": matching_contexts, + "context_match": context_match, + "status": "error", + "error": f"{type(error).__name__}: {error}", + } + ) + raise + audit.append( + { + "query": query, + "eval_contexts": matching_contexts, + "context_match": context_match, + "status": "ok", + "actual_response": response, + } + ) + return response + + # A plain callback does not prove what its model calls cost. Propagate + # explicit accounting metadata only when the provider/stub supplies it. + call_agent.cost_status = getattr(base_call_agent, "cost_status", "unavailable") + call_agent.total_tokens = getattr(base_call_agent, "total_tokens", None) + call_agent.total_cost = getattr(base_call_agent, "total_cost", None) + return call_agent + + +def _serialize_model(value: Any) -> dict[str, Any]: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json", by_alias=True) + if isinstance(value, dict): + return value + return dict(vars(value)) + + +def _enum_value(value: Any) -> Any: + return getattr(value, "value", value) + + +def _optimizer_cost(result: Any, call_agent) -> tuple[str, Optional[int], Optional[float]]: + if getattr(call_agent, "cost_status", "unavailable") != "measured": + return "unavailable", None, None + agent_tokens = getattr(call_agent, "total_tokens", None) + agent_cost = getattr(call_agent, "total_cost", None) + if agent_tokens is None or agent_cost is None: + return "unavailable", None, None + + usage = getattr(result, "total_token_usage", None) or {} + optimizer_tokens = usage.get("total") if isinstance(usage, dict) else None + optimizer_cost = getattr(result, "total_llm_cost", None) + if optimizer_tokens is None or optimizer_cost is None: + return "unavailable", None, None + if int(optimizer_tokens) <= 0 and int(getattr(result, "total_reflection_lm_calls", 0) or 0) > 0: + return "unavailable", None, None + return ( + "measured", + int(optimizer_tokens) + int(agent_tokens), + float(optimizer_cost) + float(agent_cost), + ) + + +async def _run_optimize_for_candidate( + *, + optimizer_config_path: Path, + champion_prompt_path: Path, + train_evalset_path: Path, + val_evalset_path: Path, + output_dir: Path, + call_agent, +) -> OptimizerCandidate: + """Invoke the native optimizer and return its best prompt plus audit metadata.""" + + from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt + + target = TargetPrompt().add_path("system", str(champion_prompt_path)) + # SDK 已修复盘符冒号误切（_load_eval_set_from_file 用存在性守卫）， + # 直接传绝对路径，避免进程级 chdir 带来的全局状态耦合。 + result = await AgentOptimizer.optimize( + config_path=str(optimizer_config_path), + call_agent=call_agent, + target_prompt=target, + train_dataset_path=str(train_evalset_path), + validation_dataset_path=str(val_evalset_path), + output_dir=str(output_dir), + update_source=False, + verbose=0, + ) + status = _enum_value(result.status) + if status != "SUCCEEDED" or not result.best_prompts: + raise RuntimeError( + f"AgentOptimizer 未产生 Candidate " + f"(status={status}, finish_reason={_enum_value(result.finish_reason)}, " + f"error={getattr(result, 'error_message', '')})" + ) + prompt_name = "system" if "system" in result.best_prompts else next(iter(result.best_prompts)) + rounds: list[dict[str, Any]] = [] + for index, item in enumerate(result.rounds or [], start=1): + round_record = _serialize_model(item) + round_number = int(round_record.get("round", round_record.get("round_index", index))) + round_record["artifact_path"] = str(output_dir / "rounds" / f"round_{round_number:03d}.json") + rounds.append(round_record) + info = { + "algorithm": result.algorithm, + "status": status, + "finish_reason": _enum_value(result.finish_reason), + "stop_reason": _enum_value(getattr(result, "stop_reason", None)), + "total_rounds": result.total_rounds, + "baseline_pass_rate": result.baseline_pass_rate, + "best_pass_rate": result.best_pass_rate, + "pass_rate_improvement": result.pass_rate_improvement, + } + cost_status, total_tokens, total_cost = _optimizer_cost(result, call_agent) + artifacts = { + "optimizer_dir": str(output_dir), + "optimizer_result": str(output_dir / "result.json"), + "optimizer_summary": str(output_dir / "summary.txt"), + "optimizer_rounds": str(output_dir / "rounds"), + "optimizer_config_snapshot": str(output_dir / "config.snapshot.json"), + "optimizer_run_log": str(output_dir / "run.log"), + "optimizer_baseline_prompt": str(output_dir / "baseline_prompts" / f"{prompt_name}.md"), + "optimizer_best_prompt": str(output_dir / "best_prompts" / f"{prompt_name}.md"), + } + return OptimizerCandidate( + prompt=result.best_prompts[prompt_name], + info=info, + rounds=rounds, + artifacts=artifacts, + cost_status=cost_status, + total_tokens=total_tokens, + total_cost=total_cost, + ) + + +def _failure_frozen( + *, + run_id: str, + champion_prompt_path: Path, + train_evalset_path: Path, + val_evalset_path: Path, + metric_config_path: Path, + optimizer_config_path: Path, + config: dict[str, Any], + model_info: dict[str, Any], + error: BaseException, +) -> runner.FrozenInputs: + champion = champion_prompt_path.read_text(encoding="utf-8") + started_at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + return runner.FrozenInputs( + run_id=run_id, + champion_sha256=runner.sha256_text(champion), + challenger_sha256="", + train_sha256=runner.sha256_file(train_evalset_path), + val_sha256=runner.sha256_file(val_evalset_path), + metric_config_sha256=runner.sha256_file(metric_config_path), + run_config_sha256=runner.sha256_json(config), + optimizer_config_sha256=(runner.sha256_file(optimizer_config_path) if optimizer_config_path.exists() else None), + seed=int(config.get("seed", 42)), + started_at=started_at, + mode="optimize", + candidate_source="agent_optimizer", + gate_config=config["gate"], + model_info=model_info, + evaluator_info={ + "name": "AgentEvaluator", + "metric_config_sha256": runner.sha256_file(metric_config_path), + }, + optimizer_info={ + "status": "FAILED", + "error_type": type(error).__name__, + "error": str(error), + }, + ) + + +async def _amain(args: argparse.Namespace, *, call_agent=None) -> int: + if args.config: + config_path: Optional[Path] = _resolve(_HERE, args.config) + else: + default_run_json = _HERE / "run.json" + config_path = default_run_json if default_run_json.exists() else None + config = _load_config(config_path) + champion_path = _resolve(_HERE, config["champion_prompt"]) + train_path = _resolve(_HERE, config["train_evalset"]) + val_path = _resolve(_HERE, config["val_evalset"]) + metric_path = _resolve(_HERE, config["metric_config"]) + optimizer_path = _resolve(_HERE, args.optimizer_config or "optimizer.json") + seed = int(config.get("seed", 42)) + gate_config = _gate_config(config["gate"]) + run_id = runner.new_run_id() + run_dir = _HERE / "runs" / run_id + repro_cmd = _build_repro_cmd(args) + model_info: dict[str, Any] = {"provider": "none", "model_name": "fake-trace"} + optimizer: Optional[OptimizerCandidate] = None + audited_calls: list[dict[str, Any]] = [] + + if args.mode == "optimize": + run_dir.mkdir(parents=True, exist_ok=False) + optimizer_dir = run_dir / "optimizer" + optimizer_dir.mkdir() + try: + base_call_agent = call_agent + if base_call_agent is None: + # 懒加载：fake 模式无需引入真实模型 SDK 依赖链。 + import live_agent # type: ignore[unresolved-import] + + model_info = live_agent.model_info_from_env() + base_call_agent = live_agent.build_call_agent(champion_path) + else: + model_info = {"provider": "test-stub", "model_name": "injected-call-agent"} + contexts = _case_contexts(train_path, val_path) + contextual_call_agent = _audited_call_agent(base_call_agent, contexts, audited_calls) + optimizer = await _run_optimize_for_candidate( + optimizer_config_path=optimizer_path, + champion_prompt_path=champion_path, + train_evalset_path=train_path, + val_evalset_path=val_path, + output_dir=optimizer_dir, + call_agent=contextual_call_agent, + ) + (run_dir / "call_agent_audit.json").write_text( + json.dumps(audited_calls, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + optimizer.artifacts["call_agent_audit"] = str(run_dir / "call_agent_audit.json") + challenger_text = optimizer.prompt + candidate_source = "agent_optimizer" + scenario = None + regression_call_agent = contextual_call_agent + except BaseException as error: + audit_path = run_dir / "call_agent_audit.json" + audit_path.write_text( + json.dumps(audited_calls, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + error_path = run_dir / "optimizer_error.json" + error_path.write_text( + json.dumps( + { + "error_type": type(error).__name__, + "error": str(error), + "call_agent_audit": audited_calls, + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + frozen = _failure_frozen( + run_id=run_id, + champion_prompt_path=champion_path, + train_evalset_path=train_path, + val_evalset_path=val_path, + metric_config_path=metric_path, + optimizer_config_path=optimizer_path, + config=config, + model_info=model_info, + error=error, + ) + (run_dir / "frozen.json").write_text( + json.dumps(asdict(frozen), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + optimizer_artifacts = { + "optimizer_error": str(error_path), + "call_agent_audit": str(audit_path), + "optimizer_dir": str(optimizer_dir), + } + for name in ("result.json", "summary.txt", "config.snapshot.json", "run.log"): + candidate_path = optimizer_dir / name + if candidate_path.exists(): + optimizer_artifacts[f"optimizer_{candidate_path.stem}"] = str(candidate_path) + failure_report = report.build_optimizer_failure_report( + frozen=frozen, + artifact_dir=run_dir, + error=error, + repro_cmd=repro_cmd, + optimizer_artifacts=optimizer_artifacts, + ) + report.write_report(failure_report, out_dir=run_dir) + report.write_report(failure_report, out_dir=_HERE) + print(f"决策: REJECT（优化器失败：{error}）", file=sys.stderr) + print(f"报告: {_HERE / 'optimization_report.md'}") + print(f"Artifacts: {run_dir}") + return 2 if args.apply else 1 + elif args.candidate_file: + challenger_text = _resolve(_HERE, args.candidate_file).read_text(encoding="utf-8") + candidate_source = "candidate_file" + scenario = None + regression_call_agent = None + elif args.scenario: + challenger_text = fake_agent.build_candidate(args.scenario) + candidate_source = "candidate_file" + scenario = args.scenario + regression_call_agent = None + else: + print("fake 模式必须指定 --scenario 或 --candidate-file。", file=sys.stderr) + return 2 + + before_sha = runner.sha256_file(champion_path) + artifact = await runner.run_pair( + champion_prompt_path=champion_path, + challenger_text=challenger_text, + train_evalset_path=train_path, + val_evalset_path=val_path, + metric_config_path=metric_path, + artifact_root=_HERE / "runs", + artifact_dir=run_dir, + mode=args.mode, + candidate_source=candidate_source, + scenario=scenario, + seed=seed, + call_agent=regression_call_agent, + run_config=config, + gate_config=config["gate"], + optimizer_config_path=optimizer_path if args.mode == "optimize" else None, + model_info=model_info, + optimizer_info=optimizer.info if optimizer else {}, + cost_status=optimizer.cost_status if optimizer else "measured", + total_tokens=optimizer.total_tokens if optimizer else 0, + total_cost=optimizer.total_cost if optimizer else 0.0, + optimizer_artifacts=optimizer.artifacts if optimizer else {}, + optimizer_rounds=optimizer.rounds if optimizer else [], + ) + if runner.sha256_file(champion_path) != before_sha: + raise RuntimeError("Champion prompt 在回归评测后未恢复。") + + case_deltas = [ + gates.CaseDelta( + eval_id=case.eval_id, + split=case.split, + slice_name=case.slice_name, + risk_level=case.risk_level, + protected=case.protected, + champion_status=case.champion_status, + challenger_status=case.challenger_status, + champion_score=case.champion_score, + challenger_score=case.challenger_score, + ) + for case in artifact.cases + ] + decision = gates.evaluate( + case_deltas, + cost_status=artifact.cost_status, + total_tokens=artifact.total_tokens, + total_cost=artifact.total_cost, + config=gate_config, + ) + + applied = False + after_apply_sha: Optional[str] = None + if args.apply and decision.accepted: + from trpc_agent_sdk.evaluation import TargetPrompt + + target = TargetPrompt().add_path("system", str(champion_path)) + snapshot = await target.read_all() + try: + await target.write_all({"system": challenger_text}) + after_apply_sha = runner.sha256_file(champion_path) + applied = True + except BaseException: + await target.write_all(snapshot) + raise + elif args.apply: + print("Gate REJECT，--apply 已被拒绝；Champion 未修改。", file=sys.stderr) + + report_dict = report.build_report_dict( + artifact, + decision, + applied=applied, + before_apply_sha256=before_sha, + after_apply_sha256=after_apply_sha, + repro_cmd=repro_cmd, + ) + report.write_report(report_dict, out_dir=artifact.artifact_dir) + report.write_report(report_dict, out_dir=_HERE) + if args.mode == "optimize": + (run_dir / "call_agent_audit.json").write_text( + json.dumps(audited_calls, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + print(f"决策: {'ACCEPT' if decision.accepted else 'REJECT'}") + if decision.violated: + print(f"违反 gate: {', '.join(decision.violated)}") + print(f"报告: {_HERE / 'optimization_report.md'}") + print(f"Artifacts: {artifact.artifact_dir}") + if args.apply and not decision.accepted: + return 2 + return 0 if decision.accepted else 1 + + +def _parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Issue #91 Evaluation + Optimization pipeline") + parser.add_argument("--config", default=None) + parser.add_argument("--mode", choices=("fake", "optimize"), default="fake") + parser.add_argument("--scenario", choices=("success", "no_effect", "overfit")) + parser.add_argument("--candidate-file") + parser.add_argument("--optimizer-config") + parser.add_argument("--apply", action="store_true") + return parser.parse_args(argv) + + +def main(argv: Optional[list[str]] = None) -> int: + return asyncio.run(_amain(_parse_args(argv))) + + +async def amain(argv: Optional[list[str]] = None, *, call_agent=None) -> int: + return await _amain(_parse_args(argv), call_agent=call_agent) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/optimization/eval_optimize_loop/prompts/system.md b/examples/optimization/eval_optimize_loop/prompts/system.md new file mode 100644 index 000000000..d575b0cdc --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/system.md @@ -0,0 +1,16 @@ +# System Prompt (Champion) + +你是一个数学问答助手。请用中文回答用户的算术问题。 + +## 输出格式 + +请直接给出最终数字答案，无需展示步骤。 + +## 工具使用 + +当前未配置工具。 + + diff --git a/examples/optimization/eval_optimize_loop/report.py b/examples/optimization/eval_optimize_loop/report.py new file mode 100644 index 000000000..3d26b44ec --- /dev/null +++ b/examples/optimization/eval_optimize_loop/report.py @@ -0,0 +1,291 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Build machine-readable and human-readable optimization reports.""" + +from __future__ import annotations + +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any, Optional + +import attribution +import gates +from runner import FrozenInputs, RunArtifact, SplitResult + + +def _split_to_dict(split: SplitResult) -> dict[str, float]: + return { + "champion_avg": round(split.champion_avg, 6), + "challenger_avg": round(split.challenger_avg, 6), + "delta": round(split.delta, 6), + } + + +def build_report_dict( + artifact: RunArtifact, + decision: gates.Decision, + *, + applied: bool, + before_apply_sha256: str, + after_apply_sha256: Optional[str], + repro_cmd: str, +) -> dict[str, Any]: + """Build the complete report from real case-level evaluator evidence.""" + + per_case: list[dict[str, Any]] = [] + category_counts: dict[str, int] = {} + transition_counts: dict[str, int] = {} + for case in artifact.cases: + result = attribution.from_case_record(case) + category_counts[result.category] = category_counts.get(result.category, 0) + 1 + transition_counts[case.transition] = transition_counts.get(case.transition, 0) + 1 + per_case.append( + { + "eval_id": case.eval_id, + "split": case.split, + "slice": case.slice_name, + "risk_level": case.risk_level, + "protected": case.protected, + "scenario_tag": case.scenario_tag, + "champion_status": case.champion_status, + "challenger_status": case.challenger_status, + "champion_score": round(case.champion_score, 6), + "challenger_score": round(case.challenger_score, 6), + "delta": round(case.delta, 6), + "transition": case.transition, + "failure_kind": case.failure_kind, + "category": result.category, + "failure_reason": result.reason, + "runs": 1, + "evidence": { + "actual_text": case.actual_text, + "expected_text": case.expected_text, + "metric_results": case.metric_results, + "actual_tool_uses": case.actual_tool_uses, + "expected_tool_uses": case.expected_tool_uses, + "actual_tool_responses": case.actual_tool_responses, + "expected_tool_responses": case.expected_tool_responses, + "error_message": case.error_message, + "metric_reasons": case.failure_reasons, + "trace_ref": case.trace_ref, + "attribution": result.evidence, + }, + } + ) + + frozen = asdict(artifact.frozen) + return { + "version": "2.0", + "candidate_source": artifact.frozen.candidate_source, + "frozen": frozen, + "results": { + "train": _split_to_dict(artifact.train), + "val": _split_to_dict(artifact.val), + }, + "train_delta": round(artifact.train.delta, 6), + "val_delta": round(artifact.val.delta, 6), + "per_case": per_case, + "transition_counts": transition_counts, + "fail_category_counts": category_counts, + "decision": { + "accepted": decision.accepted, + "violated": decision.violated, + "reasons": decision.reasons, + }, + "cost_status": artifact.cost_status, + "cost": { + "status": artifact.cost_status, + "total_tokens": artifact.total_tokens, + "total_usd": artifact.total_cost, + }, + "optimizer": { + "info": artifact.frozen.optimizer_info, + "rounds": artifact.optimizer_rounds, + "artifacts": artifact.optimizer_artifacts, + }, + "audit": { + "run_id": artifact.frozen.run_id, + "applied": applied, + "duration_seconds": round(artifact.duration_seconds, 4), + "candidate_source": artifact.frozen.candidate_source, + "scenario": artifact.frozen.scenario, + "cost_status": artifact.cost_status, + "cost": artifact.total_cost, + "artifact_dir": str(artifact.artifact_dir), + "artifacts": { + "frozen_json": str(artifact.artifact_dir / "frozen.json"), + "champion_prompts": str(artifact.artifact_dir / "champion_prompts" / "system.md"), + "challenger_prompts": str(artifact.artifact_dir / "challenger_prompts" / "system.md"), + "train_eval_log": str(artifact.artifact_dir / "train_eval.json"), + "val_eval_log": str(artifact.artifact_dir / "val_eval.json"), + **artifact.optimizer_artifacts, + }, + "before_apply_sha256": before_apply_sha256, + "after_apply_sha256": after_apply_sha256, + "repro_cmd": repro_cmd, + }, + "limitations": { + "hidden_sample_accuracy": ( + "未使用官方隐藏集，不声称满足隐藏样本准确率；" "公开 fake/标注 fixture 仅证明可复现管线行为。" + ) + }, + } + + +def build_optimizer_failure_report( + *, + frozen: FrozenInputs, + artifact_dir: Path, + error: BaseException, + repro_cmd: str, + optimizer_artifacts: dict[str, str], +) -> dict[str, Any]: + """Persist an auditable REJECT when optimization cannot produce a candidate.""" + + reason = f"{type(error).__name__}: {error}" + return { + "version": "2.0", + "candidate_source": "agent_optimizer", + "frozen": asdict(frozen), + "results": {"train": None, "val": None}, + "train_delta": None, + "val_delta": None, + "per_case": [], + "transition_counts": {}, + "fail_category_counts": {"infrastructure_failure": 1}, + "decision": { + "accepted": False, + "violated": ["OPTIMIZER_FAILURE", "G6"], + "reasons": [ + f"优化器未产生可验证 Candidate：{reason}", + "成本或优化证据不完整，禁止自动 ACCEPT/--apply。", + ], + }, + "cost_status": "unavailable", + "cost": {"status": "unavailable", "total_tokens": None, "total_usd": None}, + "optimizer": { + "info": frozen.optimizer_info, + "rounds": [], + "artifacts": optimizer_artifacts, + "error": reason, + }, + "audit": { + "run_id": frozen.run_id, + "applied": False, + "duration_seconds": 0.0, + "candidate_source": "agent_optimizer", + "scenario": None, + "cost_status": "unavailable", + "cost": None, + "artifact_dir": str(artifact_dir), + "artifacts": { + "frozen_json": str(artifact_dir / "frozen.json"), + **optimizer_artifacts, + }, + "before_apply_sha256": frozen.champion_sha256, + "after_apply_sha256": None, + "repro_cmd": repro_cmd, + }, + "limitations": {"hidden_sample_accuracy": "本次优化失败，未产生可用于隐藏样本验证的 Candidate。"}, + } + + +def render_markdown(report: dict[str, Any]) -> str: + decision = report["decision"] + verdict = "ACCEPT" if decision["accepted"] else "REJECT" + lines = [ + "# Evaluation + Optimization 报告", + "", + f"- **决策**: `{verdict}`", + f"- **运行 ID**: `{report['audit']['run_id']}`", + f"- **候选来源**: `{report['candidate_source']}`", + f"- **运行模式**: `{report['frozen']['mode']}`", + f"- **是否写回**: `{report['audit']['applied']}`", + "", + "## 聚合分数", + "", + ] + if report["results"]["train"] is None: + lines.append("优化器未产生 Candidate，未执行 Champion/Challenger 回归比较。") + else: + lines.extend( + [ + "| split | baseline | candidate | delta |", + "|---|---:|---:|---:|", + ] + ) + for split in ("train", "val"): + values = report["results"][split] + lines.append( + f"| {split} | {values['champion_avg']:.4f} | " + f"{values['challenger_avg']:.4f} | {values['delta']:+.4f} |" + ) + + lines.extend(["", "## Gate 决策", ""]) + if decision["violated"]: + lines.append(f"违反的 gate：`{', '.join(decision['violated'])}`") + else: + lines.append("全部 gate 通过。") + lines.extend(f"- {reason}" for reason in decision["reasons"]) + + lines.extend(["", "## 失败归因统计", "", "| category | count |", "|---|---:|"]) + for category, count in sorted(report["fail_category_counts"].items()): + lines.append(f"| {category} | {count} |") + + lines.extend( + [ + "", + "## 逐 case 明细", + "", + "| eval_id | split | transition | baseline | candidate | delta | category | reason |", + "|---|---|---|---:|---:|---:|---|---|", + ] + ) + for case in report["per_case"]: + reason = str(case["failure_reason"]).replace("|", "\\|").replace("\n", " ") + lines.append( + f"| {case['eval_id']} | {case['split']} | {case['transition']} | " + f"{case['champion_score']:.2f} | {case['challenger_score']:.2f} | " + f"{case['delta']:+.2f} | {case['category']} | {reason} |" + ) + + cost = report["cost"] + lines.extend( + [ + "", + "## 成本与优化器产物", + "", + f"- cost_status: `{cost['status']}`", + f"- total_tokens: `{cost['total_tokens']}`", + f"- total_usd: `{cost['total_usd']}`", + f"- optimizer rounds: `{len(report['optimizer']['rounds'])}`", + "", + "## 审计", + "", + f"- artifact_dir: `{report['audit']['artifact_dir']}`", + f"- before_apply_sha256: `{report['audit']['before_apply_sha256']}`", + f"- after_apply_sha256: `{report['audit']['after_apply_sha256']}`", + f"- repro_cmd: `{report['audit']['repro_cmd']}`", + "", + "## 限制", + "", + f"- {report['limitations']['hidden_sample_accuracy']}", + "", + ] + ) + return "\n".join(lines) + + +def write_report(report: dict[str, Any], *, out_dir: Path) -> tuple[Path, Path]: + out_dir.mkdir(parents=True, exist_ok=True) + json_path = out_dir / "optimization_report.json" + markdown_path = out_dir / "optimization_report.md" + json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + markdown_path.write_text(render_markdown(report), encoding="utf-8") + return json_path, markdown_path diff --git a/examples/optimization/eval_optimize_loop/run.json b/examples/optimization/eval_optimize_loop/run.json new file mode 100644 index 000000000..815f73026 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run.json @@ -0,0 +1,15 @@ +{ + "_comment": "eval_optimize_loop 运行配置。所有路径相对本文件所在目录；gate 阈值在此调整。", + "champion_prompt": "prompts/system.md", + "train_evalset": "data/train.evalset.json", + "val_evalset": "data/val.evalset.json", + "metric_config": "data/test_config.json", + "gate": { + "min_val_lift": 0.02, + "slice_tolerance": 0.05, + "budget_tokens": 100000, + "budget_usd": null, + "epsilon": 0.001 + }, + "seed": 42 +} diff --git a/examples/optimization/eval_optimize_loop/runner.py b/examples/optimization/eval_optimize_loop/runner.py new file mode 100644 index 000000000..af1ab138c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/runner.py @@ -0,0 +1,610 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Run Champion and Challenger evaluations and persist auditable evidence.""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import time +import uuid +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Awaitable, Callable, Optional + +from trpc_agent_sdk.evaluation import AgentEvaluator, EvalCaseResult, EvaluateResult, TargetPrompt + +CallAgent = Callable[[str], Awaitable[str]] + + +@dataclass +class FrozenInputs: + """Immutable inputs and environment facts for one experiment.""" + + run_id: str + champion_sha256: str + challenger_sha256: str + train_sha256: str + val_sha256: str + metric_config_sha256: str + run_config_sha256: str + optimizer_config_sha256: Optional[str] + seed: int + started_at: str + mode: str + candidate_source: str + scenario: Optional[str] = None + gate_config: dict[str, Any] = field(default_factory=dict) + model_info: dict[str, Any] = field(default_factory=dict) + evaluator_info: dict[str, Any] = field(default_factory=dict) + optimizer_info: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class SplitResult: + champion_avg: float + challenger_avg: float + + @property + def delta(self) -> float: + return self.challenger_avg - self.champion_avg + + +@dataclass +class CaseRecord: + """Case-level comparison plus evidence needed for attribution.""" + + eval_id: str + split: str + slice_name: str + risk_level: str + protected: bool + scenario_tag: Optional[str] + champion_status: str + challenger_status: str + champion_score: float + challenger_score: float + expected_text: Optional[str] + actual_text: Optional[str] + metric_results: list[dict[str, Any]] = field(default_factory=list) + actual_tool_uses: list[dict[str, Any]] = field(default_factory=list) + expected_tool_uses: list[dict[str, Any]] = field(default_factory=list) + actual_tool_responses: list[dict[str, Any]] = field(default_factory=list) + expected_tool_responses: list[dict[str, Any]] = field(default_factory=list) + error_message: Optional[str] = None + failure_reasons: list[str] = field(default_factory=list) + trace_ref: Optional[str] = None + + @property + def delta(self) -> float: + return self.challenger_score - self.champion_score + + @property + def transition(self) -> str: + if self.champion_status == "PASSED" and self.challenger_status != "PASSED": + return "newly_failed" + if self.champion_status != "PASSED" and self.challenger_status == "PASSED": + return "newly_passed" + if self.delta > 0: + return "score_up" + if self.delta < 0: + return "score_down" + return "unchanged" + + @property + def failure_kind(self) -> str: + if self.challenger_status == "PASSED": + return "none" + if self.error_message or self.challenger_status == "NOT_EVALUATED": + return "infrastructure_failure" + return "agent_quality_failure" + + +@dataclass +class RunArtifact: + frozen: FrozenInputs + train: SplitResult + val: SplitResult + cases: list[CaseRecord] + champion_train_avg: float + champion_val_avg: float + artifact_dir: Path + cost_status: str + total_tokens: Optional[int] + total_cost: Optional[float] + duration_seconds: float + champion_prompt_text: str + challenger_prompt_text: str + optimizer_artifacts: dict[str, str] = field(default_factory=dict) + optimizer_rounds: list[dict[str, Any]] = field(default_factory=list) + + +def new_run_id() -> str: + """Return a filesystem-safe, collision-resistant UTC run id.""" + + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%fZ") + return f"{timestamp}-{uuid.uuid4().hex[:8]}" + + +def sha256_file(path: Path) -> str: + """Hash UTF-8 text with normalized newlines; binary files hash as bytes.""" + + raw = path.read_bytes() + try: + text = raw.decode("utf-8").replace("\r\n", "\n").replace("\r", "\n") + raw = text.encode("utf-8") + except UnicodeDecodeError: + pass + return hashlib.sha256(raw).hexdigest() + + +def sha256_text(text: str) -> str: + normalized = text.replace("\r\n", "\n").replace("\r", "\n") + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def sha256_json(value: Any) -> str: + payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return sha256_text(payload) + + +def _load_evalset(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _case_state(case: dict[str, Any]) -> dict[str, Any]: + return (case.get("session_input") or {}).get("state") or {} + + +def _content_text(content: Any) -> Optional[str]: + if content is None: + return None + parts = getattr(content, "parts", None) or [] + texts = [str(getattr(part, "text", "")) for part in parts if getattr(part, "text", None)] + return "\n".join(texts) if texts else None + + +def _expected_text(case: dict[str, Any]) -> Optional[str]: + conversation = case.get("conversation") or [] + if not conversation: + return None + parts = (conversation[0].get("final_response") or {}).get("parts") or [] + texts = [str(part.get("text", "")) for part in parts if part.get("text")] + return "\n".join(texts) if texts else None + + +def _model_to_dict(value: Any) -> dict[str, Any]: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json", by_alias=True) + if isinstance(value, dict): + return value + return {key: getattr(value, key) for key in ("id", "name", "args", "response") if hasattr(value, key)} + + +def _intermediate_items(invocation: Any, field_name: str) -> list[dict[str, Any]]: + intermediate = getattr(invocation, "intermediate_data", None) + values = getattr(intermediate, field_name, None) or [] + return [_model_to_dict(value) for value in values] + + +def _metric_evidence(result: EvalCaseResult) -> tuple[list[dict[str, Any]], list[str]]: + metrics: list[dict[str, Any]] = [] + reasons: list[str] = [] + for metric in result.overall_eval_metric_results or []: + details = getattr(metric, "details", None) + reason = getattr(details, "reason", None) if details is not None else None + if reason: + reasons.append(str(reason)) + metrics.append( + { + "metric_name": metric.metric_name, + "score": metric.score, + "threshold": metric.threshold, + "eval_status": getattr(metric.eval_status, "name", str(metric.eval_status)), + "reason": reason, + "rubric_scores": [_model_to_dict(score) for score in (getattr(details, "rubric_scores", None) or [])], + } + ) + for per_invocation in result.eval_metric_result_per_invocation or []: + for metric in per_invocation.eval_metric_results or []: + details = getattr(metric, "details", None) + reason = getattr(details, "reason", None) if details is not None else None + if reason and str(reason) not in reasons: + reasons.append(str(reason)) + return metrics, reasons + + +def _status(result: EvalCaseResult) -> str: + return getattr(result.final_eval_status, "name", str(result.final_eval_status)) + + +def _score(result: EvalCaseResult) -> float: + metrics = list(result.overall_eval_metric_results or []) + if not metrics: + return 1.0 if _status(result) == "PASSED" else 0.0 + scores = [float(metric.score) if metric.score is not None else 0.0 for metric in metrics] + return sum(scores) / len(scores) if scores else 0.0 + + +def _worst_run(runs: list[EvalCaseResult]) -> Optional[EvalCaseResult]: + if not runs: + return None + return min(runs, key=lambda result: (0 if _status(result) == "NOT_EVALUATED" else 1, _score(result))) + + +def _flatten_results(result: EvaluateResult) -> dict[str, list[EvalCaseResult]]: + flattened: dict[str, list[EvalCaseResult]] = {} + for aggregate in (result.results_by_eval_set_id or {}).values(): + for eval_id, runs in (aggregate.eval_results_by_eval_id or {}).items(): + flattened[eval_id] = list(runs or []) + return flattened + + +async def _run_evaluator( + evalset_path: Path, + *, + call_agent: Optional[CallAgent] = None, + metric_config_path: Optional[Path] = None, +) -> EvaluateResult: + """Run the public evaluator API with absolute paths. + + The SDK's ``_load_eval_set_from_file`` guards the "file.json:case_id" + selector with existence checks, so Windows drive-letter paths are safe + to pass directly and no process-wide chdir workaround is needed. + """ + + metric_arg = str(metric_config_path) if metric_config_path is not None else None + executer = AgentEvaluator.get_executer( + str(evalset_path), + call_agent=call_agent, + eval_metrics_file_path_or_dir=metric_arg, + print_detailed_results=False, + print_summary_report=False, + ) + try: + await executer.evaluate() + except AssertionError as exc: + # Case failures are represented as AssertionError by the public facade; + # the structured result remains available from the executer. Any other + # assertion (config/contract errors, SDK bugs) aborts before a result + # exists and must not be masked as an empty NOT_EVALUATED outcome. + if executer.get_result() is None: + raise RuntimeError( + f"Evaluator aborted before producing results ({type(exc).__name__}): {exc}" + ) from exc + result = executer.get_result() + return result or EvaluateResult() + + +def _make_trace_evalset(base_evalset: dict[str, Any], actual_cases: list[dict[str, Any]]) -> dict[str, Any]: + trace = dict(base_evalset) + trace["eval_cases"] = actual_cases + trace["eval_set_id"] = f"{base_evalset.get('eval_set_id', 'set')}_trace" + return trace + + +def _write_trace_evalset( + target_path: Path, + base_evalset: dict[str, Any], + actual_cases: list[dict[str, Any]], + metric_config_path: Path, +) -> None: + target_path.parent.mkdir(parents=True, exist_ok=True) + local_config = target_path.parent / metric_config_path.name + if not local_config.exists(): + shutil.copyfile(metric_config_path, local_config) + target_path.write_text( + json.dumps(_make_trace_evalset(base_evalset, actual_cases), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + +def _case_record( + *, + case: dict[str, Any], + split: str, + champion_runs: list[EvalCaseResult], + challenger_runs: list[EvalCaseResult], +) -> CaseRecord: + champion = _worst_run(champion_runs) + challenger = _worst_run(challenger_runs) + state = _case_state(case) + actual_text: Optional[str] = None + expected_text = _expected_text(case) + actual_tools: list[dict[str, Any]] = [] + expected_tools: list[dict[str, Any]] = [] + actual_responses: list[dict[str, Any]] = [] + expected_responses: list[dict[str, Any]] = [] + metric_results: list[dict[str, Any]] = [] + reasons: list[str] = [] + error_message: Optional[str] = None + + if challenger is not None: + error_message = challenger.error_message + metric_results, reasons = _metric_evidence(challenger) + for per_invocation in challenger.eval_metric_result_per_invocation or []: + actual = per_invocation.actual_invocation + expected = per_invocation.expected_invocation + if actual_text is None: + actual_text = _content_text(getattr(actual, "final_response", None)) + if expected_text is None and expected is not None: + expected_text = _content_text(getattr(expected, "final_response", None)) + actual_tools.extend(_intermediate_items(actual, "tool_uses")) + actual_responses.extend(_intermediate_items(actual, "tool_responses")) + if expected is not None: + expected_tools.extend(_intermediate_items(expected, "tool_uses")) + expected_responses.extend(_intermediate_items(expected, "tool_responses")) + if error_message and error_message not in reasons: + reasons.insert(0, error_message) + + champion_status = _status(champion) if champion is not None else "NOT_EVALUATED" + challenger_status = _status(challenger) if challenger is not None else "NOT_EVALUATED" + return CaseRecord( + eval_id=case["eval_id"], + split=split, + slice_name=str(state.get("slice", "default")), + risk_level=str(state.get("risk_level", "low")), + protected=bool(state.get("protected", False)), + scenario_tag=state.get("scenario_tag"), + champion_status=champion_status, + challenger_status=challenger_status, + champion_score=_score(champion) if champion is not None else 0.0, + challenger_score=_score(challenger) if challenger is not None else 0.0, + expected_text=expected_text, + actual_text=actual_text, + metric_results=metric_results, + actual_tool_uses=actual_tools, + expected_tool_uses=expected_tools, + actual_tool_responses=actual_responses, + expected_tool_responses=expected_responses, + error_message=error_message, + failure_reasons=reasons, + trace_ref=f"{split}_eval.json#eval_id={case['eval_id']}", + ) + + +def _build_case_records( + *, + train_base: dict[str, Any], + val_base: dict[str, Any], + champion_train: EvaluateResult, + champion_val: EvaluateResult, + challenger_train: EvaluateResult, + challenger_val: EvaluateResult, +) -> list[CaseRecord]: + records: list[CaseRecord] = [] + for split, cases, champion_result, challenger_result in ( + ("train", train_base["eval_cases"], champion_train, challenger_train), + ("val", val_base["eval_cases"], champion_val, challenger_val), + ): + champion_map = _flatten_results(champion_result) + challenger_map = _flatten_results(challenger_result) + for case in cases: + eval_id = case["eval_id"] + records.append( + _case_record( + case=case, + split=split, + champion_runs=champion_map.get(eval_id, []), + challenger_runs=challenger_map.get(eval_id, []), + ) + ) + return records + + +def _dump_eval_json(path: Path, *, champion: EvaluateResult, challenger: EvaluateResult) -> None: + def serialize(result: EvaluateResult) -> dict[str, Any]: + return result.model_dump(mode="json", by_alias=True) + + path.write_text( + json.dumps( + {"champion": serialize(champion), "challenger": serialize(challenger)}, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + +async def run_pair( + *, + champion_prompt_path: Path, + challenger_text: str, + train_evalset_path: Path, + val_evalset_path: Path, + metric_config_path: Path, + artifact_root: Path, + mode: str, + candidate_source: str, + scenario: Optional[str], + seed: int, + artifact_dir: Optional[Path] = None, + call_agent: Optional[CallAgent] = None, + run_config: Optional[dict[str, Any]] = None, + gate_config: Optional[dict[str, Any]] = None, + optimizer_config_path: Optional[Path] = None, + model_info: Optional[dict[str, Any]] = None, + optimizer_info: Optional[dict[str, Any]] = None, + cost_status: Optional[str] = None, + total_tokens: Optional[int] = None, + total_cost: Optional[float] = None, + optimizer_artifacts: Optional[dict[str, str]] = None, + optimizer_rounds: Optional[list[dict[str, Any]]] = None, +) -> RunArtifact: + """Evaluate both prompts on train and validation with guaranteed restoration.""" + + started = datetime.now(timezone.utc) + started_at = started.isoformat(timespec="milliseconds").replace("+00:00", "Z") + started_monotonic = time.monotonic() + run_id = artifact_dir.name if artifact_dir is not None else new_run_id() + artifact_dir = artifact_dir or artifact_root / run_id + if artifact_dir.exists(): + if (artifact_dir / "frozen.json").exists(): + raise FileExistsError(f"run artifact already exists: {artifact_dir}") + else: + artifact_dir.mkdir(parents=True, exist_ok=False) + + champion_text = champion_prompt_path.read_text(encoding="utf-8") + run_config = run_config or {} + gate_config = gate_config or {} + optimizer_info = optimizer_info or {} + frozen = FrozenInputs( + run_id=run_id, + champion_sha256=sha256_text(champion_text), + challenger_sha256=sha256_text(challenger_text), + train_sha256=sha256_file(train_evalset_path), + val_sha256=sha256_file(val_evalset_path), + metric_config_sha256=sha256_file(metric_config_path), + run_config_sha256=sha256_json(run_config), + optimizer_config_sha256=( + sha256_file(optimizer_config_path) + if optimizer_config_path is not None and optimizer_config_path.exists() + else None + ), + seed=seed, + started_at=started_at, + mode=mode, + candidate_source=candidate_source, + scenario=scenario, + gate_config=gate_config, + model_info=model_info or {}, + evaluator_info={ + "name": "AgentEvaluator", + "metric_config": str(metric_config_path), + "metric_config_sha256": sha256_file(metric_config_path), + }, + optimizer_info=optimizer_info, + ) + + (artifact_dir / "frozen.json").write_text( + json.dumps(asdict(frozen), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + champion_dir = artifact_dir / "champion_prompts" + challenger_dir = artifact_dir / "challenger_prompts" + champion_dir.mkdir() + challenger_dir.mkdir() + (champion_dir / "system.md").write_text(champion_text, encoding="utf-8") + (challenger_dir / "system.md").write_text(challenger_text, encoding="utf-8") + + train_base = _load_evalset(train_evalset_path) + val_base = _load_evalset(val_evalset_path) + target = TargetPrompt().add_path("system", str(champion_prompt_path)) + snapshot = await target.read_all() + + if mode == "fake": + import fake_agent # type: ignore[unresolved-import] + + paths = { + "champion_train": artifact_dir / "champion_train.evalset.json", + "champion_val": artifact_dir / "champion_val.evalset.json", + "challenger_train": artifact_dir / "challenger_train.evalset.json", + "challenger_val": artifact_dir / "challenger_val.evalset.json", + } + _write_trace_evalset( + paths["champion_train"], + train_base, + fake_agent.gen_actual_conversation(champion_text, train_base), + metric_config_path, + ) + _write_trace_evalset( + paths["champion_val"], + val_base, + fake_agent.gen_actual_conversation(champion_text, val_base), + metric_config_path, + ) + _write_trace_evalset( + paths["challenger_train"], + train_base, + fake_agent.gen_actual_conversation(challenger_text, train_base), + metric_config_path, + ) + _write_trace_evalset( + paths["challenger_val"], + val_base, + fake_agent.gen_actual_conversation(challenger_text, val_base), + metric_config_path, + ) + champion_train = await _run_evaluator(paths["champion_train"], metric_config_path=metric_config_path) + champion_val = await _run_evaluator(paths["champion_val"], metric_config_path=metric_config_path) + challenger_train = await _run_evaluator(paths["challenger_train"], metric_config_path=metric_config_path) + challenger_val = await _run_evaluator(paths["challenger_val"], metric_config_path=metric_config_path) + resolved_cost_status = "measured" + resolved_tokens = 0 + resolved_cost = 0.0 + else: + if call_agent is None: + raise ValueError("optimize mode requires a real call_agent callback") + champion_train = await _run_evaluator( + train_evalset_path, + call_agent=call_agent, + metric_config_path=metric_config_path, + ) + champion_val = await _run_evaluator( + val_evalset_path, + call_agent=call_agent, + metric_config_path=metric_config_path, + ) + try: + await target.write_all({"system": challenger_text}) + challenger_train = await _run_evaluator( + train_evalset_path, + call_agent=call_agent, + metric_config_path=metric_config_path, + ) + challenger_val = await _run_evaluator( + val_evalset_path, + call_agent=call_agent, + metric_config_path=metric_config_path, + ) + finally: + await target.write_all(snapshot) + resolved_cost_status = cost_status or "unavailable" + resolved_tokens = total_tokens if resolved_cost_status == "measured" else None + resolved_cost = total_cost if resolved_cost_status == "measured" else None + + cases = _build_case_records( + train_base=train_base, + val_base=val_base, + champion_train=champion_train, + champion_val=champion_val, + challenger_train=challenger_train, + challenger_val=challenger_val, + ) + train_cases = [case for case in cases if case.split == "train"] + val_cases = [case for case in cases if case.split == "val"] + train = SplitResult( + champion_avg=sum(case.champion_score for case in train_cases) / max(len(train_cases), 1), + challenger_avg=sum(case.challenger_score for case in train_cases) / max(len(train_cases), 1), + ) + val = SplitResult( + champion_avg=sum(case.champion_score for case in val_cases) / max(len(val_cases), 1), + challenger_avg=sum(case.challenger_score for case in val_cases) / max(len(val_cases), 1), + ) + _dump_eval_json(artifact_dir / "train_eval.json", champion=champion_train, challenger=challenger_train) + _dump_eval_json(artifact_dir / "val_eval.json", champion=champion_val, challenger=challenger_val) + + return RunArtifact( + frozen=frozen, + train=train, + val=val, + cases=cases, + champion_train_avg=train.champion_avg, + champion_val_avg=val.champion_avg, + artifact_dir=artifact_dir, + cost_status=resolved_cost_status, + total_tokens=resolved_tokens, + total_cost=resolved_cost, + duration_seconds=time.monotonic() - started_monotonic, + champion_prompt_text=champion_text, + challenger_prompt_text=challenger_text, + optimizer_artifacts=optimizer_artifacts or {}, + optimizer_rounds=optimizer_rounds or [], + ) diff --git a/examples/optimization/eval_optimize_loop/tests/conftest.py b/examples/optimization/eval_optimize_loop/tests/conftest.py new file mode 100644 index 000000000..ba1239c83 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/conftest.py @@ -0,0 +1,61 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Shared pytest setup and safe Champion prompt restoration.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest +import pytest_asyncio + +_HERE = Path(__file__).resolve().parent +_LOOP_ROOT = _HERE.parent +_REPO_ROOT = _LOOP_ROOT.parents[2] + +for path in (str(_REPO_ROOT), str(_LOOP_ROOT)): + if path not in sys.path: + sys.path.insert(0, path) + + +_BASELINE_CHAMPION = """# System Prompt (Champion) + +你是一个数学问答助手。请用中文回答用户的算术问题。 + +## 输出格式 + +请直接给出最终数字答案，无需展示步骤。 + +## 工具使用 + +当前未配置工具。 + + +""" + + +@pytest.fixture(scope="session") +def loop_root() -> Path: + return _LOOP_ROOT + + +@pytest_asyncio.fixture(autouse=True) +async def _restore_champion_prompt(): + """Restore through the production TargetPrompt abstraction.""" + + from trpc_agent_sdk.evaluation import TargetPrompt + + champion = _LOOP_ROOT / "prompts" / "system.md" + target = TargetPrompt().add_path("system", str(champion)) + await target.write_all({"system": _BASELINE_CHAMPION}) + yield + await target.write_all({"system": _BASELINE_CHAMPION}) diff --git a/examples/optimization/eval_optimize_loop/tests/test_apply.py b/examples/optimization/eval_optimize_loop/tests/test_apply.py new file mode 100644 index 000000000..4d77c87d3 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_apply.py @@ -0,0 +1,123 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""验证 Champion prompt 在 dry-run / apply REJECT / apply ACCEPT 三种情况下的写回行为。""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.asyncio + + +def _sha(p: Path) -> str: + import runner + + return runner.sha256_file(p) + + +@pytest.fixture(autouse=True) +def _no_api_key(monkeypatch): + for k in ("OPENAI_API_KEY", "TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + monkeypatch.delenv(k, raising=False) + + +async def test_dry_run_keeps_champion_unchanged(loop_root: Path) -> None: + champion = loop_root / "prompts" / "system.md" + before = _sha(champion) + import pipeline + + await pipeline.amain(["--mode", "fake", "--scenario", "success"]) + assert _sha(champion) == before + + +async def test_apply_rejected_keeps_champion_unchanged(loop_root: Path) -> None: + champion = loop_root / "prompts" / "system.md" + before = _sha(champion) + import pipeline + + rc = await pipeline.amain(["--mode", "fake", "--scenario", "no_effect", "--apply"]) + # REJECT + --apply -> exit 2 + assert rc == 2 + assert _sha(champion) == before + + +async def test_apply_accepted_writes_champion(loop_root: Path) -> None: + """success 场景 + --apply：源 prompt 应被 Challenger 覆盖。 + + 跑完后立即恢复原 Champion 内容，避免污染其他测试。 + 恢复也必须通过 TargetPrompt，避免绕过生产写入抽象。 + """ + champion = loop_root / "prompts" / "system.md" + original_text = champion.read_text(encoding="utf-8") + original_sha = _sha(champion) + try: + import pipeline + + rc = await pipeline.amain(["--mode", "fake", "--scenario", "success", "--apply"]) + assert rc == 0 + new_sha = _sha(champion) + # 必须变了 + assert new_sha != original_sha + # 报告中记录的 after_apply_sha256 必须等于 challenger_sha256 + import json + + report = json.loads((pipeline._HERE / "optimization_report.json").read_text(encoding="utf-8")) + assert report["audit"]["applied"] is True + assert report["audit"]["after_apply_sha256"] == report["frozen"]["challenger_sha256"] + finally: + from trpc_agent_sdk.evaluation import TargetPrompt + + target = TargetPrompt().add_path("system", str(champion)) + await target.write_all({"system": original_text}) + assert _sha(champion) == original_sha + + +async def test_champion_restored_when_challenger_evaluation_crashes( + loop_root: Path, + monkeypatch, +) -> None: + """Exercise the optimize-mode prompt swap, then fail mid-evaluation.""" + + import fake_agent + import runner + from trpc_agent_sdk.evaluation import EvaluateResult + + champion = loop_root / "prompts" / "system.md" + before = _sha(champion) + challenger = fake_agent.build_candidate("success") + calls = 0 + + async def evaluator_stub(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 3: + assert champion.read_text(encoding="utf-8") == challenger + raise RuntimeError("simulated challenger evaluator crash") + return EvaluateResult() + + async def call_agent_stub(query: str) -> str: + return query + + monkeypatch.setattr(runner, "_run_evaluator", evaluator_stub) + with pytest.raises(RuntimeError, match="simulated challenger evaluator crash"): + await runner.run_pair( + champion_prompt_path=champion, + challenger_text=challenger, + train_evalset_path=loop_root / "data" / "train.evalset.json", + val_evalset_path=loop_root / "data" / "val.evalset.json", + metric_config_path=loop_root / "data" / "test_config.json", + artifact_root=loop_root / "runs", + mode="optimize", + candidate_source="agent_optimizer", + scenario=None, + seed=42, + call_agent=call_agent_stub, + ) + assert _sha(champion) == before diff --git a/examples/optimization/eval_optimize_loop/tests/test_attribution.py b/examples/optimization/eval_optimize_loop/tests/test_attribution.py new file mode 100644 index 000000000..9a57f16ab --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_attribution.py @@ -0,0 +1,114 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Evidence-based failure attribution tests.""" + +from __future__ import annotations + +from attribution import AttributionInput, CATEGORIES, classify, classify_with_reason + + +def test_categories_complete() -> None: + assert set(CATEGORIES) == { + "reply_mismatch", + "format_fail", + "tool_call_error", + "param_error", + "rubric_fail", + "knowledge_fail", + "infrastructure_failure", + "insufficient_evidence", + "none", + } + + +def test_passed_returns_none() -> None: + assert classify(AttributionInput(status="PASSED")) == "none" + + +def test_rubric_fail_from_metric_name() -> None: + inp = AttributionInput( + status="FAILED", + metric_names_failed=("llm_rubric_response",), + metric_reasons=("reasoning_clear rubric failed",), + ) + assert classify(inp) == "rubric_fail" + + +def test_tool_call_error_from_trajectory() -> None: + inp = AttributionInput( + status="FAILED", + expected_tool_uses=({"name": "search", "args": {"q": "x"}},), + actual_tool_uses=(), + ) + assert classify(inp) == "tool_call_error" + + +def test_param_error_exposes_parameter_diff() -> None: + inp = AttributionInput( + status="FAILED", + expected_tool_uses=({"name": "search", "args": {"q": "x"}},), + actual_tool_uses=({"name": "search", "args": {"q": "y"}},), + ) + result = classify_with_reason(inp) + assert result.category == "param_error" + assert result.evidence["parameter_differences"] == [ + { + "tool_name": "search", + "expected_args": {"q": "x"}, + "actual_args": {"q": "y"}, + } + ] + + +def test_knowledge_fail_from_evaluator_reason() -> None: + inp = AttributionInput( + status="FAILED", + metric_names_failed=("llm_rubric_knowledge_recall",), + metric_reasons=("knowledge recall did not support rubric kr1",), + ) + assert classify(inp) == "knowledge_fail" + + +def test_format_fail_from_response_evidence() -> None: + inp = AttributionInput( + status="FAILED", + expected_text="步骤：4 + 7 = 11\n答案：11 个", + actual_text="11", + ) + assert classify(inp) == "format_fail" + + +def test_reply_mismatch_from_response_evidence() -> None: + inp = AttributionInput( + status="FAILED", + expected_text="答案：11", + actual_text="答案：22", + ) + assert classify(inp) == "reply_mismatch" + + +def test_infrastructure_failure_is_not_agent_quality_failure() -> None: + inp = AttributionInput( + status="NOT_EVALUATED", + error_message="TimeoutError: provider unavailable", + ) + assert classify(inp) == "infrastructure_failure" + + +def test_missing_raw_evidence_is_not_fabricated_reply_mismatch() -> None: + inp = AttributionInput( + status="FAILED", + metric_names_failed=("final_response_avg_score",), + metric_reasons=("score below threshold",), + ) + assert classify(inp) == "insufficient_evidence" + + +def test_scenario_tag_does_not_control_classification() -> None: + inp = AttributionInput(status="FAILED", scenario_tag="knowledge_lookup") + assert classify(inp) == "insufficient_evidence" diff --git a/examples/optimization/eval_optimize_loop/tests/test_audit_robustness.py b/examples/optimization/eval_optimize_loop/tests/test_audit_robustness.py new file mode 100644 index 000000000..151662fff --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_audit_robustness.py @@ -0,0 +1,99 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""验证审计回调对改写 query 的容错、事件文本拼接的健壮性与 GateConfig 边界。""" + +from __future__ import annotations + +import pytest + +import gates +import live_agent +import pipeline + + +async def _echo(query: str) -> str: + return f"reply:{query}" + + +@pytest.mark.asyncio +async def test_audited_call_agent_matches_trimmed_query() -> None: + """优化器对 query 做 trim/换行重排后仍应命中登记的上下文。""" + + contexts = {pipeline._normalized_query("小明有 3 个苹果"): [{"eval_id": "case_1"}]} + audit: list[dict] = [] + call_agent = pipeline._audited_call_agent(_echo, contexts, audit) + + await call_agent(" 小明有 3 个苹果 \n") + + assert audit[0]["context_match"] == "matched" + assert audit[0]["eval_contexts"] == [{"eval_id": "case_1"}] + assert audit[0]["status"] == "ok" + + +@pytest.mark.asyncio +async def test_audited_call_agent_degrades_on_unknown_query() -> None: + """未登记的 query 不应抛错中断优化流程，而是空上下文继续并标记。""" + + audit: list[dict] = [] + call_agent = pipeline._audited_call_agent(_echo, {}, audit) + + response = await call_agent("完全陌生的模板包裹 query") + + assert response == "reply:完全陌生的模板包裹 query" + assert audit[0]["context_match"] == "unmatched" + assert audit[0]["eval_contexts"] == [] + + +class _PartNoThought: + """模拟不带 thought 属性的 provider Part。""" + + def __init__(self, text: str): + self.text = text + + +class _PartWithThought: + def __init__(self, text: str, thought: bool): + self.text = text + self.thought = thought + + +class _Content: + def __init__(self, parts): + self.parts = parts + + +class _Event: + def __init__(self, parts, final: bool = True): + self.content = _Content(parts) + self._final = final + + def is_final_response(self) -> bool: + return self._final + + +def test_response_text_tolerates_parts_without_thought_attr() -> None: + """Part 完全没有 thought 属性时不应抛 AttributeError。""" + + event = _Event([_PartNoThought("你好"), _PartNoThought("世界")]) + assert live_agent.response_text_from_event(event) == "你好世界" + + +def test_response_text_skips_thought_parts_and_non_final_events() -> None: + event = _Event([_PartWithThought("内心推理", True), _PartWithThought("答案：10", False)]) + assert live_agent.response_text_from_event(event) == "答案：10" + assert live_agent.response_text_from_event(_Event([_PartNoThought("忽略")], final=False)) == "" + + +def test_gate_config_rejects_nonpositive_budget() -> None: + """budget_tokens<=0 会让 G6 把任意 measured 成本误判为超预算，必须拒绝。""" + + with pytest.raises(ValueError, match="budget_tokens"): + gates.GateConfig(budget_tokens=0) + with pytest.raises(ValueError, match="budget_usd"): + gates.GateConfig(budget_usd=-1.0) + assert gates.GateConfig().budget_tokens == 100_000 diff --git a/examples/optimization/eval_optimize_loop/tests/test_dataset.py b/examples/optimization/eval_optimize_loop/tests/test_dataset.py new file mode 100644 index 000000000..a7399e658 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_dataset.py @@ -0,0 +1,64 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""数据集 sanity check： +- train / val 各 3 条 +- eval_id 互斥 +""" + +from __future__ import annotations + +import json +from pathlib import Path + + +def _load(p: Path) -> dict: + return json.loads(p.read_text(encoding="utf-8")) + + +def test_train_size(loop_root: Path) -> None: + data = _load(loop_root / "data" / "train.evalset.json") + assert len(data["eval_cases"]) == 3 + + +def test_val_size(loop_root: Path) -> None: + data = _load(loop_root / "data" / "val.evalset.json") + assert len(data["eval_cases"]) == 3 + + +def test_eval_id_disjoint(loop_root: Path) -> None: + train = _load(loop_root / "data" / "train.evalset.json") + val = _load(loop_root / "data" / "val.evalset.json") + train_ids = {c["eval_id"] for c in train["eval_cases"]} + val_ids = {c["eval_id"] for c in val["eval_cases"]} + assert train_ids & val_ids == set(), f"train/val eval_id 重叠：{train_ids & val_ids}" + + +def test_eval_id_unique_within_set(loop_root: Path) -> None: + for fname in ("train.evalset.json", "val.evalset.json"): + data = _load(loop_root / "data" / fname) + ids = [c["eval_id"] for c in data["eval_cases"]] + assert len(ids) == len(set(ids)), f"{fname} 中 eval_id 重复" + + +def test_state_metadata_keys(loop_root: Path) -> None: + """每条 case 必须带 split / slice / risk_level / protected / scenario_tag。""" + required = {"split", "slice", "risk_level", "protected", "scenario_tag"} + for fname in ("train.evalset.json", "val.evalset.json"): + data = _load(loop_root / "data" / fname) + for c in data["eval_cases"]: + state = c.get("session_input", {}).get("state", {}) + missing = required - set(state.keys()) + assert not missing, f"{fname} {c['eval_id']} 缺 state 字段：{missing}" + + +def test_one_protected_and_one_high_risk_in_val(loop_root: Path) -> None: + """val 集至少含一个 protected 与一个 high risk case，让 G3/G4 可触发。""" + data = _load(loop_root / "data" / "val.evalset.json") + states = [c["session_input"]["state"] for c in data["eval_cases"]] + assert any(s.get("protected") for s in states) + assert any(s.get("risk_level") == "high" for s in states) diff --git a/examples/optimization/eval_optimize_loop/tests/test_evaluator_error_handling.py b/examples/optimization/eval_optimize_loop/tests/test_evaluator_error_handling.py new file mode 100644 index 000000000..0107bbe8e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_evaluator_error_handling.py @@ -0,0 +1,81 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""验证 _run_evaluator 不吞非 case-failure 断言，以及复现命令的 shell 引用。""" + +from __future__ import annotations + +import argparse +import shlex +from pathlib import Path + +import pytest + +import pipeline +import runner +from trpc_agent_sdk.evaluation import EvaluateResult + + +class _StubExecuter: + """evaluate() 抛 AssertionError；get_result() 返回预置结果。""" + + def __init__(self, result): + self._result = result + + async def evaluate(self): + raise AssertionError("boom") + + def get_result(self): + return self._result + + +@pytest.mark.asyncio +async def test_assertion_without_result_is_not_swallowed(tmp_path: Path, monkeypatch) -> None: + """配置/契约类断言（结果尚未生成）必须重新抛出，不能伪装成空结果。""" + + stub = _StubExecuter(result=None) + monkeypatch.setattr( + runner.AgentEvaluator, "get_executer", staticmethod(lambda *a, **k: stub) + ) + evalset = tmp_path / "set.evalset.json" + evalset.write_text("{}", encoding="utf-8") + + with pytest.raises(RuntimeError, match="aborted before producing results"): + await runner._run_evaluator(evalset) + + +@pytest.mark.asyncio +async def test_case_failure_assertion_keeps_structured_result(tmp_path: Path, monkeypatch) -> None: + """case 失败断言仍返回 executer 里的结构化结果。""" + + expected = EvaluateResult(results_by_eval_set_id={}) + stub = _StubExecuter(result=expected) + monkeypatch.setattr( + runner.AgentEvaluator, "get_executer", staticmethod(lambda *a, **k: stub) + ) + evalset = tmp_path / "set.evalset.json" + evalset.write_text("{}", encoding="utf-8") + + result = await runner._run_evaluator(evalset) + assert result is expected + + +def test_repro_cmd_quotes_paths_with_spaces() -> None: + """含空格的路径必须被引用，复现命令可安全粘贴执行。""" + + args = argparse.Namespace( + mode="fake", + scenario="success", + candidate_file="C:/tmp/my candidate.md", + apply=True, + optimizer_config=None, + ) + cmd = pipeline._build_repro_cmd(args) + parsed = shlex.split(cmd) + assert "--candidate-file=C:/tmp/my candidate.md" in parsed + assert "--apply" in parsed + assert not any(part == "candidate.md" for part in parsed) diff --git a/examples/optimization/eval_optimize_loop/tests/test_evaluator_evidence.py b/examples/optimization/eval_optimize_loop/tests/test_evaluator_evidence.py new file mode 100644 index 000000000..e518e390f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_evaluator_evidence.py @@ -0,0 +1,260 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Evaluator evidence extraction and labeled holdout tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +import attribution +import gates +import report +import runner + +pytestmark = pytest.mark.asyncio + + +def _content(text: str) -> dict: + return {"role": "model", "parts": [{"text": text}]} + + +def _invocation( + invocation_id: str, + *, + query: str, + answer: str, + tools: list[dict], +) -> dict: + return { + "invocation_id": invocation_id, + "user_content": {"role": "user", "parts": [{"text": query}]}, + "final_response": _content(answer), + "intermediate_data": {"tool_uses": tools, "tool_responses": []}, + } + + +async def test_tool_and_parameter_categories_come_from_real_evaluator_trace( + tmp_path: Path, +) -> None: + expected_tool = {"id": "t1", "name": "search", "args": {"query": "pricing"}} + trace = { + "eval_set_id": "attribution_trace", + "eval_cases": [ + { + "eval_id": "tool_missing", + "eval_mode": "trace", + "actual_conversation": [ + _invocation( + "actual-1", + query="find pricing", + answer="pricing found", + tools=[], + ) + ], + "conversation": [ + _invocation( + "expected-1", + query="find pricing", + answer="pricing found", + tools=[expected_tool], + ) + ], + }, + { + "eval_id": "param_wrong", + "eval_mode": "trace", + "actual_conversation": [ + _invocation( + "actual-2", + query="find pricing", + answer="pricing found", + tools=[ + { + "id": "a2", + "name": "search", + "args": {"query": "weather"}, + } + ], + ) + ], + "conversation": [ + _invocation( + "expected-2", + query="find pricing", + answer="pricing found", + tools=[expected_tool], + ) + ], + }, + ], + } + config = { + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "threshold": 1.0, + "criterion": { + "tool_trajectory": { + "default": { + "name": {"match": "exact", "case_insensitive": False}, + "arguments": {"match": "exact"}, + }, + "order_sensitive": False, + "subset_matching": False, + } + }, + } + ] + } + evalset_path = tmp_path / "tool_trace.evalset.json" + config_path = tmp_path / "test_config.json" + evalset_path.write_text(json.dumps(trace, ensure_ascii=False, indent=2), encoding="utf-8") + config_path.write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding="utf-8") + + result = await runner._run_evaluator( + evalset_path, + metric_config_path=config_path, + ) + flattened = runner._flatten_results(result) + + categories: dict[str, str] = {} + for case in trace["eval_cases"]: + record = runner._case_record( + case=case, + split="holdout", + champion_runs=flattened[case["eval_id"]], + challenger_runs=flattened[case["eval_id"]], + ) + assert record.actual_text == "pricing found" + assert record.expected_text == "pricing found" + assert record.metric_results + assert record.trace_ref + categories[case["eval_id"]] = attribution.from_case_record(record).category + + assert categories == { + "tool_missing": "tool_call_error", + "param_wrong": "param_error", + } + + +async def test_labeled_evidence_holdout_reports_actual_accuracy( + loop_root: Path, + tmp_path: Path, +) -> None: + """The checked-in public holdout is measured, never called a hidden set.""" + + payload = json.loads((loop_root / "data" / "attribution_holdout.json").read_text(encoding="utf-8")) + correct = 0 + predictions: dict[str, str] = {} + records: list[runner.CaseRecord] = [] + for case in payload["cases"]: + result = attribution.classify_with_reason( + attribution.AttributionInput( + status=case["status"], + metric_names_failed=tuple(case.get("failed_metrics", [])), + metric_reasons=tuple(case.get("metric_reasons", [])), + expected_text=case.get("expected_text"), + actual_text=case.get("actual_text"), + expected_tool_uses=tuple(case.get("expected_tool_uses", [])), + actual_tool_uses=tuple(case.get("actual_tool_uses", [])), + expected_tool_responses=tuple(case.get("expected_tool_responses", [])), + actual_tool_responses=tuple(case.get("actual_tool_responses", [])), + error_message=case.get("error_message"), + ) + ) + predictions[case["eval_id"]] = result.category + correct += result.category == case["expected_category"] + records.append( + runner.CaseRecord( + eval_id=case["eval_id"], + split="val", + slice_name="holdout", + risk_level="low", + protected=False, + scenario_tag="deliberately_ignored", + champion_status="FAILED", + challenger_status=case["status"], + champion_score=0.0, + challenger_score=0.0, + expected_text=case.get("expected_text"), + actual_text=case.get("actual_text"), + metric_results=[ + { + "metric_name": metric_name, + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": "; ".join(case.get("metric_reasons", [])), + "rubric_scores": ( + [{"id": "r1", "score": 0.0, "reason": "failed"}] if "rubric" in metric_name else [] + ), + } + for metric_name in case.get("failed_metrics", []) + ], + expected_tool_uses=case.get("expected_tool_uses", []), + actual_tool_uses=case.get("actual_tool_uses", []), + expected_tool_responses=case.get("expected_tool_responses", []), + actual_tool_responses=case.get("actual_tool_responses", []), + error_message=case.get("error_message"), + failure_reasons=case.get("metric_reasons", []), + trace_ref=f"attribution_holdout.json#eval_id={case['eval_id']}", + ) + ) + + assert len(predictions) == 8 + assert correct / len(predictions) == 1.0 + + frozen = runner.FrozenInputs( + run_id="holdout-report", + champion_sha256="champion", + challenger_sha256="challenger", + train_sha256="train", + val_sha256="val", + metric_config_sha256="metric", + run_config_sha256="run", + optimizer_config_sha256=None, + seed=42, + started_at="2026-07-28T00:00:00.000Z", + mode="fixture", + candidate_source="labeled_holdout", + ) + split = runner.SplitResult(champion_avg=0.0, challenger_avg=0.0) + artifact = runner.RunArtifact( + frozen=frozen, + train=split, + val=split, + cases=records, + champion_train_avg=0.0, + champion_val_avg=0.0, + artifact_dir=tmp_path, + cost_status="measured", + total_tokens=0, + total_cost=0.0, + duration_seconds=0.0, + champion_prompt_text="", + challenger_prompt_text="", + ) + report_payload = report.build_report_dict( + artifact, + gates.Decision(accepted=False, violated=["G1"], reasons=["fixture"]), + applied=False, + before_apply_sha256="champion", + after_apply_sha256=None, + repro_cmd="pytest test_evaluator_evidence.py", + ) + report.write_report(report_payload, out_dir=tmp_path) + persisted = json.loads((tmp_path / "optimization_report.json").read_text(encoding="utf-8")) + per_case = {case["eval_id"]: case for case in persisted["per_case"]} + assert per_case["holdout_tool"]["evidence"]["expected_tool_uses"] + assert per_case["holdout_param"]["evidence"]["attribution"]["parameter_differences"] + assert per_case["holdout_rubric"]["evidence"]["metric_results"][0]["rubric_scores"] + assert per_case["holdout_infra"]["failure_kind"] == "infrastructure_failure" + assert per_case["holdout_reply"]["failure_kind"] == "agent_quality_failure" diff --git a/examples/optimization/eval_optimize_loop/tests/test_fake_agent.py b/examples/optimization/eval_optimize_loop/tests/test_fake_agent.py new file mode 100644 index 000000000..10912510c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_fake_agent.py @@ -0,0 +1,264 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""fake_agent 行为约束： +- 只有 FAKE_CONTROLS 块内 ``KEY=true`` 才启用行为 +- 注释文字、自然语言叙述不得触发 marker +- scenario 三类候选可正确生成 +- 真值表：Champion/Success/Overfit 在 train/val protected/val 其他 上的对错 +""" + +from __future__ import annotations + +from pathlib import Path + +import fake_agent + +# ---- 固定的 evalset 样例（与 data/evalset 同 schema，独立于文件） ---- +_TRAIN_CASE = { + "eval_id": "train_x", + "session_input": { + "app_name": "x", + "user_id": "train", + "state": { + "split": "train", + "slice": "math", + "risk_level": "low", + "protected": False, + "scenario_tag": "add_steps", + }, + }, + "conversation": [ + { + "invocation_id": "t", + "user_content": {"parts": [{"text": "q"}], "role": "user"}, + "final_response": {"parts": [{"text": "步骤：1\n答案：1 个"}], "role": "model"}, + } + ], +} +_VAL_PROTECTED_CASE = { + "eval_id": "val_p", + "session_input": { + "app_name": "x", + "user_id": "val", + "state": { + "split": "val", + "slice": "risky", + "risk_level": "high", + "protected": True, + "scenario_tag": "protected", + }, + }, + "conversation": [ + { + "invocation_id": "v", + "user_content": {"parts": [{"text": "q2"}], "role": "user"}, + "final_response": {"parts": [{"text": "答案：2 人"}], "role": "model"}, + } + ], +} +_VAL_OTHER_CASE = { + "eval_id": "val_o", + "session_input": { + "app_name": "x", + "user_id": "val", + "state": { + "split": "val", + "slice": "math", + "risk_level": "low", + "protected": False, + "scenario_tag": "add_steps", + }, + }, + "conversation": [ + { + "invocation_id": "v2", + "user_content": {"parts": [{"text": "q3"}], "role": "user"}, + "final_response": {"parts": [{"text": "答案：3 个"}], "role": "model"}, + } + ], +} + + +# ---- parse_behavior：只读 FAKE_CONTROLS 块，严格 true ---- +def test_parse_behavior_baseline_no_block() -> None: + b = fake_agent.parse_behavior("完全没有控制块的 prompt") + assert not b.add_steps + assert not b.memorize_train + + +def test_parse_behavior_explicit_false() -> None: + text = """ + + """ + b = fake_agent.parse_behavior(text) + assert not b.add_steps + assert not b.memorize_train + + +def test_parse_behavior_add_steps_true() -> None: + text = """ + + """ + b = fake_agent.parse_behavior(text) + assert b.add_steps + assert not b.memorize_train + + +def test_parse_behavior_memorize_true() -> None: + text = """ + + """ + b = fake_agent.parse_behavior(text) + assert not b.add_steps + assert b.memorize_train + + +def test_parse_behavior_rejects_true_with_caps() -> None: + """值必须严格小写 true；True / TRUE / yes 均视为 false。""" + text = """ + + """ + b = fake_agent.parse_behavior(text) + assert not b.add_steps + assert not b.memorize_train + + +def test_parse_behavior_ignores_marker_in_comments() -> None: + """注释 / 文档叙述里出现 ADD_STEPS=true 不得触发行为。""" + text = """ + # 示例：ADD_STEPS=true 是启用标记 + 文档中提到 MEMORIZE_TRAIN=true 只在 fake 模式生效 + + """ + b = fake_agent.parse_behavior(text) + assert not b.add_steps + assert not b.memorize_train + + +# ---- candidates ---- +def test_candidate_builders_exist() -> None: + for s in ("success", "no_effect", "overfit"): + c = fake_agent.build_candidate(s) + assert isinstance(c, str) and len(c) > 0 + + +def test_success_candidate_has_add_steps_true() -> None: + b = fake_agent.parse_behavior(fake_agent.build_candidate("success")) + assert b.add_steps + assert not b.memorize_train + + +def test_overfit_candidate_has_memorize_true() -> None: + b = fake_agent.parse_behavior(fake_agent.build_candidate("overfit")) + assert not b.add_steps + assert b.memorize_train + + +def test_no_effect_candidate_is_baseline() -> None: + b = fake_agent.parse_behavior(fake_agent.build_candidate("no_effect")) + assert not b.add_steps + assert not b.memorize_train + + +def test_champion_prompt_baseline_behavior() -> None: + """prompts/system.md 的 baseline 必须解析为全 false。""" + from pathlib import Path + + champ = Path(__file__).resolve().parent.parent / "prompts" / "system.md" + b = fake_agent.parse_behavior(champ.read_text(encoding="utf-8")) + assert not b.add_steps + assert not b.memorize_train + + +# ---- 真值表：根据 README 决策矩阵 ---- +def test_baseline_train_wrong_val_protected_right_val_other_wrong() -> None: + """FAKE_CONTROLS 全 false：train 错 / val protected 对 / val 其他 错。""" + prompt = """ + + """ + assert "无法回答" in fake_agent.gen_final_response_for_case(prompt, case=_TRAIN_CASE) + assert "无法回答" not in fake_agent.gen_final_response_for_case( + prompt, + case=_VAL_PROTECTED_CASE, + ) + assert "无法回答" in fake_agent.gen_final_response_for_case( + prompt, + case=_VAL_OTHER_CASE, + ) + + +def test_add_steps_all_correct() -> None: + """ADD_STEPS=true：所有 case 都对（输出 expected_response 原文）。""" + prompt = fake_agent.build_candidate("success") + for case in (_TRAIN_CASE, _VAL_PROTECTED_CASE, _VAL_OTHER_CASE): + out = fake_agent.gen_final_response_for_case(prompt, case=case) + assert "无法回答" not in out + expected = case["conversation"][0]["final_response"]["parts"][0]["text"] + assert out == expected + + +def test_memorize_train_only_train_correct() -> None: + """MEMORIZE_TRAIN=true：train 对 / val (含 protected) 全错。""" + prompt = fake_agent.build_candidate("overfit") + train_out = fake_agent.gen_final_response_for_case(prompt, case=_TRAIN_CASE) + assert "无法回答" not in train_out + assert train_out == _TRAIN_CASE["conversation"][0]["final_response"]["parts"][0]["text"] + + val_prot = fake_agent.gen_final_response_for_case(prompt, case=_VAL_PROTECTED_CASE) + assert "无法回答" in val_prot + + val_other = fake_agent.gen_final_response_for_case(prompt, case=_VAL_OTHER_CASE) + assert "无法回答" in val_other + + +def test_gen_actual_conversation_produces_trace_cases() -> None: + """gen_actual_conversation 输出结构正确：每个 case 含 eval_mode=trace 和 actual_conversation。""" + evalset = {"eval_set_id": "x", "eval_cases": [_TRAIN_CASE, _VAL_PROTECTED_CASE]} + out = fake_agent.gen_actual_conversation( + fake_agent.build_candidate("success"), + evalset, + ) + assert len(out) == 2 + for new_case in out: + assert new_case["eval_mode"] == "trace" + assert len(new_case["actual_conversation"]) == 1 + assert "final_response" in new_case["actual_conversation"][0] + + +# ---- 暗道禁止 ---- +def test_no_sha_switch_no_dark_path() -> None: + """静态扫描 fake_agent 源码，禁止任何哈希相关分支。""" + src = (Path(fake_agent.__file__)).read_text(encoding="utf-8") + forbidden = ["hashlib", "sha256", ".__hash__"] + for kw in forbidden: + assert kw not in src, f"fake_agent.py 不允许使用 {kw}" + + +def test_no_identity_keyword_branching() -> None: + """静态扫描，禁止 'champion' / 'challenger' / 'scenario ==' 形式的身份分支。""" + src = (Path(fake_agent.__file__)).read_text(encoding="utf-8").lower() + assert "champion" not in src + assert "challenger" not in src + # 不允许出现基于 scenario 字符串的分支 + assert "scenario ==" not in src + assert "scenario_param" not in src diff --git a/examples/optimization/eval_optimize_loop/tests/test_gates.py b/examples/optimization/eval_optimize_loop/tests/test_gates.py new file mode 100644 index 000000000..690717849 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_gates.py @@ -0,0 +1,205 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""G1-G7 gate 单元测试。纯规则，无需 SDK。""" + +from __future__ import annotations + +import gates + + +def _case( + eval_id="c1", + split="val", + slice_name="math", + risk_level="low", + protected=False, + champion_status="PASSED", + challenger_status="PASSED", + champion_score=1.0, + challenger_score=1.0, +): + return gates.CaseDelta( + eval_id=eval_id, + split=split, + slice_name=slice_name, + risk_level=risk_level, + protected=protected, + champion_status=champion_status, + challenger_status=challenger_status, + champion_score=champion_score, + challenger_score=challenger_score, + ) + + +def test_g1_accept_when_val_lift_meets_threshold() -> None: + cases = [_case(champion_score=0.5, challenger_score=0.8)] + d = gates.evaluate(cases) + assert d.accepted + assert "G1" not in d.violated + + +def test_g1_reject_when_val_lift_insufficient() -> None: + cases = [_case(champion_score=0.5, challenger_score=0.51)] + d = gates.evaluate(cases, config=gates.GateConfig(min_val_lift=0.02)) + assert not d.accepted + assert "G1" in d.violated + + +def test_g2_overfit_train_up_val_down_rejected() -> None: + train = [ + _case("t1", split="train", champion_score=0.3, challenger_score=0.9), + _case("t2", split="train", champion_score=0.3, challenger_score=0.9), + ] + val = [ + _case("v1", champion_score=0.7, challenger_score=0.4), + ] + d = gates.evaluate(train + val) + assert "G2" in d.violated + assert not d.accepted + # 理由中必须提到过拟合 + assert any("过拟合" in r for r in d.reasons) + + +def test_g2_not_triggered_when_both_improve() -> None: + train = [_case("t", split="train", champion_score=0.3, challenger_score=0.9)] + val = [_case("v", champion_score=0.5, challenger_score=0.8)] + d = gates.evaluate(train + val) + assert "G2" not in d.violated + + +def test_g3_new_hard_fail_rejected() -> None: + cases = [ + _case( + eval_id="h", + risk_level="high", + champion_status="PASSED", + challenger_status="FAILED", + champion_score=1.0, + challenger_score=0.0, + ), + # 必须有 val lift，否则会先被 G1 卡掉，看不出 G3 + _case(eval_id="v", champion_score=0.0, challenger_score=1.0), + ] + d = gates.evaluate(cases) + assert "G3" in d.violated + + +def test_g3_high_risk_but_already_failing_not_counted() -> None: + cases = [ + _case( + eval_id="h", + risk_level="high", + champion_status="FAILED", + challenger_status="FAILED", + champion_score=0.0, + challenger_score=0.0, + ), + _case(eval_id="v", champion_score=0.0, challenger_score=1.0), + ] + d = gates.evaluate(cases) + assert "G3" not in d.violated + + +def test_g4_protected_regression_rejected() -> None: + cases = [ + _case( + eval_id="p", + protected=True, + champion_score=1.0, + challenger_score=0.5, + ), + _case(eval_id="v", champion_score=0.0, challenger_score=1.0), + ] + d = gates.evaluate(cases) + assert "G4" in d.violated + + +def test_g5_slice_regression_rejected() -> None: + cases = [ + _case("a1", slice_name="alpha", champion_score=1.0, challenger_score=1.0), + _case( + "a2", + slice_name="beta", + champion_score=1.0, + challenger_score=0.0, + ), + # 整体 val lift 必须够，否则会被 G1 拦下 + _case("a3", slice_name="gamma", champion_score=0.0, challenger_score=1.0), + _case("a4", slice_name="gamma", champion_score=0.0, challenger_score=1.0), + ] + d = gates.evaluate(cases, config=gates.GateConfig(slice_tolerance=0.05)) + assert "G5" in d.violated + + +def test_g6_cost_unavailable_rejected() -> None: + cases = [_case(champion_score=0.0, challenger_score=1.0)] + d = gates.evaluate(cases, cost_status="unavailable") + assert "G6" in d.violated + + +def test_g6_cost_over_budget_rejected() -> None: + cases = [_case(champion_score=0.0, challenger_score=1.0)] + d = gates.evaluate( + cases, + cost_status="measured", + total_tokens=10_000_000, + config=gates.GateConfig(budget_tokens=100_000), + ) + assert "G6" in d.violated + + +def test_g6_usd_cost_over_budget_rejected() -> None: + cases = [_case(champion_score=0.0, challenger_score=1.0)] + decision = gates.evaluate( + cases, + cost_status="measured", + total_tokens=100, + total_cost=1.5, + config=gates.GateConfig(budget_usd=1.0), + ) + assert "G6" in decision.violated + + +def test_g6_measured_with_missing_value_rejected() -> None: + cases = [_case(champion_score=0.0, challenger_score=1.0)] + decision = gates.evaluate( + cases, + cost_status="measured", + total_tokens=None, + total_cost=None, + ) + assert "G6" in decision.violated + + +def test_g6_cost_measured_zero_passes() -> None: + cases = [_case(champion_score=0.0, challenger_score=1.0)] + d = gates.evaluate(cases, cost_status="measured", total_tokens=0) + assert "G6" not in d.violated + + +def test_g7_epsilon_guard() -> None: + """val_delta 接近 0 时，即使 G1-G6 全过，G7 也拦下。""" + cases = [ + _case(champion_score=1.0, challenger_score=1.0001), + ] + d = gates.evaluate( + cases, + cost_status="measured", + config=gates.GateConfig(min_val_lift=0.0, epsilon=0.001), + ) + assert "G7" in d.violated + + +def test_all_pass_accept() -> None: + cases = [ + _case(eval_id="t1", split="train", champion_score=0.4, challenger_score=0.9), + _case(eval_id="v1", champion_score=0.4, challenger_score=0.9), + ] + d = gates.evaluate(cases) + assert d.accepted + assert d.violated == [] diff --git a/examples/optimization/eval_optimize_loop/tests/test_native_optimize.py b/examples/optimization/eval_optimize_loop/tests/test_native_optimize.py new file mode 100644 index 000000000..e6701950e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_native_optimize.py @@ -0,0 +1,231 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""End-to-end tests for the native AgentOptimizer integration.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.asyncio + +_CANDIDATE = """# Native Optimizer Candidate + +Return the labeled reference answer for every registered evaluation query. + +TEST_NATIVE_GOOD=true +""" + + +def _answers(loop_root: Path) -> dict[str, str]: + answers: dict[str, str] = {} + for name in ("train.evalset.json", "val.evalset.json"): + payload = json.loads((loop_root / "data" / name).read_text(encoding="utf-8")) + for case in payload["eval_cases"]: + invocation = case["conversation"][0] + query = invocation["user_content"]["parts"][0]["text"] + answer = invocation["final_response"]["parts"][0]["text"] + answers[query] = answer + return answers + + +def _install_optimizer_stub( + monkeypatch, + *, + total_tokens: int, + total_cost: float, + reflection_calls: int, +) -> list[dict]: + from trpc_agent_sdk.evaluation import AgentOptimizer + + calls: list[dict] = [] + + async def optimize_stub(**kwargs): + calls.append(kwargs) + assert kwargs["update_source"] is False + baseline = await kwargs["target_prompt"].read_all() + assert "TEST_NATIVE_GOOD" not in baseline["system"] + + output_dir = Path(kwargs["output_dir"]) + (output_dir / "rounds").mkdir(parents=True, exist_ok=True) + (output_dir / "best_prompts").mkdir(parents=True, exist_ok=True) + (output_dir / "baseline_prompts").mkdir(parents=True, exist_ok=True) + (output_dir / "best_prompts" / "system.md").write_text(_CANDIDATE, encoding="utf-8") + (output_dir / "baseline_prompts" / "system.md").write_text(baseline["system"], encoding="utf-8") + round_record = { + "round_index": 1, + "input_prompt_sha256": "baseline", + "candidate_prompt": _CANDIDATE, + "accepted": True, + "validation_score": 1.0, + } + (output_dir / "rounds" / "round_001.json").write_text( + json.dumps(round_record, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + (output_dir / "result.json").write_text( + json.dumps({"status": "SUCCEEDED", "best_prompts": {"system": _CANDIDATE}}), + encoding="utf-8", + ) + (output_dir / "config.snapshot.json").write_text("{}\n", encoding="utf-8") + (output_dir / "run.log").write_text("stub optimizer succeeded\n", encoding="utf-8") + (output_dir / "summary.txt").write_text("stub optimizer succeeded\n", encoding="utf-8") + + return SimpleNamespace( + algorithm="test_native_stub", + status="SUCCEEDED", + finish_reason="COMPLETED", + stop_reason="completed", + error_message="", + best_prompts={"system": _CANDIDATE}, + total_rounds=1, + rounds=[round_record], + baseline_pass_rate=0.0, + best_pass_rate=1.0, + pass_rate_improvement=1.0, + total_token_usage={ + "prompt": total_tokens, + "completion": 0, + "total": total_tokens, + }, + total_llm_cost=total_cost, + total_reflection_lm_calls=reflection_calls, + ) + + monkeypatch.setattr(AgentOptimizer, "optimize", staticmethod(optimize_stub)) + return calls + + +def _call_agent(loop_root: Path): + answers = _answers(loop_root) + prompt_path = loop_root / "prompts" / "system.md" + + async def call_agent(query: str) -> str: + if query not in answers: + raise KeyError(query) + if "TEST_NATIVE_GOOD" in prompt_path.read_text(encoding="utf-8"): + return answers[query] + return "答案：0" + + # This deterministic callback makes no provider call; zero is measured. + call_agent.cost_status = "measured" + call_agent.total_tokens = 0 + call_agent.total_cost = 0.0 + return call_agent + + +async def test_native_optimize_candidate_reaches_regression_and_gate( + monkeypatch, + loop_root: Path, +) -> None: + calls = _install_optimizer_stub( + monkeypatch, + total_tokens=123, + total_cost=0.0123, + reflection_calls=1, + ) + + import pipeline + + rc = await pipeline.amain( + ["--mode", "optimize"], + call_agent=_call_agent(loop_root), + ) + assert rc == 0 + assert len(calls) == 1 + + optimize_call = calls[0] + assert optimize_call["update_source"] is False + assert Path(optimize_call["train_dataset_path"]).name == "train.evalset.json" + assert Path(optimize_call["validation_dataset_path"]).name == "val.evalset.json" + + payload = json.loads((loop_root / "optimization_report.json").read_text(encoding="utf-8")) + assert payload["candidate_source"] == "agent_optimizer" + assert payload["decision"]["accepted"] is True + assert payload["results"]["train"]["delta"] > 0 + assert payload["results"]["val"]["delta"] > 0 + assert payload["optimizer"]["rounds"][0]["candidate_prompt"] == _CANDIDATE + assert Path(payload["optimizer"]["rounds"][0]["artifact_path"]).is_file() + assert payload["cost"] == { + "status": "measured", + "total_tokens": 123, + "total_usd": 0.0123, + } + assert all(Path(path).exists() for path in payload["optimizer"]["artifacts"].values()) + + audit_path = Path(payload["optimizer"]["artifacts"]["call_agent_audit"]) + audit = json.loads(audit_path.read_text(encoding="utf-8")) + assert audit + assert all("query" in entry and "eval_contexts" in entry for entry in audit) + assert all(context["eval_id"] != entry["query"] for entry in audit for context in entry["eval_contexts"]) + + +async def test_cost_unavailable_rejects_apply_and_preserves_champion( + monkeypatch, + loop_root: Path, +) -> None: + _install_optimizer_stub( + monkeypatch, + total_tokens=0, + total_cost=0.0, + reflection_calls=1, + ) + champion = loop_root / "prompts" / "system.md" + before = champion.read_bytes() + + import pipeline + + rc = await pipeline.amain( + ["--mode", "optimize", "--apply"], + call_agent=_call_agent(loop_root), + ) + assert rc == 2 + assert champion.read_bytes() == before + + payload = json.loads((loop_root / "optimization_report.json").read_text(encoding="utf-8")) + assert payload["decision"]["accepted"] is False + assert "G6" in payload["decision"]["violated"] + assert payload["audit"]["applied"] is False + assert payload["cost"] == { + "status": "unavailable", + "total_tokens": None, + "total_usd": None, + } + + +async def test_missing_model_config_writes_auditable_reject( + monkeypatch, + loop_root: Path, +) -> None: + for key in ( + "TRPC_AGENT_API_KEY", + "TRPC_AGENT_BASE_URL", + "TRPC_AGENT_MODEL_NAME", + ): + monkeypatch.delenv(key, raising=False) + + import pipeline + + rc = await pipeline.amain(["--mode", "optimize"]) + assert rc == 1 + payload = json.loads( + (loop_root / "optimization_report.json").read_text(encoding="utf-8") + ) + assert payload["decision"]["violated"] == ["OPTIMIZER_FAILURE", "G6"] + assert payload["cost"] == { + "status": "unavailable", + "total_tokens": None, + "total_usd": None, + } + assert "TRPC_AGENT_API_KEY" in payload["optimizer"]["error"] + assert Path( + payload["optimizer"]["artifacts"]["optimizer_error"] + ).is_file() diff --git a/examples/optimization/eval_optimize_loop/tests/test_report_fields.py b/examples/optimization/eval_optimize_loop/tests/test_report_fields.py new file mode 100644 index 000000000..a623cd667 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_report_fields.py @@ -0,0 +1,148 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Validate optimization report schema and persisted evidence.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.asyncio + + +REQUIRED_FROZEN_KEYS = { + "champion_sha256", + "challenger_sha256", + "train_sha256", + "val_sha256", + "metric_config_sha256", + "run_config_sha256", + "optimizer_config_sha256", + "seed", + "started_at", + "mode", + "candidate_source", + "gate_config", + "model_info", + "evaluator_info", + "optimizer_info", +} + +REQUIRED_PER_CASE_KEYS = { + "eval_id", + "split", + "slice", + "risk_level", + "protected", + "champion_status", + "challenger_status", + "champion_score", + "challenger_score", + "delta", + "category", + "transition", + "failure_kind", + "failure_reason", + "evidence", +} + +REQUIRED_AUDIT_KEYS = { + "applied", + "duration_seconds", + "candidate_source", + "artifact_dir", + "before_apply_sha256", + "after_apply_sha256", + "repro_cmd", + "artifacts", +} + + +@pytest.fixture(autouse=True) +def _no_api_key(monkeypatch): + for k in ("OPENAI_API_KEY", "TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + monkeypatch.delenv(k, raising=False) + + +async def test_report_has_all_required_fields(loop_root: Path) -> None: + import pipeline + + await pipeline.amain(["--mode", "fake", "--scenario", "no_effect"]) + report = json.loads((loop_root / "optimization_report.json").read_text(encoding="utf-8")) + + assert report["version"] == "2.0" + assert REQUIRED_FROZEN_KEYS <= set(report["frozen"].keys()) + assert {"train", "val"} <= set(report["results"].keys()) + for k in ("train", "val"): + assert {"champion_avg", "challenger_avg", "delta"} <= set(report["results"][k].keys()) + assert "train_delta" in report and "val_delta" in report + assert "cost_status" in report + assert "cost" in report + assert {"accepted", "violated", "reasons"} <= set(report["decision"].keys()) + assert REQUIRED_AUDIT_KEYS <= set(report["audit"].keys()) + + assert len(report["per_case"]) == 6 + for c in report["per_case"]: + assert REQUIRED_PER_CASE_KEYS <= set(c.keys()) + assert c["evidence"]["actual_text"] is not None + assert c["evidence"]["expected_text"] is not None + assert c["evidence"]["trace_ref"] + assert "metric_results" in c["evidence"] + + assert report["frozen"]["mode"] == "fake" + assert report["frozen"]["candidate_source"] == "candidate_file" + + +async def test_train_val_delta_consistent(loop_root: Path) -> None: + import pipeline + + await pipeline.amain(["--mode", "fake", "--scenario", "no_effect"]) + report = json.loads((loop_root / "optimization_report.json").read_text(encoding="utf-8")) + + # report.results.train.delta == report.train_delta + assert abs(report["results"]["train"]["delta"] - report["train_delta"]) < 1e-9 + assert abs(report["results"]["val"]["delta"] - report["val_delta"]) < 1e-9 + + +async def test_markdown_presented_and_aligned(loop_root: Path) -> None: + import pipeline + + await pipeline.amain(["--mode", "fake", "--scenario", "no_effect"]) + md = (loop_root / "optimization_report.md").read_text(encoding="utf-8") + # 必须含核心章节 + assert "决策" in md + assert "Gate 决策" in md + assert "逐 case 明细" in md + assert "审计" in md + assert "REJECT" in md or "ACCEPT" in md + + +async def test_artifacts_files_exist(loop_root: Path) -> None: + import pipeline + + await pipeline.amain(["--mode", "fake", "--scenario", "no_effect"]) + report = json.loads((loop_root / "optimization_report.json").read_text(encoding="utf-8")) + artifacts = report["audit"]["artifacts"] + for k, p in artifacts.items(): + path = Path(p) + assert path.exists(), f"{k} artifact 文件不存在：{p}" + + +async def test_checked_in_example_report_matches_v2_schema(loop_root: Path) -> None: + payload = json.loads( + (loop_root / "optimization_report.example.json").read_text(encoding="utf-8") + ) + assert payload["version"] == "2.0" + assert REQUIRED_FROZEN_KEYS <= set(payload["frozen"]) + assert REQUIRED_AUDIT_KEYS <= set(payload["audit"]) + assert REQUIRED_PER_CASE_KEYS <= set(payload["per_case"][0]) + assert payload["optimizer"]["rounds"][0]["artifact_path"] + categories = {case["category"] for case in payload["per_case"]} + assert {"tool_call_error", "param_error", "rubric_fail"} <= categories diff --git a/examples/optimization/eval_optimize_loop/tests/test_sha256_normalize.py b/examples/optimization/eval_optimize_loop/tests/test_sha256_normalize.py new file mode 100644 index 000000000..657433864 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_sha256_normalize.py @@ -0,0 +1,50 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""sha256 规范化测试：Windows CRLF 与 Unix LF 内容应产生相同 hash。""" + +from __future__ import annotations + +import re +from pathlib import Path + +from runner import new_run_id, sha256_file, sha256_text + + +def test_sha256_text_crlf_matches_lf() -> None: + lf_text = "步骤：1\n答案：1 个\n" + crlf_text = "步骤：1\r\n答案：1 个\r\n" + assert sha256_text(lf_text) == sha256_text(crlf_text) + + +def test_sha256_text_lf_unchanged() -> None: + text = "hello\nworld\n" + # 规范化后与原文本一致 → 与裸 hashlib 一致 + import hashlib + + expected = hashlib.sha256(text.encode("utf-8")).hexdigest() + assert sha256_text(text) == expected + + +def test_sha256_file_crlf_matches_lf(tmp_path: Path) -> None: + lf_file = tmp_path / "lf.md" + crlf_file = tmp_path / "crlf.md" + lf_file.write_bytes("步骤：1\n答案：1 个\n".encode("utf-8")) + crlf_file.write_bytes("步骤：1\r\n答案：1 个\r\n".encode("utf-8")) + assert sha256_file(lf_file) == sha256_file(crlf_file) + + +def test_run_id_uses_utc_microseconds_and_random_suffix() -> None: + run_ids = [new_run_id() for _ in range(200)] + assert len(set(run_ids)) == len(run_ids) + assert all( + re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{6}Z-[0-9a-f]{8}", + run_id, + ) + for run_id in run_ids + ) diff --git a/examples/optimization/eval_optimize_loop/tests/test_truth_table.py b/examples/optimization/eval_optimize_loop/tests/test_truth_table.py new file mode 100644 index 000000000..6a05a590f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_truth_table.py @@ -0,0 +1,140 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2025 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""端到端真值表测试：直接调用 pipeline + runner + SDK evaluator， +对三个 fake 场景断言 case-level 分数、train/val delta、decision。 + +该测试是 Issue #91 Milestone A 的验收测试： +test_fake_agent.py 单独通过不能视为验收完成。 +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture(autouse=True) +def _no_api_key(monkeypatch): + """强制无 API key，验证 fake 模式不需 key 也能跑。""" + for k in ("OPENAI_API_KEY", "TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + monkeypatch.delenv(k, raising=False) + + +def _per_case_scores(report: dict) -> dict[str, dict[str, float]]: + """把 per_case 列表整理成 {eval_id: {champ, chall, delta}}。""" + out = {} + for c in report["per_case"]: + out[c["eval_id"]] = { + "champion": c["champion_score"], + "challenger": c["challenger_score"], + "delta": c["delta"], + "split": c["split"], + "protected": c["protected"], + } + return out + + +async def _run(scenario: str) -> dict: + import pipeline + + rc = await pipeline.amain(["--mode", "fake", "--scenario", scenario]) + report_path = pipeline._HERE / "optimization_report.json" + report = json.loads(report_path.read_text(encoding="utf-8")) + report["_exit_code"] = rc + return report + + +# ---- 期望的真值表（与 README 一致） ---- +# Champion: train=[0,0,0] val=[0,1,0] (protected case 初始正确) +# success: train=[1,1,1] val=[1,1,1] ACCEPT +# no_effect: 同 Champion REJECT G1 +# overfit: train=[1,1,1] val=[0,0,0] REJECT G2 + 过拟合 + +# eval_id 来自 data/train.evalset.json + data/val.evalset.json +_TRAIN_IDS = ["train_add_steps", "train_noop", "train_overfit_mem"] +_VAL_IDS = ["val_add_steps_b", "val_protected_highrisk", "val_noop_b"] + + +async def test_success_candidate_truth_table() -> None: + """success 场景：train/val 全对，ACCEPT。""" + report = await _run("success") + + scores = _per_case_scores(report) + # Champion baseline + for tid in _TRAIN_IDS: + assert scores[tid]["champion"] == 0.0, f"{tid} champion should be 0" + assert scores["val_protected_highrisk"]["champion"] == 1.0, "protected case 初始正确" + for vid in ("val_add_steps_b", "val_noop_b"): + assert scores[vid]["champion"] == 0.0, f"{vid} champion should be 0" + + # Challenger 全对 + for tid in _TRAIN_IDS: + assert scores[tid]["challenger"] == 1.0, f"{tid} challenger should be 1" + for vid in _VAL_IDS: + assert scores[vid]["challenger"] == 1.0, f"{vid} challenger should be 1" + + # delta + assert report["train_delta"] > 0 + assert report["val_delta"] > 0 + assert report["val_delta"] >= 0.02, "val_delta >= min_val_lift" + + # decision + assert report["decision"]["accepted"] is True + assert report["decision"]["violated"] == [] + + +async def test_no_effect_candidate_truth_table() -> None: + """no_effect 场景：candidate 行为 = Champion，REJECT G1。""" + report = await _run("no_effect") + + scores = _per_case_scores(report) + # Champion 与 Challenger 分数完全相同 + for eid in _TRAIN_IDS + _VAL_IDS: + assert scores[eid]["champion"] == scores[eid]["challenger"], f"{eid} champ=chall" + # delta == 0 + assert abs(report["train_delta"]) < 1e-9 + assert abs(report["val_delta"]) < 1e-9 + + # decision: REJECT 且违反 G1 + assert report["decision"]["accepted"] is False + assert "G1" in report["decision"]["violated"] + + +async def test_overfit_candidate_truth_table() -> None: + """overfit 场景：train 涨 val 跌，REJECT G2 且理由含'过拟合'。""" + report = await _run("overfit") + + scores = _per_case_scores(report) + # Champion: train=[0,0,0], val=[0,1,0] + for tid in _TRAIN_IDS: + assert scores[tid]["champion"] == 0.0 + assert scores["val_protected_highrisk"]["champion"] == 1.0 + for vid in ("val_add_steps_b", "val_noop_b"): + assert scores[vid]["champion"] == 0.0 + + # Challenger: train=[1,1,1], val=[0,0,0] (含 protected 也跌) + for tid in _TRAIN_IDS: + assert scores[tid]["challenger"] == 1.0, f"{tid} challenger should be 1 (memorized)" + for vid in _VAL_IDS: + assert scores[vid]["challenger"] == 0.0, f"{vid} challenger should be 0 (forgot)" + + # delta: train > 0, val < 0 + assert report["train_delta"] > 0 + assert report["val_delta"] < 0, f"val_delta={report['val_delta']} should be negative" + + # decision: REJECT 且违反 G2（过拟合） + assert report["decision"]["accepted"] is False + assert "G2" in report["decision"]["violated"], report["decision"] + assert any("过拟合" in r for r in report["decision"]["reasons"]), report["decision"] + # G3（high-risk 新增 fail）与 G4（protected 退化）也应触发 + assert "G3" in report["decision"]["violated"] + assert "G4" in report["decision"]["violated"] diff --git a/tests/evaluation/test_agent_evaluator.py b/tests/evaluation/test_agent_evaluator.py index 8609ccc5d..ccbad24f2 100644 --- a/tests/evaluation/test_agent_evaluator.py +++ b/tests/evaluation/test_agent_evaluator.py @@ -5,17 +5,25 @@ # tRPC-Agent-Python is licensed under Apache-2.0. """Unit tests for agent evaluator (agent_evaluator).""" +import os + import pytest import trpc_agent_sdk.runners # noqa: F401 from trpc_agent_sdk.evaluation import EvalStatus +from trpc_agent_sdk.evaluation import EvalCase from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import EvalConfig from trpc_agent_sdk.evaluation import EvalMetricResult +from trpc_agent_sdk.evaluation import EvalSet from trpc_agent_sdk.evaluation import EvalSetAggregateResult from trpc_agent_sdk.evaluation import EvaluateResult from trpc_agent_sdk.evaluation import AgentEvaluator +from trpc_agent_sdk.evaluation import Invocation from trpc_agent_sdk.evaluation import PassNC +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import Part class TestPassNC: @@ -97,3 +105,77 @@ def test_pass_at_k_delegates(self): def test_pass_hat_k_delegates(self): """Test AgentEvaluator.pass_hat_k delegates to _eval_pass.""" assert AgentEvaluator.pass_hat_k(10, 5, 2) == 0.25 + + +class TestLoadEvalSetFromFile: + """Test suite for AgentEvaluator._load_eval_set_from_file. + + Covers the case-selector colon parsing. The drive-letter guard + ("an existing path containing ':' must load as-is") is exercised on + Windows by the absolute ``tmp_path`` ("C:\\...") and on POSIX by a + file whose name literally contains a colon. + """ + + @staticmethod + def _write_eval_set(tmp_path, case_ids): + """Write a minimal eval set file and return its absolute path.""" + cases = [ + EvalCase( + eval_id=case_id, + conversation=[ + Invocation( + invocation_id="i", + user_content=Content(parts=[Part(text="hi")]), + ), + ], + ) + for case_id in case_ids + ] + eval_set = EvalSet(eval_set_id="set1", eval_cases=cases) + file_path = tmp_path / "set.evalset.json" + file_path.write_text(eval_set.model_dump_json(), encoding="utf-8") + return str(file_path) + + def test_load_absolute_path_without_selector(self, tmp_path): + """An existing absolute path must load as-is. + + On Windows ``tmp_path`` starts with a drive letter, so the ':' guard + is hit here; POSIX coverage of the same guard lives in + ``test_load_existing_path_containing_colon``. + """ + file_path = self._write_eval_set(tmp_path, ["case_a", "case_b"]) + eval_set = AgentEvaluator._load_eval_set_from_file(file_path, EvalConfig(criteria={})) + assert [c.eval_id for c in eval_set.eval_cases] == ["case_a", "case_b"] + + @pytest.mark.skipif(os.name == "nt", reason="':' is not a legal filename character on Windows") + def test_load_existing_path_containing_colon(self, tmp_path): + """A colon inside an existing full path must not be split as a selector.""" + source = self._write_eval_set(tmp_path, ["case_a", "case_b"]) + colon_path = tmp_path / "drive:like.evalset.json" + with open(source, "r", encoding="utf-8") as f: + colon_path.write_text(f.read(), encoding="utf-8") + eval_set = AgentEvaluator._load_eval_set_from_file(str(colon_path), EvalConfig(criteria={})) + assert [c.eval_id for c in eval_set.eval_cases] == ["case_a", "case_b"] + + def test_load_with_case_selector(self, tmp_path): + """"file.json:case_id" selects a single case from the set. + + The full string never exists on disk while the part before the last + colon does, so the rpartition branch runs on every platform. + """ + file_path = self._write_eval_set(tmp_path, ["case_a", "case_b"]) + eval_set = AgentEvaluator._load_eval_set_from_file(f"{file_path}:case_b", EvalConfig(criteria={})) + assert [c.eval_id for c in eval_set.eval_cases] == ["case_b"] + assert eval_set.eval_set_id == "set1_case_b" + + def test_load_with_unknown_case_selector_raises(self, tmp_path): + """Selecting a case id that does not exist raises ValueError.""" + file_path = self._write_eval_set(tmp_path, ["case_a"]) + with pytest.raises(ValueError, match="not found"): + AgentEvaluator._load_eval_set_from_file(f"{file_path}:missing", EvalConfig(criteria={})) + + def test_load_missing_file_raises(self, tmp_path): + """A non-existing path (with or without ':') raises FileNotFoundError.""" + missing = str(tmp_path / "nope.evalset.json") + with pytest.raises(FileNotFoundError): + AgentEvaluator._load_eval_set_from_file(missing, EvalConfig(criteria={})) diff --git a/trpc_agent_sdk/evaluation/_agent_evaluator.py b/trpc_agent_sdk/evaluation/_agent_evaluator.py index a2e47009d..8aa6750a0 100644 --- a/trpc_agent_sdk/evaluation/_agent_evaluator.py +++ b/trpc_agent_sdk/evaluation/_agent_evaluator.py @@ -660,14 +660,18 @@ def _load_eval_set_from_file( FileNotFoundError: If file doesn't exist ValueError: If file format is invalid or eval case not found """ - # Check if file_path contains a case selector (ADK style: "file.json:case_id") + # Check if file_path contains a case selector (ADK style: "file.json:case_id"). + # A Windows drive letter also contains ":", so only treat the colon as a + # selector when the full string is not an existing file and the part + # before the last colon is. selected_case_id = None actual_file_path = eval_set_file - if ":" in eval_set_file: - parts = eval_set_file.split(":", 1) - actual_file_path = parts[0] - selected_case_id = parts[1] + if ":" in eval_set_file and not os.path.exists(eval_set_file): + file_part, _, case_part = eval_set_file.rpartition(":") + if file_part and os.path.exists(file_part): + actual_file_path = file_part + selected_case_id = case_part if not os.path.exists(actual_file_path): raise FileNotFoundError(f"Eval set file not found: {actual_file_path}")