构建 Evaluation + Optimization 的自动回归与提示词优化闭环

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-24-38/baseline_prompts/system_prompt.md

@@ -0,0 +1 @@
+You parse free-text postal addresses and return a JSON object.

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-24-38/best_prompts/system_prompt.md

@@ -0,0 +1 @@
+You parse free-text postal addresses and return a JSON object.

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-24-38/config.snapshot.json

@@ -0,0 +1,48 @@
+{
+  "evaluate": {
+    "metrics": [
+      {
+        "metric_name": "final_response_avg_score",
+        "threshold": 1.0,
+        "criterion": {
+          "final_response": {
+            "text": {
+              "match": "exact",
+              "case_insensitive": false
+            }
+          }
+        }
+      }
+    ],
+    "num_runs": 1
+  },
+  "optimize": {
+    "eval_case_parallelism": 2,
+    "stop": {
+      "required_metrics": "all"
+    },
+    "algorithm": {
+      "name": "gepa_reflective",
+      "seed": 42,
+      "reflection_lm": {
+        "model_name": "${TRPC_AGENT_MODEL_NAME}",
+        "base_url": "${TRPC_AGENT_BASE_URL}",
+        "api_key": "${TRPC_AGENT_API_KEY}",
+        "generation_config": { "max_tokens": 4096, "temperature": 0.6 }
+      },
+      "candidate_selection_strategy": "pareto",
+      "module_selector": "round_robin",
+      "frontier_type": "objective",
+      "reflection_minibatch_size": 3,
+      "reflection_history_top_k": 2,
+      "skip_perfect_score": true,
+      "perfect_score": 1.0,
+      "use_merge": true,
+      "max_merge_invocations": 3,
+      "merge_val_overlap_floor": 5,
+      "max_metric_calls": 60,
+      "score_threshold": 1.0,
+      "max_iterations_without_improvement": 4
+    }
+  }
+}

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-24-38/result.json

@@ -0,0 +1,37 @@
+{
+  "schemaVersion": "v1",
+  "algorithm": "gepa_reflective",
+  "status": "FAILED",
+  "finishReason": "error",
+  "stopReason": null,
+  "errorMessage": "frontier_type='objective' requires objective_scores to be provided by the evaluator, but none were found. Use an evaluator that returns objective_scores or use frontier_type='instance'.",
+  "baselinePassRate": 0.0,
+  "bestPassRate": 0.0,
+  "passRateImprovement": 0.0,
+  "baselineMetricBreakdown": {},
+  "bestMetricBreakdown": {},
+  "metricThresholds": {
+    "final_response_avg_score": 1.0
+  },
+  "perMetricBestCandidates": {},
+  "baselinePrompts": {
+    "system_prompt": "You parse free-text postal addresses and return a JSON object.\n"
+  },
+  "bestPrompts": {
+    "system_prompt": "You parse free-text postal addresses and return a JSON object.\n"
+  },
+  "totalRounds": 0,
+  "rounds": [],
+  "totalReflectionLmCalls": 0,
+  "totalJudgeModelCalls": 0,
+  "totalLlmCost": 0.0,
+  "totalTokenUsage": {
+    "prompt": 0,
+    "completion": 0,
+    "total": 0
+  },
+  "durationSeconds": 0.024374,
+  "startedAt": "2026-07-30T09:24:38.791593+00:00",
+  "finishedAt": "2026-07-30T09:24:38.815967+00:00",
+  "extras": {}
+}

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-24-38/summary.txt

@@ -0,0 +1,19 @@
+Optimization complete  |  status=FAILED  |  algorithm=gepa_reflective
+
+pass_rate     : 0.0000 -> 0.0000   (+0.0000, no improvement)
+rounds        : 0 accepted / 0 total
+duration      : 0.02s
+started_at    : 2026-07-30T09:24:38.791593+00:00
+finished_at   : 2026-07-30T09:24:38.815967+00:00
+error_message : frontier_type='objective' requires objective_scores to be provided by the evaluator, but none were found. Use an evaluator that returns objective_scores or use frontier_type='instance'.
+update_source : false
+output_dir    : /Users/lvjiajia/Desktop/trpc-agent-python-main/examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-24-38
+
+metric breakdown (threshold | baseline -> best):
+  - final_response_avg_score                 threshold 1.0000   nan -> nan
+
+best prompts:
+  - system_prompt                            63 chars   (best_prompts/system_prompt.md)
+
+artifacts directory: /Users/lvjiajia/Desktop/trpc-agent-python-main/examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-24-38
+  result.json   summary.txt   rounds/   run.log   baseline_prompts/   best_prompts/   config.snapshot.json

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-25-41/baseline_prompts/system_prompt.md

@@ -0,0 +1 @@
+You parse free-text postal addresses and return a JSON object.

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-25-41/best_prompts/system_prompt.md

@@ -0,0 +1 @@
+You parse free-text postal addresses and return a JSON object.

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-25-41/config.snapshot.json

@@ -0,0 +1,48 @@
+{
+  "evaluate": {
+    "metrics": [
+      {
+        "metric_name": "final_response_avg_score",
+        "threshold": 1.0,
+        "criterion": {
+          "final_response": {
+            "text": {
+              "match": "exact",
+              "case_insensitive": false
+            }
+          }
+        }
+      }
+    ],
+    "num_runs": 1
+  },
+  "optimize": {
+    "eval_case_parallelism": 2,
+    "stop": {
+      "required_metrics": "all"
+    },
+    "algorithm": {
+      "name": "gepa_reflective",
+      "seed": 42,
+      "reflection_lm": {
+        "model_name": "${TRPC_AGENT_MODEL_NAME}",
+        "base_url": "${TRPC_AGENT_BASE_URL}",
+        "api_key": "${TRPC_AGENT_API_KEY}",
+        "generation_config": { "max_tokens": 4096, "temperature": 0.6 }
+      },
+      "candidate_selection_strategy": "pareto",
+      "module_selector": "round_robin",
+      "frontier_type": "objective",
+      "reflection_minibatch_size": 3,
+      "reflection_history_top_k": 2,
+      "skip_perfect_score": true,
+      "perfect_score": 1.0,
+      "use_merge": true,
+      "max_merge_invocations": 3,
+      "merge_val_overlap_floor": 5,
+      "max_metric_calls": 60,
+      "score_threshold": 1.0,
+      "max_iterations_without_improvement": 4
+    }
+  }
+}

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-25-41/result.json

@@ -0,0 +1,37 @@
+{
+  "schemaVersion": "v1",
+  "algorithm": "gepa_reflective",
+  "status": "FAILED",
+  "finishReason": "error",
+  "stopReason": null,
+  "errorMessage": "frontier_type='objective' requires objective_scores to be provided by the evaluator, but none were found. Use an evaluator that returns objective_scores or use frontier_type='instance'.",
+  "baselinePassRate": 0.0,
+  "bestPassRate": 0.0,
+  "passRateImprovement": 0.0,
+  "baselineMetricBreakdown": {},
+  "bestMetricBreakdown": {},
+  "metricThresholds": {
+    "final_response_avg_score": 1.0
+  },
+  "perMetricBestCandidates": {},
+  "baselinePrompts": {
+    "system_prompt": "You parse free-text postal addresses and return a JSON object.\n"
+  },
+  "bestPrompts": {
+    "system_prompt": "You parse free-text postal addresses and return a JSON object.\n"
+  },
+  "totalRounds": 0,
+  "rounds": [],
+  "totalReflectionLmCalls": 0,
+  "totalJudgeModelCalls": 0,
+  "totalLlmCost": 0.0,
+  "totalTokenUsage": {
+    "prompt": 0,
+    "completion": 0,
+    "total": 0
+  },
+  "durationSeconds": 0.024032,
+  "startedAt": "2026-07-30T09:25:41.748195+00:00",
+  "finishedAt": "2026-07-30T09:25:41.772227+00:00",
+  "extras": {}
+}

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-25-41/summary.txt

@@ -0,0 +1,19 @@
+Optimization complete  |  status=FAILED  |  algorithm=gepa_reflective
+
+pass_rate     : 0.0000 -> 0.0000   (+0.0000, no improvement)
+rounds        : 0 accepted / 0 total
+duration      : 0.02s
+started_at    : 2026-07-30T09:25:41.748195+00:00
+finished_at   : 2026-07-30T09:25:41.772227+00:00
+error_message : frontier_type='objective' requires objective_scores to be provided by the evaluator, but none were found. Use an evaluator that returns objective_scores or use frontier_type='instance'.
+update_source : false
+output_dir    : /Users/lvjiajia/Desktop/trpc-agent-python-main/examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-25-41
+
+metric breakdown (threshold | baseline -> best):
+  - final_response_avg_score                 threshold 1.0000   nan -> nan
+
+best prompts:
+  - system_prompt                            63 chars   (best_prompts/system_prompt.md)
+
+artifacts directory: /Users/lvjiajia/Desktop/trpc-agent-python-main/examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-25-41
+  result.json   summary.txt   rounds/   run.log   baseline_prompts/   best_prompts/   config.snapshot.json

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-27-07/baseline_prompts/system_prompt.md

@@ -0,0 +1 @@
+You parse free-text postal addresses and return a JSON object.

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-27-07/best_prompts/system_prompt.md

@@ -0,0 +1 @@
+You parse free-text postal addresses and return a JSON object. The JSON must always contain exactly four keys in the following order: "city", "country", "postal_code", "street". If a field is not present in the input, set its value to null. Do not omit any keys. Output only the JSON object, with no extra whitespace, spaces, newlines, or explanatory text. Use compact JSON format (no spaces after colons or commas). Example: for "Sydney, Australia" output {"city":"Sydney","country":"Australia","postal_code":null,"street":null}.

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-27-07/config.snapshot.json

@@ -0,0 +1,48 @@
+{
+  "evaluate": {
+    "metrics": [
+      {
+        "metric_name": "final_response_avg_score",
+        "threshold": 1.0,
+        "criterion": {
+          "final_response": {
+            "text": {
+              "match": "exact",
+              "case_insensitive": false
+            }
+          }
+        }
+      }
+    ],
+    "num_runs": 1
+  },
+  "optimize": {
+    "eval_case_parallelism": 2,
+    "stop": {
+      "required_metrics": "all"
+    },
+    "algorithm": {
+      "name": "gepa_reflective",
+      "seed": 42,
+      "reflection_lm": {
+        "model_name": "${TRPC_AGENT_MODEL_NAME}",
+        "base_url": "${TRPC_AGENT_BASE_URL}",
+        "api_key": "${TRPC_AGENT_API_KEY}",
+        "generation_config": { "max_tokens": 4096, "temperature": 0.6 }
+      },
+      "candidate_selection_strategy": "pareto",
+      "module_selector": "round_robin",
+      "frontier_type": "objective",
+      "reflection_minibatch_size": 3,
+      "reflection_history_top_k": 2,
+      "skip_perfect_score": true,
+      "perfect_score": 1.0,
+      "use_merge": true,
+      "max_merge_invocations": 3,
+      "merge_val_overlap_floor": 5,
+      "max_metric_calls": 60,
+      "score_threshold": 1.0,
+      "max_iterations_without_improvement": 4
+    }
+  }
+}

examples/optimization/advanced_strategies/runs/advanced_2026-07-30T17-27-07/result.json

@@ -0,0 +1,85 @@
+{
+  "schemaVersion": "v1",
+  "algorithm": "gepa_reflective",
+  "status": "SUCCEEDED",
+  "finishReason": "completed",
+  "stopReason": "score_threshold",
+  "errorMessage": "",
+  "baselinePassRate": 0.16666666666666666,
+  "bestPassRate": 1.0,
+  "passRateImprovement": 0.8333333333333334,
+  "baselineMetricBreakdown": {
+    "final_response_avg_score": 0.16666666666666666
+  },
+  "bestMetricBreakdown": {
+    "final_response_avg_score": 1.0
+  },
+  "metricThresholds": {
+    "final_response_avg_score": 1.0
+  },
+  "perMetricBestCandidates": {
+    "final_response_avg_score": [
+      1
+    ]
+  },
+  "baselinePrompts": {
+    "system_prompt": "You parse free-text postal addresses and return a JSON object.\n"
+  },
+  "bestPrompts": {
+    "system_prompt": "You parse free-text postal addresses and return a JSON object. The JSON must always contain exactly four keys in the following order: \"city\", \"country\", \"postal_code\", \"street\". If a field is not present in the input, set its value to null. Do not omit any keys. Output only the JSON object, with no extra whitespace, spaces, newlines, or explanatory text. Use compact JSON format (no spaces after colons or commas). Example: for \"Sydney, Australia\" output {\"city\":\"Sydney\",\"country\":\"Australia\",\"postal_code\":null,\"street\":null}."
+  },
+  "totalRounds": 1,
+  "rounds": [
+    {
+      "round": 1,
+      "optimizedFieldNames": [
+        "system_prompt"
+      ],
+      "candidatePrompts": {
+        "system_prompt": "You parse free-text postal addresses and return a JSON object. The JSON must always contain exactly four keys in the following order: \"city\", \"country\", \"postal_code\", \"street\". If a field is not present in the input, set its value to null. Do not omit any keys. Output only the JSON object, with no extra whitespace, spaces, newlines, or explanatory text. Use compact JSON format (no spaces after colons or commas). Example: for \"Sydney, Australia\" output {\"city\":\"Sydney\",\"country\":\"Australia\",\"postal_code\":null,\"street\":null}."
+      },
+      "trainPassRate": 0.0,
+      "validationPassRate": 1.0,
+      "metricBreakdown": {
+        "final_response_avg_score": 1.0
+      },
+      "accepted": true,
+      "acceptanceReason": "GEPA accepted proposal (val_score=1.0000)",
+      "failedCaseIds": [],
+      "failedCasesTruncated": 0,
+      "perFieldDiagnosis": {},
+      "reflectionLmCalls": 1,
+      "roundLlmCost": 0.0,
+      "roundTokenUsage": {
+        "prompt": 1800,
+        "completion": 899,
+        "total": 2699
+      },
+      "startedAt": "2026-07-30T09:27:16.459534+00:00",
+      "durationSeconds": 26.582981,
+      "kind": "reflective",
+      "trainMinibatchSize": 3,
+      "trainSubsampleParentScore": 0.0,
+      "trainSubsampleCandidateScore": 1.0,
+      "skipReason": null,
+      "errorMessage": null,
+      "budgetUsed": 18,
+      "budgetTotal": 60,
+      "extras": {}
+    }
+  ],
+  "totalReflectionLmCalls": 1,
+  "totalJudgeModelCalls": 0,
+  "totalLlmCost": 0.0,
+  "totalTokenUsage": {
+    "prompt": 1800,
+    "completion": 899,
+    "total": 2699
+  },
+  "durationSeconds": 35.247283,
+  "startedAt": "2026-07-30T09:27:07.797621+00:00",
+  "finishedAt": "2026-07-30T09:27:43.044904+00:00",
+  "extras": {
+    "total_metric_calls": 18
+  }
+}