feat: 5 generic harness fixes — HermesRunner, judge decontamination, B1-B4 defects

skill_eval/agent_runner.py

@@ -21,6 +21,52 @@
 LOG = logging.getLogger("skill_eval.agent_runner")
 
 
+# ---------------------------------------------------------------------------
+# Infrastructure return codes
+# ---------------------------------------------------------------------------
+#
+# These belong to the runner contract, not to any one backend: every runner
+# reports them and `functional.py` reads them to keep a harness failure out of
+# the skill's pass rate.  A real CLI exit status is >= 0, so the negative space
+# is free for sentinels — and POSIX already uses it the same way (subprocess
+# returns -N when a child dies on signal N), which is why "any rc < 0 is
+# infrastructure" is a safe rule downstream.
+#
+# They MUST stay distinct.  Collapsing them onto a shared -1 makes a provider
+# timeout indistinguishable from a missing binary, and both then read as skill
+# failure.
+RC_TIMEOUT = -1
+RC_LAUNCH_FAILED = -2
+# The provider refused or aborted the request while the CLI itself exited
+# cleanly.  This is the only infra class a return code cannot express on its
+# own: `hermes -z` exits 0 and prints the provider's error where the assistant's
+# answer belongs, so the runner has to synthesise the code from out-of-band
+# evidence (its usage file).  Without it, a quota exhaustion is scored as a
+# skill that failed its assertions — MVE AC-6's exact prohibition.
+RC_PROVIDER_ERROR = -3
+
+
+# ---------------------------------------------------------------------------
+# Run provenance
+# ---------------------------------------------------------------------------
+#
+# The key inside `token_counts` under which a runner reports WHO served the
+# call — model, provider, session id, cost status.  It rides in `token_counts`
+# because that dict is already threaded from `run_prompt` through
+# `parse_output` into every execution-metrics block; a parallel channel would
+# have to be plumbed through the same four call sites and could drift out of
+# step with the counts it describes.
+#
+# Underscore-prefixed so it cannot collide with a token bucket, and every
+# consumer reads counts by explicit key (`counts.get("input_tokens", 0)`), so a
+# non-integer value here is never summed as spend.
+#
+# It is a runner-contract constant rather than a Hermes-local one for the same
+# reason the RC_* codes are: `evolve_evidence.py` reads it for whichever runner
+# it was handed, and must not import a backend to do so.
+RUN_META_KEY = "_meta"
+
+
 # ---------------------------------------------------------------------------
 # Abstract interface
 # ---------------------------------------------------------------------------
@@ -80,9 +126,39 @@ def total_tokens(self, token_counts: dict) -> int:
         """Return total token consumption from a token_counts dict.
 
         Default: input_tokens + output_tokens (cache tokens excluded).
+
+        Runners whose backend reports token classes outside this pair (e.g.
+        reasoning tokens) must override this, or that spend becomes invisible
+        to cost-efficiency comparison.
         """
         return token_counts.get("input_tokens", 0) + token_counts.get("output_tokens", 0)
 
+    def supports_trigger_eval(self) -> bool:
+        """Whether this runner exposes structured tool events for trigger eval.
+
+        Default True preserves existing behaviour. Runners emitting only plain
+        text must override to False so the harness reports the gap instead of
+        inferring activation from prose.
+        """
+        return True
+
+    def config_snapshot(self) -> dict:
+        """Return the non-secret configuration that shaped this runner's runs.
+
+        This lands verbatim in an evidence document that gets committed and read
+        by people who were not in the room, so an implementation MUST return
+        only values that are safe to publish: never an API key, a token, or any
+        other credential.  `evolve_evidence` filters credential-shaped keys as
+        well, but a runner that relies on that filter is one rename away from a
+        leak that cannot be undone.
+
+        Defaults to `{}` rather than raising: every runner has to answer the
+        question, or each consumer needs an `hasattr` guard and a runner that
+        simply never implemented the hook becomes indistinguishable from one
+        that genuinely has no configuration.
+        """
+        return {}
+
 
 # ---------------------------------------------------------------------------
 # Errors
@@ -207,10 +283,16 @@ def run_prompt(
         except subprocess.TimeoutExpired:
             elapsed = time.monotonic() - start
             LOG.debug("Claude timed out after %ds", timeout)
-            return "", f"Timed out after {timeout}s", -1, elapsed
+            return "", f"Timed out after {timeout}s", RC_TIMEOUT, elapsed
         except FileNotFoundError:
             elapsed = time.monotonic() - start
             LOG.debug("Claude CLI not found on PATH")
+            # Known wart: this should be RC_LAUNCH_FAILED, but the -1 is part of
+            # ClaudeRunner's published contract (tests assert it), so changing
+            # it belongs in its own change.  Diagnosis is the only casualty —
+            # functional.py treats every rc < 0 as infrastructure, so a missing
+            # binary is still excluded from the pass rate, just labelled less
+            # precisely than a HermesRunner launch failure would be.
             return "", "claude CLI not found", -1, elapsed
 
     def parse_output(self, raw: str) -> dict:
@@ -345,3 +427,18 @@ def get_runner(name: str = "claude") -> AgentRunner:
 
 # Register the built-in ClaudeRunner
 register_runner("claude", ClaudeRunner)
+
+
+def _register_optional_runners() -> None:
+    """Self-register bundled optional runners.
+
+    Guarded so a failure here can never break the zero-dependency audit path,
+    which must keep working with no agent CLI installed at all.
+    """
+    try:
+        import skill_eval.hermes_runner  # noqa: F401  (import registers)
+    except Exception as exc:  # pragma: no cover - defensive
+        LOG.debug("Hermes runner unavailable: %s", exc)
+
+
+_register_optional_runners()

skill_eval/cli.py

@@ -154,6 +154,64 @@ def main(argv: list[str] | None = None) -> int:
     snapshot_parser.add_argument("--version", type=str, default=None,
                                  help="Version label (default: auto from metadata)")
 
+    # evolve-check command — deterministic two-split promotion gate
+    evolve_parser = subparsers.add_parser(
+        "evolve-check",
+        help="Gate a candidate on held-in/held-out evidence (never promotes)",
+    )
+    evolve_parser.add_argument("evidence_path", help="Path to the evidence JSON file")
+    evolve_parser.add_argument("--lineage", type=str, default=None,
+                               help="Append the decision to this append-only JSONL ledger")
+    evolve_parser.add_argument("--tolerance", type=float, default=None,
+                               help="Permitted regression per split (default: 0.0)")
+    evolve_parser.add_argument("--min-improvement", type=float, default=None,
+                               help="Improvement one split must clear (default: 0.01)")
+    evolve_parser.add_argument("--allow-small-n", action="store_true",
+                               help="Permit fewer than 3 runs per split (discouraged)")
+
+    # evolve-evidence command — produces what evolve-check consumes
+    evidence_parser = subparsers.add_parser(
+        "evolve-evidence",
+        help="Run a frozen suite through both arms and emit gate-ready evidence",
+    )
+    evidence_parser.add_argument("--skill", required=True,
+                                 help="Path to the skill directory under test")
+    evidence_parser.add_argument("--suite", required=True,
+                                 help="Path to the frozen suite JSON")
+    evidence_parser.add_argument("--baseline", type=str, default=None,
+                                 help="Path to the incumbent skill the candidate "
+                                      "must beat. Without it the baseline is the "
+                                      "absence of any skill, and the gate answers "
+                                      "'is this better than nothing?' — a question "
+                                      "every non-empty first version passes and "
+                                      "which says nothing about whether v7 beats "
+                                      "v6. Roughly doubles the provider spend.")
+    evidence_parser.add_argument("--out", required=True,
+                                 help="Path to write the evidence JSON")
+    evidence_parser.add_argument("--agent", type=str, default="claude",
+                                 help="Registered agent runner to execute tasks (default: claude)")
+    evidence_parser.add_argument("--judge-agent", type=str, default=None,
+                                 help="Runner that grades assertions the deterministic "
+                                      "matchers defer (default: none, and a suite that "
+                                      "needs one is refused)")
+    evidence_parser.add_argument("--force-self-judge", action="store_true",
+                                 help="Permit --judge-agent to name the same runner "
+                                      "as --agent. Refused by default: the model "
+                                      "would grade its own output and the measured "
+                                      "delta would blend skill transfer with judge "
+                                      "leniency. The override records the confound "
+                                      "in the evidence rather than hiding it.")
+    evidence_parser.add_argument("--runs", type=int, default=3,
+                                 help="Runs per split (default: 3)")
+    evidence_parser.add_argument("--candidate-id", type=str, default=None,
+                                 help="Candidate id (default: <skill-name>@<artifact-hash>)")
+    evidence_parser.add_argument("--lineage", type=str, default=None,
+                                 help="Append the decision to this append-only JSONL ledger")
+    evidence_parser.add_argument("--allow-small-n", action="store_true",
+                                 help="Permit fewer than 3 runs per split (discouraged)")
+    evidence_parser.add_argument("--timeout", type=int, default=120,
+                                 help="Timeout per agent invocation in seconds (default: 120)")
+
     # regression command (Phase 2)
     regression_parser = subparsers.add_parser("regression",
                                                help="Check for regressions against baseline")
@@ -179,6 +237,12 @@ def main(argv: list[str] | None = None) -> int:
                                     help="Timeout per claude invocation in seconds (default: 120)")
     functional_parser.add_argument("--agent", type=str, default="claude",
                                     help="Agent runner to use (default: claude)")
+    functional_parser.add_argument("--judge-agent", type=str, default=None,
+                                    help="Runner that grades non-deterministic "
+                                         "assertions (default: none — they are "
+                                         "reported unevaluated). Never defaults "
+                                         "to --agent: a model grading its own "
+                                         "output inflates its own score.")
 
     # trigger command (Phase 3)
     trigger_parser = subparsers.add_parser("trigger",
@@ -346,6 +410,41 @@ def main(argv: list[str] | None = None) -> int:
         from skill_eval.regression import save_snapshot
         return save_snapshot(args.skill_path, version=args.version)
 
+    elif args.command == "evolve-check":
+        from skill_eval.evolution_gate import (
+            DEFAULT_MIN_IMPROVEMENT,
+            DEFAULT_TOLERANCE,
+            run_evolve_check,
+        )
+        return run_evolve_check(
+            args.evidence_path,
+            lineage_path=args.lineage,
+            tolerance=DEFAULT_TOLERANCE if args.tolerance is None else args.tolerance,
+            min_improvement=(
+                DEFAULT_MIN_IMPROVEMENT
+                if args.min_improvement is None
+                else args.min_improvement
+            ),
+            allow_small_n=args.allow_small_n,
+        )
+
+    elif args.command == "evolve-evidence":
+        from skill_eval.evolve_evidence import run_evolve_evidence
+        return run_evolve_evidence(
+            skill_path=args.skill,
+            suite_path=args.suite,
+            out_path=args.out,
+            baseline_path=args.baseline,
+            agent=args.agent,
+            runs=args.runs,
+            judge_agent=args.judge_agent,
+            candidate_id=args.candidate_id,
+            lineage_path=args.lineage,
+            allow_small_n=args.allow_small_n,
+            timeout=args.timeout,
+            force_self_judge=args.force_self_judge,
+        )
+
     elif args.command == "regression":
         from skill_eval.regression import check_regression
         return check_regression(args.skill_path, baseline_path=args.baseline,
@@ -362,6 +461,7 @@ def main(argv: list[str] | None = None) -> int:
             dry_run=args.dry_run,
             timeout=args.timeout,
             agent=args.agent,
+            judge_agent=args.judge_agent,
         )
 
     elif args.command == "trigger":

skill_eval/compare.py

@@ -183,7 +183,10 @@ def _run_single_skill(
     parsed = runner.parse_output(stdout)
 
     text = parsed["text"]
-    assertion_results, pass_rate = grade_output(text, eval_case.assertions, timeout=timeout)
+    # Judge with the executing runner -- see functional.py for why (blocker B1).
+    assertion_results, pass_rate = grade_output(
+        text, eval_case.assertions, timeout=timeout, judge_runner=runner,
+    )
 
     return {
         "pass_rate": pass_rate,

skill_eval/eval_schemas.py

@@ -68,12 +68,30 @@ def from_dict(cls, data: dict) -> "GradingResult":
 
 @dataclass
 class RunPairResult:
-    """Paired with-skill / without-skill results for one eval case run."""
+    """Paired with-skill / without-skill results for one eval case run.
+
+    `infrastructure_error` is set when an arm failed to execute at all — a
+    provider timeout, a CLI that would not launch, a child killed by signal.
+    It maps the affected arm ("with_skill" / "without_skill") to a
+    {kind, rc, detail} record.  Such a pair is reported but kept out of the
+    aggregate means: a harness failure produced no evidence about the skill,
+    and scoring it as 0% would understate the skill by exactly the harness's
+    own flakiness.
+
+    `raw_streams` carries the untouched subprocess output of both arms —
+    `{arm: {"stdout": str, "stderr": str}}` — and is populated only when the
+    caller asks for it, because the benchmark path writes every pair into
+    `benchmark.json` and a full stream-json transcript per arm would dwarf the
+    report it lives in.  The evidence path asks for it: there, the streams are
+    the thing being retained, and they leave for a JSONL ledger of their own.
+    """
     eval_id: str
     run_index: int
     with_skill: Optional[dict] = None
     without_skill: Optional[dict] = None
     delta_pass_rate: float = 0.0
+    infrastructure_error: Optional[dict] = None
+    raw_streams: Optional[dict] = None
 
     def to_dict(self) -> dict:
         return asdict(self)