From c34542ed14902109dd66555a347694c075a64919 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Wed, 16 Sep 2026 09:35:37 +0200 Subject: [PATCH] Fail a routing run that lost a skill before it started A staged skill the CLI never reports at session init is not in the room the run claims to measure. Its cases still run, every one reads as a missed trigger, and the report tells the author a description does not trigger when that description was never offered to the agent. The two outcomes are indistinguishable in the verdict. The condition was already detected and already rendered as a warning above the numbers. It just did not reach the gate, so the run went green apart from a line that is easy to read past. This is the same class as the two checks beside it: infrastructure rather than score. It is placed before the accuracy bar so that neither a clean sweep nor `--min-accuracy 0` can license a room that was not the one asked for. Worth stating why it is not hypothetical. Descriptions share one listing budget, and on overflow the text is dropped starting with the skills invoked least. A skill staged for an eval has been invoked zero times, so it is first in that queue, while the runner's own user-level skills compete for the same space. Signed-off-by: John Lybeck --- skillscope/cli.py | 27 ++++++++++++++++++++++----- tests/test_skillscope.py | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/skillscope/cli.py b/skillscope/cli.py index f9d93b4..03b3ebb 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -148,12 +148,16 @@ def _structural_or_exit(skills: list[str] | None = None) -> list[references.Refe return found -def routing_gate(totals: dict, min_accuracy: float) -> str | None: +def routing_gate( + totals: dict, min_accuracy: float, missing_skills: list[str] | None = None +) -> str | None: """Why this routing run should fail, or ``None`` if it should not. - The first two answers are infrastructure, not score, and hold whatever the - bar is: a run where nothing was graded, or where no skill ever activated, - has not measured routing at all. + The first three answers are infrastructure, not score, and hold whatever + the bar is: a run where nothing was graded, where no skill ever activated, + or where a staged skill never reached the agent has not measured routing at + all. The last of those is the quietest, because the cases still run and + still produce a number. """ if totals["graded"] == 0: return "every case errored; treating the run as a failure." @@ -163,6 +167,15 @@ def routing_gate(totals: dict, min_accuracy: float) -> str | None: "or activation detection is broken. Failing rather than reporting " "a 0% routing rate as if it were real." ) + if missing_skills: + return ( + "the CLI did not report these installed skills at session init: " + f"{', '.join(sorted(missing_skills))}. A skill the agent was never " + "offered cannot be routed to, so every case for it reads as a " + "missed trigger and the author is told a description that was never " + "listed does not trigger. Failing rather than scoring a room that " + "was not the one asked for." + ) if min_accuracy <= 0: return None # Compared against the exact ratio rather than the reported accuracy, @@ -415,7 +428,11 @@ def cmd_routing(args: argparse.Namespace) -> int: if (code := _fail_if_expired()) is not None: return code - reason = routing_gate(summary["totals"], args.min_accuracy) + reason = routing_gate( + summary["totals"], + args.min_accuracy, + summary["skills_missing_from_session"], + ) if reason: print(f"[routing] {reason}", file=sys.stderr) return 1 diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 11cdee6..0b1d7df 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -2032,6 +2032,28 @@ def totals(self, passed: int, graded: int, **extra) -> dict: def gate(self, passed: int, graded: int, bar: float = 1.0, **extra) -> str | None: return cli.routing_gate(self.totals(passed, graded, **extra), bar) + def test_a_skill_the_agent_never_saw_fails_the_run(self) -> None: + """The quiet one. A staged skill that never reached the agent still has + cases, and every one of them reads as a missed trigger, so the run + produces a plausible number for a room that was not the one asked for.""" + reason = cli.routing_gate(self.totals(11, 12), 1.0, ["absent-skill"]) + self.assertIn("absent-skill", reason) + self.assertIn("session init", reason) + + def test_a_missing_skill_outranks_the_accuracy_bar(self) -> None: + """Infrastructure before score: a perfect sweep of a wrong room is not a + pass, and `--min-accuracy 0` does not license one either.""" + for bar in (1.0, 0.0): + with self.subTest(bar=bar): + reason = cli.routing_gate(self.totals(12, 12), bar, ["absent-skill"]) + self.assertIsNotNone(reason) + self.assertNotIn("--min-accuracy", reason) + + def test_nothing_missing_leaves_the_gate_alone(self) -> None: + for missing in (None, []): + with self.subTest(missing=missing): + self.assertIsNone(cli.routing_gate(self.totals(12, 12), 1.0, missing)) + def test_the_default_bar_is_every_graded_case(self) -> None: self.assertEqual(cli.build_parser().parse_args(["routing"]).min_accuracy, 1.0)