diff --git a/skillscope/cli.py b/skillscope/cli.py index f9d93b4..03b3ebb 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -148,12 +148,16 @@ def _structural_or_exit(skills: list[str] | None = None) -> list[references.Refe return found -def routing_gate(totals: dict, min_accuracy: float) -> str | None: +def routing_gate( + totals: dict, min_accuracy: float, missing_skills: list[str] | None = None +) -> str | None: """Why this routing run should fail, or ``None`` if it should not. - The first two answers are infrastructure, not score, and hold whatever the - bar is: a run where nothing was graded, or where no skill ever activated, - has not measured routing at all. + The first three answers are infrastructure, not score, and hold whatever + the bar is: a run where nothing was graded, where no skill ever activated, + or where a staged skill never reached the agent has not measured routing at + all. The last of those is the quietest, because the cases still run and + still produce a number. """ if totals["graded"] == 0: return "every case errored; treating the run as a failure." @@ -163,6 +167,15 @@ def routing_gate(totals: dict, min_accuracy: float) -> str | None: "or activation detection is broken. Failing rather than reporting " "a 0% routing rate as if it were real." ) + if missing_skills: + return ( + "the CLI did not report these installed skills at session init: " + f"{', '.join(sorted(missing_skills))}. A skill the agent was never " + "offered cannot be routed to, so every case for it reads as a " + "missed trigger and the author is told a description that was never " + "listed does not trigger. Failing rather than scoring a room that " + "was not the one asked for." + ) if min_accuracy <= 0: return None # Compared against the exact ratio rather than the reported accuracy, @@ -415,7 +428,11 @@ def cmd_routing(args: argparse.Namespace) -> int: if (code := _fail_if_expired()) is not None: return code - reason = routing_gate(summary["totals"], args.min_accuracy) + reason = routing_gate( + summary["totals"], + args.min_accuracy, + summary["skills_missing_from_session"], + ) if reason: print(f"[routing] {reason}", file=sys.stderr) return 1 diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 11cdee6..0b1d7df 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -2032,6 +2032,28 @@ def totals(self, passed: int, graded: int, **extra) -> dict: def gate(self, passed: int, graded: int, bar: float = 1.0, **extra) -> str | None: return cli.routing_gate(self.totals(passed, graded, **extra), bar) + def test_a_skill_the_agent_never_saw_fails_the_run(self) -> None: + """The quiet one. A staged skill that never reached the agent still has + cases, and every one of them reads as a missed trigger, so the run + produces a plausible number for a room that was not the one asked for.""" + reason = cli.routing_gate(self.totals(11, 12), 1.0, ["absent-skill"]) + self.assertIn("absent-skill", reason) + self.assertIn("session init", reason) + + def test_a_missing_skill_outranks_the_accuracy_bar(self) -> None: + """Infrastructure before score: a perfect sweep of a wrong room is not a + pass, and `--min-accuracy 0` does not license one either.""" + for bar in (1.0, 0.0): + with self.subTest(bar=bar): + reason = cli.routing_gate(self.totals(12, 12), bar, ["absent-skill"]) + self.assertIsNotNone(reason) + self.assertNotIn("--min-accuracy", reason) + + def test_nothing_missing_leaves_the_gate_alone(self) -> None: + for missing in (None, []): + with self.subTest(missing=missing): + self.assertIsNone(cli.routing_gate(self.totals(12, 12), 1.0, missing)) + def test_the_default_bar_is_every_graded_case(self) -> None: self.assertEqual(cli.build_parser().parse_args(["routing"]).min_accuracy, 1.0)