diff --git a/benchmarks/guardian/experiments/246-cross-vendor/README.md b/benchmarks/guardian/experiments/246-cross-vendor/README.md new file mode 100644 index 00000000..54640525 --- /dev/null +++ b/benchmarks/guardian/experiments/246-cross-vendor/README.md @@ -0,0 +1,71 @@ +# Cross-vendor skeptic: still blocked, and now on measured limits (#246) + +Two attempts at the third arm, both stopped by the validity gate before any +number was reported. The gate is the deliverable here; the comparison is not. + +## Why a gate at all + +`judge_finding` contains provider errors per finding and returns None, so a +spent quota, an HTTP 429, a 402 and unparseable JSON all arrive as **the same +row shape as a skeptic that refused to refute anything** — which is exactly what +#246 predicts for a same-vendor skeptic. A broken arm would confirm the +hypothesis. `MAX_UNRULED_RATE` refuses to print a comparison above 5% unruled, +keeps the rows, and exits 2. + +It fired on both attempts. Neither produced a number that could be read. + +## Attempt 1 — free tier. The quota is 50 requests per day. + +`nvidia/nemotron-3-super-120b-a12b:free`, the whole corpus: **96 of 135 findings +unruled**, and the pattern is a cutoff in time rather than a property of the +input. + +| PR | evidence | unruled | +|---|---|---:| +| 122 (first) | no | **0 / 29** | +| 140 | yes | 29 / 39 | +| 141, 142, 143, 144 | — | **100%** | + +39 answered, then silence. With the day's earlier probes (4 candidate calls, 6 +in a smoke run) that is 49 successful requests before the wall — the documented +free allowance is 50/day, and this is that number arrived at from the data. + +Rows: `free-nemotron-quota-cutoff.jsonl`. + +## Attempt 2 — paid. Two separate faults, one after the other. + +`qwen/qwen3.7-plus`, chosen because capability has to match: comparing +`gemini-2.5-flash` against a small free model measures strong-versus-weak and +calls it same-versus-cross. + +**First fault — thinking.** 118 of 135 unruled. `qwen3.7-plus` puts its chain of +thought in a separate `reasoning` field and fills `content` only at the end, so +on the larger prompts it spent the entire 8,000-token budget thinking and +returned `content: null`. Raising the budget would have fixed the symptom and +broken the experiment: the arm it is compared against is not doing extended +thinking, so the two arms would differ in a dimension nobody chose. The provider +now sends `reasoning: {"enabled": …}` explicitly in both directions, off by +default, so a run can state what it did. + +**Second fault — credits.** The re-run failed on HTTP **402**: the account has +`total_credits: $0` against `total_usage: $0.159`. The trial allowance is spent; +paid models are unavailable until it is topped up. + +The second fault took half an hour to identify because the warning read only +"Skeptic judgement failed; finding stays unruled" — 118 identical lines, no +type, no message. It now carries both. A quota, a truncated reasoning model and +a 402 have three different remedies and all three land in that one `except`. + +## To unblock + +1. **Top up OpenRouter.** The measured cost of the run with reasoning off is + ~$0.15: 135 findings × ~2,700 prompt tokens at \$0.32/M, plus short + completions at \$1.28/M. A dollar covers it several times over. +2. **Free tier across three days.** The input is a frozen recording, so the run + is deterministic and can be split — 50 requests a day, 135 needed. Fragile, + and it needs resume support the tool does not have. +3. A local ollama, which this issue already names as the no-cost option. + +Everything else is in place: the corpus with both finder orientations, the +recordings, per-arm models, the scoring, and the gate that stopped two wrong +answers from being published. diff --git a/benchmarks/guardian/experiments/246-cross-vendor/free-nemotron-quota-cutoff.jsonl b/benchmarks/guardian/experiments/246-cross-vendor/free-nemotron-quota-cutoff.jsonl new file mode 100644 index 00000000..1d69cabb --- /dev/null +++ b/benchmarks/guardian/experiments/246-cross-vendor/free-nemotron-quota-cutoff.jsonl @@ -0,0 +1,74 @@ +{"timestamp": "2026-08-17T13:17:43.517549+00:00", "pr": 122, "pass": "122@2026-06-10T20-22-31.032903+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 0, "confirmed": 3, "unruled": 0, "recall": 0.18181818181818182, "precision": 0.6666666666666666, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:18:05.186110+00:00", "pr": 122, "pass": "122@2026-06-10T20-22-31.032903+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.18181818181818182, "precision": 0.6666666666666666, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:18:15.602819+00:00", "pr": 122, "pass": "122@2026-06-10T20-22-59.496508+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:18:24.136220+00:00", "pr": 122, "pass": "122@2026-06-10T20-22-59.496508+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:18:36.000053+00:00", "pr": 122, "pass": "122@2026-06-10T20-23-51.775172+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.18181818181818182, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:18:50.719370+00:00", "pr": 122, "pass": "122@2026-06-10T20-23-51.775172+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 1, "confirmed": 1, "unruled": 0, "recall": 0.18181818181818182, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:19:01.976110+00:00", "pr": 122, "pass": "122@2026-06-10T20-32-35.483908+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 3, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:19:23.184478+00:00", "pr": 122, "pass": "122@2026-06-10T20-32-35.483908+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 2, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 3, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:19:36.855990+00:00", "pr": 122, "pass": "122@2026-06-10T20-33-10.513615+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 0, "confirmed": 3, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:20:22.191259+00:00", "pr": 122, "pass": "122@2026-06-10T20-33-10.513615+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:20:35.820094+00:00", "pr": 122, "pass": "122@2026-06-10T21-58-22.314534+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:21:08.068793+00:00", "pr": 122, "pass": "122@2026-06-10T21-58-22.314534+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:21:15.458172+00:00", "pr": 122, "pass": "122@2026-06-11T00-11-53.683357+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:21:21.049843+00:00", "pr": 122, "pass": "122@2026-06-11T00-11-53.683357+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:21:29.468118+00:00", "pr": 122, "pass": "122@2026-06-11T00-12-40.927704+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:21:35.271913+00:00", "pr": 122, "pass": "122@2026-06-11T00-12-40.927704+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:21:46.009233+00:00", "pr": 122, "pass": "122@2026-06-11T00-14-36.809184+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.18181818181818182, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:21:58.953594+00:00", "pr": 122, "pass": "122@2026-06-11T00-14-36.809184+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.18181818181818182, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:22:06.863138+00:00", "pr": 122, "pass": "122@2026-06-11T00-32-51.894263+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:22:18.621710+00:00", "pr": 122, "pass": "122@2026-06-11T00-32-51.894263+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:22:34.978570+00:00", "pr": 122, "pass": "122@2026-06-11T00-36-53.749615+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 1, "uncertain": 0, "confirmed": 0, "unruled": 0, "recall": 0.0, "precision": 1.0, "noise": 0, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:23:29.360852+00:00", "pr": 122, "pass": "122@2026-06-11T00-36-53.749615+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:23:35.269160+00:00", "pr": 122, "pass": "122@2026-06-11T00-42-55.162925+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:23:43.563281+00:00", "pr": 122, "pass": "122@2026-06-11T00-42-55.162925+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:23:53.848653+00:00", "pr": 122, "pass": "122@2026-06-11T00-48-45.262981+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 2, "refuted": 1, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:24:05.969279+00:00", "pr": 122, "pass": "122@2026-06-11T00-48-45.262981+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.18181818181818182, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:24:25.419473+00:00", "pr": 140, "pass": "140@2026-06-10T20-24-40.504167+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 3, "refuted": 0, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 3, "killed_gt": []} +{"timestamp": "2026-08-17T13:25:25.250544+00:00", "pr": 140, "pass": "140@2026-06-10T20-24-40.504167+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 3, "refuted": 0, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 3, "killed_gt": []} +{"timestamp": "2026-08-17T13:25:47.211803+00:00", "pr": 140, "pass": "140@2026-06-10T20-25-19.261202+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 5, "refuted": 2, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 3, "killed_gt": []} +{"timestamp": "2026-08-17T13:26:24.777165+00:00", "pr": 140, "pass": "140@2026-06-10T20-25-19.261202+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 5, "refuted": 0, "uncertain": 3, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 5, "killed_gt": []} +{"timestamp": "2026-08-17T13:26:33.938777+00:00", "pr": 140, "pass": "140@2026-06-10T20-26-01.543236+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:08.103278+00:00", "pr": 140, "pass": "140@2026-06-10T20-26-01.543236+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:17.614733+00:00", "pr": 140, "pass": "140@2026-06-11T00-16-26.605409+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:17.769243+00:00", "pr": 140, "pass": "140@2026-06-11T00-16-26.605409+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:27.876729+00:00", "pr": 140, "pass": "140@2026-06-11T00-17-25.717492+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 3, "refuted": 0, "uncertain": 0, "confirmed": 3, "unruled": 0, "recall": 0.06666666666666667, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:28.068613+00:00", "pr": 140, "pass": "140@2026-06-11T00-17-25.717492+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 3, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 3, "recall": 0.06666666666666667, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:38.870523+00:00", "pr": 140, "pass": "140@2026-06-11T00-33-44.053747+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.06666666666666667, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:39.001629+00:00", "pr": 140, "pass": "140@2026-06-11T00-33-44.053747+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.06666666666666667, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:45.424281+00:00", "pr": 140, "pass": "140@2026-06-11T00-37-50.526658+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:45.544269+00:00", "pr": 140, "pass": "140@2026-06-11T00-37-50.526658+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:53.801567+00:00", "pr": 140, "pass": "140@2026-06-11T00-50-08.134629+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 1, "confirmed": 0, "unruled": 0, "recall": 0.06666666666666667, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:53.921093+00:00", "pr": 140, "pass": "140@2026-06-11T00-50-08.134629+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.06666666666666667, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:06.369511+00:00", "pr": 140, "pass": "140@2026-07-29T17-29-19.861365+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 22, "refuted": 10, "uncertain": 2, "confirmed": 10, "unruled": 0, "recall": 0.26666666666666666, "precision": 0.3333333333333333, "noise": 8, "killed_gt": ["count-routers-on-squared", "weights-keyerror-custom-yaml"]} +{"timestamp": "2026-08-17T13:29:07.917533+00:00", "pr": 140, "pass": "140@2026-07-29T17-29-19.861365+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 22, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 22, "recall": 0.4, "precision": 0.2727272727272727, "noise": 16, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:21.415626+00:00", "pr": 141, "pass": "141@2026-06-10T20-26-53.683221+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 1.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:21.527539+00:00", "pr": 141, "pass": "141@2026-06-10T20-26-53.683221+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 1.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:28.243544+00:00", "pr": 141, "pass": "141@2026-06-10T20-29-48.066882+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 1, "confirmed": 0, "unruled": 0, "recall": 1.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:28.360785+00:00", "pr": 141, "pass": "141@2026-06-10T20-29-48.066882+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 1.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:37.370316+00:00", "pr": 141, "pass": "141@2026-06-10T21-59-24.111065+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 1.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:37.495452+00:00", "pr": 141, "pass": "141@2026-06-10T21-59-24.111065+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 1.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:49.082379+00:00", "pr": 142, "pass": "142@2026-06-10T20-30-15.605690+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.5, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:49.243691+00:00", "pr": 142, "pass": "142@2026-06-10T20-30-15.605690+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.5, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:56.956875+00:00", "pr": 142, "pass": "142@2026-06-10T21-52-43.992582+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:57.130875+00:00", "pr": 142, "pass": "142@2026-06-10T21-52-43.992582+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 2, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:04.860920+00:00", "pr": 142, "pass": "142@2026-06-10T21-53-15.240002+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:05.012683+00:00", "pr": 142, "pass": "142@2026-06-10T21-53-15.240002+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:14.805023+00:00", "pr": 142, "pass": "142@2026-06-11T00-45-11.791674+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 3, "confirmed": 0, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:14.969320+00:00", "pr": 142, "pass": "142@2026-06-11T00-45-11.791674+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 3, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:23.741156+00:00", "pr": 142, "pass": "142@2026-06-11T00-50-52.772949+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 3, "refuted": 2, "uncertain": 1, "confirmed": 0, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:23.921669+00:00", "pr": 142, "pass": "142@2026-06-11T00-50-52.772949+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 3, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:43.638312+00:00", "pr": 143, "pass": "143@2026-06-11T00-46-29.982484+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:43.909808+00:00", "pr": 143, "pass": "143@2026-06-11T00-46-29.982484+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:21.206192+00:00", "pr": 143, "pass": "143@2026-07-29T17-15-00.138863+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 28, "refuted": 8, "uncertain": 1, "confirmed": 19, "unruled": 0, "recall": 1.0, "precision": 0.3, "noise": 14, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:22.809114+00:00", "pr": 143, "pass": "143@2026-07-29T17-15-00.138863+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 28, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 28, "recall": 1.0, "precision": 0.21428571428571427, "noise": 22, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:44.357216+00:00", "pr": 144, "pass": "144@2026-06-10T20-32-21.181637+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:44.503817+00:00", "pr": 144, "pass": "144@2026-06-10T20-32-21.181637+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 2, "recall": 0.0, "precision": 0.0, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:58.741622+00:00", "pr": 144, "pass": "144@2026-06-11T00-25-01.536057+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 1, "uncertain": 0, "confirmed": 0, "unruled": 0, "recall": 0.0, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:58.913488+00:00", "pr": 144, "pass": "144@2026-06-11T00-25-01.536057+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:33:06.869381+00:00", "pr": 144, "pass": "144@2026-06-11T00-47-31.514757+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 1, "uncertain": 0, "confirmed": 0, "unruled": 0, "recall": 0.0, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:33:06.990972+00:00", "pr": 144, "pass": "144@2026-06-11T00-47-31.514757+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:33:14.521992+00:00", "pr": 144, "pass": "144@2026-06-11T00-52-31.320750+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:33:14.623054+00:00", "pr": 144, "pass": "144@2026-06-11T00-52-31.320750+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:34:10.907748+00:00", "pr": 144, "pass": "144@2026-07-29T17-28-33.271744+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 20, "refuted": 7, "uncertain": 0, "confirmed": 13, "unruled": 0, "recall": 0.6, "precision": 0.23076923076923078, "noise": 5, "killed_gt": []} +{"timestamp": "2026-08-17T13:34:12.013311+00:00", "pr": 144, "pass": "144@2026-07-29T17-28-33.271744+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 20, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 20, "recall": 0.6, "precision": 0.15, "noise": 8, "killed_gt": []} diff --git a/scripts/skeptic_arms.py b/scripts/skeptic_arms.py index cec3f1b0..4837ab15 100644 --- a/scripts/skeptic_arms.py +++ b/scripts/skeptic_arms.py @@ -66,10 +66,17 @@ REPO_ROOT = Path(__file__).resolve().parent.parent BENCH_DIR = REPO_ROOT / "benchmarks" / "guardian" -#: The two arms. Named rather than derived, because "the opposite of the primary" -#: is exactly the rule under test — deriving the arm from the finder would build -#: the hypothesis into the instrument. -ARMS = ("gemini", "mistral") +#: The default pair. Named rather than derived, because "the opposite of the +#: primary" is exactly the rule under test — deriving an arm from the finder +#: would build the hypothesis into the instrument. +#: +#: `--arms` overrides it, and each arm may name its own model +#: (`openrouter=nvidia/nemotron-3-super-120b-a12b:free`). Per arm rather than +#: from the environment: `GUARDIAN_SKEPTIC_MODEL` applies to whichever provider +#: is built, so a single value would hand one arm the other's model — one arm +#: running something that does not exist, reported as a difference between +#: vendors. +DEFAULT_ARMS = "gemini,mistral" class NoArmError(RuntimeError): @@ -80,17 +87,36 @@ class NoArmError(RuntimeError): """ -def arm_provider(name: str, env: Mapping[str, str]) -> tuple[BaseProvider, str]: +def parse_arms(spec: str) -> dict[str, str | None]: + """`gemini,openrouter=vendor/model:free` → {arm: model or None}. + + A model per arm, never one shared. The environment's + `GUARDIAN_SKEPTIC_MODEL` is dropped in `arm_provider` for the same reason. + """ + arms: dict[str, str | None] = {} + for item in spec.split(","): + name, _, model = item.strip().partition("=") + if name: + arms[name] = model or None + return arms + + +def arm_provider( + name: str, env: Mapping[str, str], model: str | None = None +) -> tuple[BaseProvider, str]: """The skeptic for one arm, or a refusal naming the missing key. - `GUARDIAN_SKEPTIC_MODEL` is dropped rather than passed through. - `build_skeptic_provider` applies it to whichever provider it builds, so an - environment holding the production value (`gemini-2.5-flash`) would hand - that model name to the mistral arm — one arm running a model that does not - exist, while the other ran the intended one. Each arm takes its provider's - own default, and the models used are recorded on every row. + The environment's `GUARDIAN_SKEPTIC_MODEL` is dropped and `model` used + instead. `build_skeptic_provider` applies that variable to whichever + provider it builds, so an environment holding the production value + (`gemini-2.5-flash`) would hand that name to a mistral or openrouter arm — + one arm running a model that does not exist while the other ran the intended + one, and the difference reported as a comparison between vendors. The models + actually used are recorded on every row. """ per_arm = {k: v for k, v in env.items() if k != "GUARDIAN_SKEPTIC_MODEL"} + if model: + per_arm["GUARDIAN_SKEPTIC_MODEL"] = model built = build_skeptic_provider({**per_arm, "GUARDIAN_SKEPTIC": name}, primary="none") if built is None: _msg = ( @@ -162,7 +188,9 @@ async def judge_one( } -async def collect_rows(repo_root: Path, limit: int | None) -> list[dict[str, Any]]: +async def collect_rows( + repo_root: Path, limit: int | None, arm_spec: dict[str, str | None] +) -> list[dict[str, Any]]: """Both arms over every frozen pass; one row per (pass, arm). Returns the rows rather than writing them. The write is the caller's, and @@ -170,7 +198,7 @@ async def collect_rows(repo_root: Path, limit: int | None) -> list[dict[str, Any the worktree collection above avoids with `to_thread`, and here there is nothing to gain by being in the loop at all. """ - arms = {name: arm_provider(name, os.environ) for name in ARMS} + arms = {name: arm_provider(name, os.environ, model) for name, model in arm_spec.items()} models = finder_models(BENCH_DIR / "results.jsonl") with tempfile.TemporaryDirectory(prefix="arms-") as tmp: @@ -241,6 +269,43 @@ async def collect_rows(repo_root: Path, limit: int | None) -> list[dict[str, Any UNKNOWN_VENDOR = "unknown" +#: The share of findings an arm may leave unruled before its numbers are refused. +#: +#: Not a tolerance — a validity gate, and it is the only thing standing between +#: this experiment and a wrong conclusion. `judge_finding` contains provider +#: errors per finding and returns None, so a spent quota, an upstream 429 and a +#: model that cannot produce JSON all arrive as "unruled". An arm that answered +#: nothing and an arm that refuted nothing are the same row shape, and "refutes +#: nothing" is precisely what #246 predicts for a same-vendor skeptic. Reported +#: without this check, a broken arm would confirm the hypothesis. +MAX_UNRULED_RATE = 0.05 + + +class ArmTooQuietError(RuntimeError): + """Raised when an arm left too many findings unruled to be scored. + + Refused rather than footnoted. A caveat under a table does not stop the + table being read, and this table's whole content is how much each skeptic + refuted. + """ + + +def validity_problems(rows: list[dict[str, Any]]) -> list[str]: + """Arms whose unruled share is too high to interpret, with the numbers.""" + totals: dict[str, list[int]] = defaultdict(lambda: [0, 0]) + for row in rows: + seen = totals[str(row["arm"])] + seen[0] += int(row.get("unruled") or 0) + seen[1] += int(row["findings"]) + return [ + f"arm {arm!r} left {unruled} of {findings} findings unruled " + f"({unruled / findings:.0%} > {MAX_UNRULED_RATE:.0%}); its numbers cannot be read as " + f"leniency because they may be silence." + for arm, (unruled, findings) in sorted(totals.items()) + if findings and unruled / findings > MAX_UNRULED_RATE + ] + + def _vendor(model: str) -> str: """The vendor behind a model name, or `UNKNOWN_VENDOR`. @@ -276,7 +341,7 @@ def report(rows: list[dict[str, Any]]) -> None: """The comparison the issue asks for, split by which vendor found the findings.""" print(f"\n{len(rows)} (pass, arm) results\n") print( - f"{'finder':9} {'skeptic':9} {'kind':7} {'passes':>6} {'refuted':>8} " + f"{'finder':10} {'skeptic':11} {'kind':8} {'passes':>6} {'refuted':>8} " f"{'of':>5} {'killed GT':>10}" ) groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) @@ -287,7 +352,7 @@ def report(rows: list[dict[str, Any]]) -> None: refuted = sum(int(r["refuted"]) for r in group) total = sum(int(r["findings"]) for r in group) killed = sum(len(r["killed_gt"]) for r in group) - print(f"{finder:9} {arm:9} {kind:7} {len(group):6} {refuted:8} {total:5} {killed:10}") + print(f"{finder:10} {arm:11} {kind:8} {len(group):6} {refuted:8} {total:5} {killed:10}") unattributed = sorted( {str(r["finder_model"]) for r in rows if _vendor(str(r["finder_model"])) == UNKNOWN_VENDOR} ) @@ -307,13 +372,26 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", type=Path, default=REPO_ROOT / ".guardian-arms.jsonl") parser.add_argument("--repo-root", type=Path, default=REPO_ROOT) + parser.add_argument( + "--arms", + default=DEFAULT_ARMS, + help="comma-separated arms, each optionally name=model (see DEFAULT_ARMS)", + ) parser.add_argument( "--limit", type=int, default=None, help="judge only the first N passes (a smoke run)" ) args = parser.parse_args() - rows = asyncio.run(collect_rows(args.repo_root, args.limit)) + rows = asyncio.run(collect_rows(args.repo_root, args.limit, parse_arms(args.arms))) args.out.parent.mkdir(parents=True, exist_ok=True) + # Written before the gate: the rows are evidence either way, and a refused + # run whose data was discarded cannot be diagnosed. args.out.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") + problems = validity_problems(rows) + if problems: + for problem in problems: + print(problem, file=sys.stderr) + print(f"Rows kept at {args.out}; no comparison printed.", file=sys.stderr) + return 2 report(rows) return 0 diff --git a/src/cgis/guardian/providers/base.py b/src/cgis/guardian/providers/base.py index 1752ffbd..b64cded9 100644 --- a/src/cgis/guardian/providers/base.py +++ b/src/cgis/guardian/providers/base.py @@ -86,7 +86,7 @@ def __init_subclass__(cls, **kwargs: object) -> None: return _msg = ( f"{cls.__name__} must declare `name: ClassVar[str]` — one of the " - 'GUARDIAN_PROVIDER values: "gemini", "mistral", "ollama". Add ' + 'GUARDIAN_PROVIDER values: "gemini", "mistral", "ollama", "openrouter". Add ' f'`name: ClassVar[str] = "..."` as the first line of ' f"{cls.__name__}'s class body." ) diff --git a/src/cgis/guardian/providers/openrouter.py b/src/cgis/guardian/providers/openrouter.py new file mode 100644 index 00000000..122c0e46 --- /dev/null +++ b/src/cgis/guardian/providers/openrouter.py @@ -0,0 +1,233 @@ +"""OpenRouter provider — an OpenAI-compatible gateway to many vendors (#246). + +#246 needs a skeptic from a vendor other than the finder's, and the two the +repository already speaks to are the two it is comparing. Ollama would answer +that at no API cost, but it needs a server this project does not have. OpenRouter +reaches a third vendor over an OpenAI-shaped HTTP API with a free tier, which is +what the comparison was blocked on. + +**Reasoning models set the token budget here.** The free models measured on +2026-08-17 spend most of their completion on reasoning before emitting the +answer: 709 tokens for `nemotron-3-nano-30b`, 2,133 for `nemotron-3-super-120b`, +2,243 for `cohere/north-mini-code`. A first probe capped output at 600 and read +the truncated reasoning as "this model cannot produce JSON" — the model was fine +and the measurement was not, which is why `DEFAULT_MAX_TOKENS` is generous and +why exceeding it is reported rather than returned as a short answer. + +Free-tier quota is not discoverable in advance: `/api/v1/auth/key` returns no +limit for a free key and completions carry no rate-limit headers. Exhaustion +arrives as HTTP 429 mid-run, and a 429 that degrades to "no judgement" would +look exactly like a skeptic that refutes nothing — the very hypothesis under +test. So a 429 raises, loudly, rather than being smoothed into an empty answer. +""" + +import json +import re +from typing import Any, ClassVar + +import httpx +from pydantic import BaseModel + +from cgis.guardian.providers.base import BaseProvider, ProviderUsage + +#: OpenAI-compatible completions endpoint. +OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" + +#: Wall-clock budget for one call. Measured latencies on the free tier span +#: 6.9s to 98.8s for a single judgement, so a timeout tuned to the fast models +#: would fail the slow ones by construction rather than by fault. +DEFAULT_REQUEST_TIMEOUT = 300.0 + +#: Completion budget for one call. Well above the 2,243 tokens the most verbose +#: model measured needed, because the cost of being wrong is asymmetric: a +#: budget that is too high wastes nothing on the free tier, and one that is too +#: low silently returns reasoning where a verdict was expected. +DEFAULT_MAX_TOKENS = 8000 + +#: Whether the model is asked to think before answering. +#: +#: Off, and stated rather than left to the model. `qwen3.7-plus` puts its chain +#: of thought in a separate `reasoning` field and fills `content` only at the +#: end, so on the larger prompts it spent the whole budget thinking and returned +#: `content: null` — 118 of 135 findings unruled in the first paid run, which the +#: validity gate caught and refused to score. +#: +#: Raising the budget would have fixed the symptom and broken the experiment. +#: The arm this one is compared against, `gemini-2.5-flash`, is not doing +#: extended thinking, so leaving the model's default on would make the two arms +#: differ in a dimension nobody chose — the same reason `temperature` is not +#: invented here, arriving from the other side: the honest move is to set it and +#: say so, not to inherit it and be unable to describe the run afterwards. +DEFAULT_REASONING = False + +_TOO_MANY_REQUESTS = 429 + + +class OpenRouterQuotaError(RuntimeError): + """Raised on HTTP 429 — the free tier's daily allowance is spent. + + Its own type, and never swallowed. Every other transport failure can be + retried into a judgement; this one cannot, and a run that treated it as + "the skeptic had nothing to say" would report exhausted quota as a lenient + skeptic. That is the exact shape of the result #246 is testing for, so it + must not be reachable by accident. + """ + + +class OpenRouterProvider(BaseProvider): + """A chat model reached through OpenRouter's OpenAI-compatible API.""" + + name: ClassVar[str] = "openrouter" + + def __init__( + self, + api_key: str, + model_name: str, + timeout: float = DEFAULT_REQUEST_TIMEOUT, + max_tokens: int = DEFAULT_MAX_TOKENS, + temperature: float | None = None, + reasoning: bool = DEFAULT_REASONING, + ) -> None: + """Store the key, model, timeout and generation budget. + + `temperature` is None by default and then not sent at all, leaving the + model's own value in place — the rule the Mistral and Ollama providers + already follow, so an unset knob never becomes a value this project + invented and could not describe afterwards. + """ + super().__init__() + self._api_key = api_key + self._model_name = model_name + self._timeout = timeout + self._max_tokens = max_tokens + self._temperature = temperature + self._reasoning = reasoning + + async def _post(self, payload: dict[str, Any]) -> str: + """One completion call; return the message content. + + `response_format` is passed through when the caller asks for a schema. + Not every model honours it — the ones measured here comply through the + prompt rather than the parameter — so the caller still parses + defensively; sending it costs nothing and helps where it is supported. + """ + headers = {"Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json"} + async with httpx.AsyncClient(timeout=self._timeout) as client: + response = await client.post(OPENROUTER_URL, headers=headers, json=payload) + if response.status_code == _TOO_MANY_REQUESTS: + # The body is carried, not summarised. OpenRouter says whether the + # limit was per-minute on one upstream or the account's daily + # allowance, and those have different remedies — wait, or stop. An + # exception that dropped it would leave the operator guessing, the + # same defect as a CalledProcessError that swallows stderr. + _msg = ( + f"OpenRouter returned 429 for {self._model_name}: {response.text.strip()[:400]}. " + f"A run that continued would record unanswered findings, which is " + f"indistinguishable from a skeptic that refutes nothing." + ) + raise OpenRouterQuotaError(_msg) + response.raise_for_status() + body = response.json() + # An error object with HTTP 200: OpenRouter reports upstream refusals + # this way, and `.json()["choices"]` would raise KeyError with nothing + # in the message about which provider declined or why. + # `not body.get("choices")` covers both shapes: the key absent, and the + # key present holding an empty list. An upstream content filter returns + # the second, and `body["choices"][0]` would then raise IndexError with + # nothing in the message about which provider declined. Raised in review + # of #412. + if not body.get("choices"): + _msg = f"OpenRouter returned no choices for {self._model_name}: {body}" + raise RuntimeError(_msg) + choice = body["choices"][0] + usage = body.get("usage") or {} + self._record_usage( + ProviderUsage( + prompt_tokens=int(usage.get("prompt_tokens") or 0), + completion_tokens=int(usage.get("completion_tokens") or 0), + ) + ) + if choice.get("finish_reason") == "length": + # Reported, not returned. A model cut off mid-reasoning yields prose + # where JSON was asked for, and the caller would record that as an + # unparseable answer — attributing to the model a limit this file + # chose. Measured the hard way: a 600-token probe read three capable + # models as incapable. + _msg = ( + f"{self._model_name} hit the {self._max_tokens}-token budget before finishing. " + f"Raise max_tokens; the reply so far is reasoning, not an answer." + ) + raise RuntimeError(_msg) + return str(choice["message"].get("content") or "") + + def _payload(self, system_prompt: str, user_prompt: str) -> dict[str, Any]: + """The request body common to both generation modes.""" + payload: dict[str, Any] = { + "model": self._model_name, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + "max_tokens": self._max_tokens, + # Always sent, in both directions, so a run can state what it did. + "reasoning": {"enabled": self._reasoning}, + } + # `is not None`, not truthiness: 0.0 is the one temperature that states + # something, and a falsy test would drop exactly that value. + if self._temperature is not None: + payload["temperature"] = self._temperature + return payload + + async def generate_content(self, system_prompt: str, user_prompt: str) -> str: + """Free-text generation.""" + return await self._retry(lambda: self._post(self._payload(system_prompt, user_prompt))) + + async def generate_structured( + self, system_prompt: str, user_prompt: str, schema: type[BaseModel] + ) -> str: + """Generation asked to conform to `schema`; returns raw JSON text.""" + payload = self._payload(system_prompt, user_prompt) + payload["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": schema.__name__, + "strict": True, + "schema": schema.model_json_schema(), + }, + } + return await self._retry(lambda: self._post(payload)) + + +#: An opening code fence with an optional language tag, and a closing one. +#: +#: The opening code fence and its optional language tag. +#: +#: A regex rather than splitting on a newline: a single-line fence +#: (```json {"verdict": "confirmed"}```) has none, so the split returned the +#: whole string and left the backticks in place. Raised in review of #412. +#: Character-class stripping was the suggested fix and is a trap — +#: `lstrip("json")` removes any of `j`, `o`, `s`, `n` and would eat into the +#: payload; this matches the fence and nothing else. +#: +#: Anchored at `^`, so it is tried from one position and its `\s*` cannot +#: backtrack across the input. The closing fence gets no regex at all: an +#: unanchored `\s*```$` is quadratic on a long run of whitespace that never +#: reaches a fence (SonarCloud python:S8786 on #412), and model output is not +#: input this module chooses. `str.endswith` answers the same question in one +#: comparison. +FENCE = "```" +_FENCE_OPEN = re.compile(r"^```[A-Za-z0-9_+-]*\s*") + + +def parse_or_raise(text: str, schema: type[BaseModel]) -> BaseModel: + """Validate `text` against `schema`, tolerating a fenced code block. + + Models that answer through the prompt rather than through + `response_format` often wrap the object in ```json fences. Stripping them + here keeps that from being counted as an unparseable answer, which would + inflate exactly the "unruled" rate an experiment reads as leniency. + """ + stripped = _FENCE_OPEN.sub("", text.strip(), count=1).strip() + if stripped.endswith(FENCE): + stripped = stripped[: -len(FENCE)].strip() + return schema.model_validate(json.loads(stripped)) diff --git a/src/cgis/guardian/review_fingerprint.py b/src/cgis/guardian/review_fingerprint.py index 2506e3b0..c1b18f0c 100644 --- a/src/cgis/guardian/review_fingerprint.py +++ b/src/cgis/guardian/review_fingerprint.py @@ -149,13 +149,13 @@ def walk_closure(read: ReadFile, active_providers: frozenset[str]) -> list[str]: `active_providers` is the union of the finder's and the skeptic's provider names. Unselected providers are pruned **during** the walk, not filtered - afterwards: `runner.py` imports all three at module level, so a full + afterwards: `runner.py` imports every one of them at module level, so a full traversal minus a set would still have reached what an unselected provider imports. """ known = frozenset( name - for name in ("gemini", "mistral", "ollama") + for name in ("gemini", "mistral", "ollama", "openrouter") if _module_path(f"{_PROVIDER_PACKAGE}.{name}", read) is not None ) unknown = active_providers - known diff --git a/src/cgis/guardian/runner.py b/src/cgis/guardian/runner.py index 76856a1d..8d5456f7 100644 --- a/src/cgis/guardian/runner.py +++ b/src/cgis/guardian/runner.py @@ -3,7 +3,7 @@ import asyncio import math import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from pathlib import Path from typing import Literal @@ -23,6 +23,7 @@ DEFAULT_OLLAMA_NUM_PREDICT, OllamaProvider, ) +from cgis.guardian.providers.openrouter import OpenRouterProvider from cgis.guardian.recording import save_finder_recording from cgis.guardian.render import render_report @@ -200,6 +201,32 @@ def _build_gemini(env: Mapping[str, str], model_override: str | None) -> tuple[B return GeminiProvider(api_key=key, model_name=model), model +def _build_openrouter( + env: Mapping[str, str], model_override: str | None +) -> tuple[BaseProvider, str]: + """Construct an OpenRouterProvider; OPENROUTER_API_KEY and a model are required. + + No default model on purpose. OpenRouter fronts hundreds of models whose + availability and cost differ by orders of magnitude, and a default would + silently pick one nobody chose — the opposite of what this provider exists + for, which is naming a third vendor explicitly (#246). + """ + key = env.get("OPENROUTER_API_KEY") + if not key: + _msg = "OPENROUTER_API_KEY must be set when GUARDIAN_PROVIDER=openrouter" + raise RuntimeError(_msg) + if not model_override: + _msg = ( + "GUARDIAN_MODEL must name an OpenRouter model (e.g. " + "'nvidia/nemotron-3-super-120b-a12b:free'); there is no default." + ) + raise RuntimeError(_msg) + return ( + OpenRouterProvider(api_key=key, model_name=model_override, temperature=temperature(env)), + model_override, + ) + + def _build_ollama(env: Mapping[str, str], model_override: str | None) -> tuple[BaseProvider, str]: """Construct an OllamaProvider; GUARDIAN_MODEL names the model (no API key).""" if not model_override: @@ -257,10 +284,18 @@ def build_provider(env: Mapping[str, str]) -> tuple[BaseProvider, str]: model_override = _model_from_env(env, "GUARDIAN_MODEL") provider_name = env.get("GUARDIAN_PROVIDER", "").lower() or _autodetect_provider(env) - builders = {"mistral": _build_mistral, "gemini": _build_gemini, "ollama": _build_ollama} + builders = { + "mistral": _build_mistral, + "gemini": _build_gemini, + "ollama": _build_ollama, + "openrouter": _build_openrouter, + } builder = builders.get(provider_name) if builder is None: - _msg = f"Unknown GUARDIAN_PROVIDER={provider_name!r}. Use 'mistral', 'gemini', or 'ollama'." + _msg = ( + f"Unknown GUARDIAN_PROVIDER={provider_name!r}. " + f"Use one of: {', '.join(sorted(builders))}." + ) raise RuntimeError(_msg) return builder(env, model_override) @@ -275,58 +310,106 @@ def _autodetect_provider(env: Mapping[str, str]) -> str: raise RuntimeError(_msg) +def _skeptic_ollama( + env: Mapping[str, str], model_override: str | None +) -> tuple[BaseProvider, str] | None: + """A local ollama skeptic; the model must be named, there is no default.""" + model = model_override or _model_from_env(env, "GUARDIAN_MODEL") + if not model: + log.warning( + "Skeptic disabled: set GUARDIAN_SKEPTIC_MODEL (or GUARDIAN_MODEL) " + "for an ollama skeptic." + ) + return None + provider = OllamaProvider( + model_name=model, + host=_ollama_host(env), + num_ctx=_ollama_num_ctx(env), + temperature=temperature(env), + penalties=_ollama_penalties(env), + num_predict=_ollama_num_predict(env), + ) + return provider, model + + +def _skeptic_openrouter( + env: Mapping[str, str], model_override: str | None +) -> tuple[BaseProvider, str] | None: + """An OpenRouter skeptic; both the key and an explicit model are required.""" + key = env.get("OPENROUTER_API_KEY") + if not key or not model_override: + log.warning( + "Skeptic disabled: an openrouter skeptic needs OPENROUTER_API_KEY and " + "GUARDIAN_SKEPTIC_MODEL; there is no default model." + ) + return None + provider = OpenRouterProvider( + api_key=key, model_name=model_override, temperature=temperature(env) + ) + return provider, model_override + + +def _skeptic_mistral( + env: Mapping[str, str], model_override: str | None +) -> tuple[BaseProvider, str] | None: + """A mistral skeptic, on the finder's temperature.""" + key = env.get("MISTRAL_API_KEY") + if not key: + log.warning("Skeptic disabled: MISTRAL_API_KEY not set.") + return None + model = model_override or DEFAULT_MISTRAL_MODEL + # Same temperature as the finder: the run registers one sampling setting, + # not one per pass. + return MistralProvider(api_key=key, model_name=model, temperature=temperature(env)), model + + +def _skeptic_gemini( + env: Mapping[str, str], model_override: str | None +) -> tuple[BaseProvider, str] | None: + """A gemini skeptic.""" + key = env.get("GEMINI_API_KEY") + if not key: + log.warning("Skeptic disabled: GEMINI_API_KEY not set.") + return None + model = model_override or DEFAULT_GEMINI_MODEL + return GeminiProvider(api_key=key, model_name=model), model + + +#: One builder per skeptic provider. A table rather than a chain of `if name ==` +#: blocks: the chain reached cognitive complexity 18 against a limit of 15 when +#: openrouter joined it (SonarCloud on #412), and each arm's requirements — which +#: key, which default model, whether one exists at all — are per-provider facts +#: that belong beside that provider rather than interleaved with the others. +_SKEPTIC_BUILDERS: dict[ + str, Callable[[Mapping[str, str], str | None], tuple[BaseProvider, str] | None] +] = { + "ollama": _skeptic_ollama, + "openrouter": _skeptic_openrouter, + "mistral": _skeptic_mistral, + "gemini": _skeptic_gemini, +} + + def build_skeptic_provider( env: Mapping[str, str], *, primary: str ) -> tuple[BaseProvider, str] | None: """Return (skeptic_provider, model) or None for single-pass (spec §5.5). Default skeptic = the provider opposite to the primary; GUARDIAN_SKEPTIC - overrides ('gemini'|'mistral'|'ollama'|'off'); GUARDIAN_SKEPTIC_MODEL overrides - the model, enabling same-provider/different-model pairs (incl. two distinct - local Ollama models — a cross-model skeptic with no API cost). A missing API - key / model degrades to None — a review never fails because of the skeptic. + overrides ('gemini'|'mistral'|'ollama'|'openrouter'|'off'); + GUARDIAN_SKEPTIC_MODEL overrides the model, enabling + same-provider/different-model pairs (incl. two distinct local Ollama models — + a cross-model skeptic with no API cost). A missing API key / model degrades + to None — a review never fails because of the skeptic. """ choice = env.get("GUARDIAN_SKEPTIC", "").lower() if choice == "off": return None - if choice not in ("", "gemini", "mistral", "ollama"): + if choice and choice not in _SKEPTIC_BUILDERS: log.warning("Unknown GUARDIAN_SKEPTIC; skeptic disabled.", value=choice) return None name = choice or ("mistral" if primary == "gemini" else "gemini") - model_override = _model_from_env(env, "GUARDIAN_SKEPTIC_MODEL") - if name == "ollama": - model = model_override or _model_from_env(env, "GUARDIAN_MODEL") - if not model: - log.warning( - "Skeptic disabled: set GUARDIAN_SKEPTIC_MODEL (or GUARDIAN_MODEL) " - "for an ollama skeptic." - ) - return None - provider = OllamaProvider( - model_name=model, - host=_ollama_host(env), - num_ctx=_ollama_num_ctx(env), - temperature=temperature(env), - penalties=_ollama_penalties(env), - num_predict=_ollama_num_predict(env), - ) - return provider, model - if name == "mistral": - key = env.get("MISTRAL_API_KEY") - if not key: - log.warning("Skeptic disabled: MISTRAL_API_KEY not set.") - return None - model = model_override or DEFAULT_MISTRAL_MODEL - # Same temperature as the finder: the run registers one sampling - # setting, not one per pass. - skeptic = MistralProvider(api_key=key, model_name=model, temperature=temperature(env)) - return skeptic, model - key = env.get("GEMINI_API_KEY") - if not key: - log.warning("Skeptic disabled: GEMINI_API_KEY not set.") - return None - model = model_override or DEFAULT_GEMINI_MODEL - return GeminiProvider(api_key=key, model_name=model), model + return _SKEPTIC_BUILDERS[name](env, _model_from_env(env, "GUARDIAN_SKEPTIC_MODEL")) def build_footer( diff --git a/src/cgis/guardian/skeptic.py b/src/cgis/guardian/skeptic.py index c4fd75fb..4e034f9f 100644 --- a/src/cgis/guardian/skeptic.py +++ b/src/cgis/guardian/skeptic.py @@ -212,8 +212,20 @@ async def judge_finding( FindingJudgement, ) return FindingJudgement.model_validate_json(extract_json(raw)) - except Exception: - log.warning("Skeptic judgement failed; finding stays unruled.", file=finding.file) + except Exception as exc: + # The reason is logged, not just the fact. Containing the error per + # finding is deliberate (#246 §3.4) — one bad response must not cost the + # pass — but a warning that says only "failed" leaves an operator with + # 118 identical lines and no way to tell a spent quota from a truncated + # reasoning model from unparseable JSON. Each of those has a different + # fix, and all three arrive here. Diagnosing one cost half an hour of + # guessing before this line carried the type and the message. + log.warning( + "Skeptic judgement failed; finding stays unruled.", + file=finding.file, + error_type=type(exc).__name__, + error=str(exc)[:300], + ) return None diff --git a/tests/unit/test_guardian_openrouter.py b/tests/unit/test_guardian_openrouter.py new file mode 100644 index 00000000..0c2f802a --- /dev/null +++ b/tests/unit/test_guardian_openrouter.py @@ -0,0 +1,246 @@ +"""The OpenRouter provider (#246), with the network stubbed. + +Every branch here exists because failing it quietly would corrupt the +experiment the provider was added for. A spent quota, a truncated reasoning +model and an upstream refusal all produce "no verdict", and a skeptic that +produces no verdict is indistinguishable from one that refutes nothing — which +is the hypothesis #246 tests. +""" + +import json +from typing import Any + +import httpx +import pytest +from pydantic import BaseModel + +from cgis.guardian.providers import openrouter as mod +from cgis.guardian.providers.openrouter import ( + OpenRouterProvider, + OpenRouterQuotaError, + parse_or_raise, +) + + +class _Judgement(BaseModel): + verdict: str + rationale: str + + +def _reply( + content: str = '{"verdict": "refuted", "rationale": "r"}', + *, + status: int = 200, + finish: str = "stop", + usage: dict[str, int] | None = None, + body: dict[str, Any] | None = None, +) -> httpx.Response: + payload = body or { + "choices": [{"message": {"content": content}, "finish_reason": finish}], + "usage": usage or {"prompt_tokens": 11, "completion_tokens": 7}, + } + return httpx.Response(status, json=payload, request=httpx.Request("POST", mod.OPENROUTER_URL)) + + +def _serve(monkeypatch: pytest.MonkeyPatch, response: httpx.Response) -> list[dict[str, Any]]: + """Route every POST to `response`; return the list of request bodies sent.""" + sent: list[dict[str, Any]] = [] + + class _Client: + def __init__(self, **_kw: object) -> None: + pass + + async def __aenter__(self) -> "_Client": + return self + + async def __aexit__(self, *_exc: object) -> None: + return None + + async def post(self, _url: str, **kwargs: object) -> httpx.Response: + sent.append(kwargs["json"]) # type: ignore[arg-type] + return response + + monkeypatch.setattr(mod.httpx, "AsyncClient", _Client) + return sent + + +def _provider( + max_tokens: int = mod.DEFAULT_MAX_TOKENS, + temperature: float | None = None, + reasoning: bool = mod.DEFAULT_REASONING, +) -> OpenRouterProvider: + return OpenRouterProvider( + api_key="k", + model_name="vendor/model:free", + max_tokens=max_tokens, + temperature=temperature, + reasoning=reasoning, + ) + + +class TestTheFailuresThatWouldLookLikeLeniency: + """Each of these returns "no verdict", and each must say why.""" + + @pytest.mark.asyncio + async def test_a_spent_quota_raises_rather_than_returning_nothing( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """429 is not an answer. The free tier gives no warning before it. + + `/api/v1/auth/key` reports no limit for a free key and completions carry + no rate-limit headers, so exhaustion is only ever discovered by hitting + it — mid-run, after some findings have been judged and some have not. + """ + _serve(monkeypatch, _reply(status=429)) + provider = _provider() + with pytest.raises(OpenRouterQuotaError, match="refutes nothing"): + await provider.generate_content("s", "u") + + @pytest.mark.asyncio + async def test_hitting_the_token_budget_raises_rather_than_returning_reasoning( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The mistake that read three capable models as incapable. + + A 600-token probe cut these reasoning models off mid-thought and + returned the reasoning as the answer, which parses as nothing. The limit + was this project's choice; attributing it to the model was the error. + """ + _serve(monkeypatch, _reply(content="We need to assess the claim", finish="length")) + provider = _provider(max_tokens=600) + with pytest.raises(RuntimeError, match="reasoning, not an answer"): + await provider.generate_content("s", "u") + + @pytest.mark.asyncio + async def test_an_error_object_with_http_200_is_reported( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """OpenRouter reports an upstream refusal this way, with a 200. + + `body["choices"]` would raise KeyError and lose the message naming which + provider declined — which is how the account's allowed-providers setting + surfaces, and it is a one-click fix nobody can apply without the text. + """ + refusal = {"error": {"message": "No allowed providers are available"}} + _serve(monkeypatch, _reply(body=refusal)) + provider = _provider() + with pytest.raises(RuntimeError, match="No allowed providers"): + await provider.generate_content("s", "u") + + @pytest.mark.asyncio + async def test_an_empty_choices_list_is_reported_not_an_indexerror( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The key present and the list empty — an upstream content filter. + + Raised in review of #412. `"choices" not in body` passed this and + `body["choices"][0]` then raised IndexError, whose message says nothing + about which provider declined or why. + """ + _serve(monkeypatch, _reply(body={"choices": [], "usage": {}})) + provider = _provider() + with pytest.raises(RuntimeError, match="no choices"): + await provider.generate_content("s", "u") + + +class TestTheRequest: + """What goes on the wire, and what deliberately does not.""" + + @pytest.mark.asyncio + async def test_an_unset_temperature_is_not_sent(self, monkeypatch: pytest.MonkeyPatch) -> None: + """An unset knob must not become a value this project invented.""" + sent = _serve(monkeypatch, _reply()) + await _provider().generate_content("s", "u") + assert "temperature" not in sent[0] + + @pytest.mark.asyncio + async def test_a_zero_temperature_is_sent(self, monkeypatch: pytest.MonkeyPatch) -> None: + """0.0 is the one temperature that states something, and it is falsy.""" + sent = _serve(monkeypatch, _reply()) + await _provider(temperature=0.0).generate_content("s", "u") + assert sent[0]["temperature"] == 0.0 + + @pytest.mark.asyncio + async def test_reasoning_is_stated_in_both_directions( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Sent whether on or off, so a run can describe what it did. + + Left unsent, `qwen3.7-plus` thinks by default, puts the thought in a + separate field and fills `content` only at the end — it spent the whole + budget reasoning and returned nothing on 118 of 135 findings. The arm it + is compared against is not thinking, so inheriting the default would + make the two arms differ in a dimension nobody chose. + """ + sent = _serve(monkeypatch, _reply()) + await _provider().generate_content("s", "u") + assert sent[0]["reasoning"] == {"enabled": False} + sent_on = _serve(monkeypatch, _reply()) + await _provider(reasoning=True).generate_content("s", "u") + assert sent_on[0]["reasoning"] == {"enabled": True} + + @pytest.mark.asyncio + async def test_structured_generation_carries_the_schema( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + sent = _serve(monkeypatch, _reply()) + await _provider().generate_structured("s", "u", _Judgement) + fmt = sent[0]["response_format"] + assert fmt["type"] == "json_schema" + assert fmt["json_schema"]["name"] == "_Judgement" + assert "verdict" in fmt["json_schema"]["schema"]["properties"] + + @pytest.mark.asyncio + async def test_usage_is_recorded(self, monkeypatch: pytest.MonkeyPatch) -> None: + _serve(monkeypatch, _reply(usage={"prompt_tokens": 100, "completion_tokens": 20})) + provider = _provider() + await provider.generate_content("s", "u") + assert provider.last_usage.total_tokens == 120 + assert provider.cumulative_usage.total_tokens == 120 + + @pytest.mark.asyncio + async def test_a_reply_with_no_usage_is_not_a_crash( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Some upstreams omit it; losing the token count must not lose the answer.""" + _serve(monkeypatch, _reply(body={"choices": [{"message": {"content": "ok"}}]})) + assert await _provider().generate_content("s", "u") == "ok" + + +class TestFencedJson: + """Models that answer through the prompt rather than `response_format`.""" + + def test_a_bare_object_parses(self) -> None: + parsed = parse_or_raise('{"verdict": "refuted", "rationale": "r"}', _Judgement) + assert parsed.verdict == "refuted" # type: ignore[attr-defined] + + def test_a_fenced_object_parses(self) -> None: + """```json fences would otherwise count as an unparseable answer. + + That inflates the "unruled" rate, and a high unruled rate is read as a + lenient skeptic — the exact confusion this whole file guards against. + """ + text = '```json\n{"verdict": "confirmed", "rationale": "r"}\n```' + parsed = parse_or_raise(text, _Judgement) + assert parsed.verdict == "confirmed" # type: ignore[attr-defined] + + def test_a_single_line_fence_parses(self) -> None: + """No newline to split on — the shape the first version left broken. + + Raised in review of #412. Splitting on `\\n` returned the whole string + when there was none, so the backticks survived and `json.loads` failed, + counting a perfectly good answer as unruled. + """ + text = '```json {"verdict": "refuted", "rationale": "r"}```' + parsed = parse_or_raise(text, _Judgement) + assert parsed.verdict == "refuted" # type: ignore[attr-defined] + + def test_a_fence_with_no_language_tag_parses(self) -> None: + text = '```\n{"verdict": "uncertain", "rationale": "r"}```' + parsed = parse_or_raise(text, _Judgement) + assert parsed.verdict == "uncertain" # type: ignore[attr-defined] + + def test_prose_still_raises(self) -> None: + """Tolerating fences must not become tolerating anything.""" + with pytest.raises(json.JSONDecodeError): + parse_or_raise("I think the claim is correct.", _Judgement) diff --git a/tests/unit/test_guardian_runner.py b/tests/unit/test_guardian_runner.py index cf1f8ba9..a885df56 100644 --- a/tests/unit/test_guardian_runner.py +++ b/tests/unit/test_guardian_runner.py @@ -195,10 +195,19 @@ def test_build_provider_ollama_num_ctx_from_env() -> None: assert provider._num_ctx == 8192 # noqa: SLF001 # white-box: ctx wiring -def test_build_provider_unknown_lists_ollama() -> None: - """A typo in GUARDIAN_PROVIDER names all three valid providers.""" - with pytest.raises(RuntimeError, match="'mistral', 'gemini', or 'ollama'"): +def test_build_provider_unknown_lists_every_provider_there_is() -> None: + """A typo in GUARDIAN_PROVIDER names the valid ones — all of them. + + The message is built from the builder table rather than written out, so it + cannot list three providers while four exist. This test used to assert the + literal `'mistral', 'gemini', or 'ollama'` and went stale the moment + openrouter arrived (#246), which is the failure mode the derivation closes. + """ + with pytest.raises(RuntimeError) as raised: build_provider({"GUARDIAN_PROVIDER": "anthropic", "GEMINI_API_KEY": "g"}) + message = str(raised.value) + for name in ("mistral", "gemini", "ollama", "openrouter"): + assert name in message def test_build_skeptic_provider_ollama_cross_model() -> None: @@ -757,3 +766,61 @@ def test_zero_and_a_normal_value_still_pass(self) -> None: """The guard must not cost the settings it exists to protect.""" assert runner.temperature_setting({"GUARDIAN_TEMPERATURE": "0"}) == (0.0, "explicit") assert runner.temperature_setting({"GUARDIAN_TEMPERATURE": "0.7"}) == (0.7, "explicit") + + +def test_build_provider_openrouter_requires_an_explicit_model() -> None: + """No default model, on purpose: OpenRouter fronts hundreds of them. + + A default would silently pick one nobody chose, which is the opposite of + what this provider is for — naming a third vendor explicitly (#246). + """ + with pytest.raises(RuntimeError, match="GUARDIAN_MODEL must name an OpenRouter model"): + build_provider({"GUARDIAN_PROVIDER": "openrouter", "OPENROUTER_API_KEY": "k"}) + + +def test_build_provider_openrouter_requires_a_key() -> None: + env = {"GUARDIAN_PROVIDER": "openrouter", "GUARDIAN_MODEL": "vendor/m:free"} + with pytest.raises(RuntimeError, match="OPENROUTER_API_KEY must be set"): + build_provider(env) + + +def test_build_provider_openrouter_builds() -> None: + provider, model = build_provider( + { + "GUARDIAN_PROVIDER": "openrouter", + "OPENROUTER_API_KEY": "k", + "GUARDIAN_MODEL": "vendor/m:free", + } + ) + assert provider.name == "openrouter" + assert model == "vendor/m:free" + + +def test_build_skeptic_openrouter_needs_both_key_and_model() -> None: + """Degrades to None rather than raising: a review never fails on the skeptic.""" + assert ( + build_skeptic_provider( + {"GUARDIAN_SKEPTIC": "openrouter", "OPENROUTER_API_KEY": "k"}, primary="mistral" + ) + is None + ) + assert ( + build_skeptic_provider( + {"GUARDIAN_SKEPTIC": "openrouter", "GUARDIAN_SKEPTIC_MODEL": "v/m"}, primary="mistral" + ) + is None + ) + + +def test_build_skeptic_openrouter_builds() -> None: + built = build_skeptic_provider( + { + "GUARDIAN_SKEPTIC": "openrouter", + "OPENROUTER_API_KEY": "k", + "GUARDIAN_SKEPTIC_MODEL": "vendor/m:free", + }, + primary="mistral", + ) + assert built is not None + assert built[0].name == "openrouter" + assert built[1] == "vendor/m:free" diff --git a/tests/unit/test_skeptic_arms.py b/tests/unit/test_skeptic_arms.py index ff20fb21..618f4ed8 100644 --- a/tests/unit/test_skeptic_arms.py +++ b/tests/unit/test_skeptic_arms.py @@ -19,7 +19,15 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "scripts")) import skeptic_arms -from skeptic_arms import ARMS, NoArmError, _vendor, arm_provider, finder_models, judge_one, report +from skeptic_arms import ( + DEFAULT_ARMS, + NoArmError, + _vendor, + arm_provider, + finder_models, + judge_one, + report, +) REPO_ROOT = Path(__file__).resolve().parent.parent.parent BENCH_DIR = REPO_ROOT / "benchmarks" / "guardian" @@ -50,9 +58,9 @@ def _capture(env: dict[str, str], **_kw: object) -> tuple[StubProvider, str]: return StubProvider([]), "m" monkeypatch.setattr(skeptic_arms, "build_skeptic_provider", _capture) - for name in ARMS: + for name in DEFAULT_ARMS.split(","): arm_provider(name, {}) - assert seen == list(ARMS) + assert seen == DEFAULT_ARMS.split(",") def test_a_skeptic_model_in_the_environment_reaches_neither_arm( self, monkeypatch: pytest.MonkeyPatch @@ -77,6 +85,38 @@ def _capture(env: dict[str, str], **_kw: object) -> tuple[StubProvider, str]: assert "GUARDIAN_SKEPTIC_MODEL" not in envs[0] assert envs[0]["X"] == "keep" + def test_an_arms_own_model_replaces_the_environments( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Per arm, because one arm may need a model the other cannot use. + + `openrouter` has no default model by design — it fronts hundreds — so it + must be told one, and telling it through the environment would tell the + other arm too. + """ + envs: list[dict[str, str]] = [] + + def _capture(env: dict[str, str], **_kw: object) -> tuple[StubProvider, str]: + envs.append(dict(env)) + return StubProvider([]), "m" + + monkeypatch.setattr(skeptic_arms, "build_skeptic_provider", _capture) + arm_provider("openrouter", {"GUARDIAN_SKEPTIC_MODEL": "gemini-2.5-flash"}, "vendor/m:free") + assert envs[0]["GUARDIAN_SKEPTIC_MODEL"] == "vendor/m:free" + + @pytest.mark.parametrize( + ("spec", "expected"), + [ + ("gemini,mistral", {"gemini": None, "mistral": None}), + ("gemini,openrouter=vendor/m:free", {"gemini": None, "openrouter": "vendor/m:free"}), + (" gemini , mistral ", {"gemini": None, "mistral": None}), + ("gemini,", {"gemini": None}), + ], + ) + def test_the_arm_spec_parses(self, spec: str, expected: dict[str, str | None]) -> None: + """A model may carry `/` and `:` — the split is on the first `=` only.""" + assert skeptic_arms.parse_arms(spec) == expected + class TestTheVendorSplit: """Both orientations are in the corpus, and that is what the split rests on.""" @@ -186,7 +226,9 @@ def _wire( path.write_text("{}", encoding="utf-8") collected: list[str] = [] monkeypatch.setattr( - skeptic_arms, "arm_provider", lambda name, _env: (StubProvider([]), f"{name}-model") + skeptic_arms, + "arm_provider", + lambda name, _env, _model=None: (StubProvider([]), f"{name}-model"), ) monkeypatch.setattr(skeptic_arms, "finder_models", lambda _p: {}) monkeypatch.setattr(skeptic_arms, "build", lambda *_a: paths) @@ -216,9 +258,11 @@ async def test_evidence_is_collected_once_per_pr_not_once_per_pass( and differs only in how long it takes — invisible in the output. """ collected = self._wire(monkeypatch, tmp_path, ["143@a", "143@b", "143@c"]) - rows = await skeptic_arms.collect_rows(REPO_ROOT, None) + rows = await skeptic_arms.collect_rows( + REPO_ROOT, None, skeptic_arms.parse_arms(DEFAULT_ARMS) + ) assert len(collected) == 1 - assert len(rows) == 3 * len(ARMS) + assert len(rows) == 3 * 2 @pytest.mark.asyncio async def test_each_pass_is_judged_by_every_arm( @@ -226,11 +270,13 @@ async def test_each_pass_is_judged_by_every_arm( ) -> None: """Pairing is the whole design: the same findings, both skeptics.""" self._wire(monkeypatch, tmp_path, ["143@a", "144@b"]) - rows = await skeptic_arms.collect_rows(REPO_ROOT, None) + rows = await skeptic_arms.collect_rows( + REPO_ROOT, None, skeptic_arms.parse_arms(DEFAULT_ARMS) + ) by_pass: dict[str, set[str]] = {} for row in rows: by_pass.setdefault(str(row["pass"]), set()).add(str(row["arm"])) - assert by_pass == {"143@a": set(ARMS), "144@b": set(ARMS)} + assert by_pass == {"143@a": {"gemini", "mistral"}, "144@b": {"gemini", "mistral"}} @pytest.mark.asyncio async def test_limit_trims_the_work_before_a_worktree_is_built( @@ -238,9 +284,9 @@ async def test_limit_trims_the_work_before_a_worktree_is_built( ) -> None: """`--limit` is the smoke run; it must cost one PR, not all of them.""" collected = self._wire(monkeypatch, tmp_path, ["143@a", "144@b"]) - rows = await skeptic_arms.collect_rows(REPO_ROOT, 1) + rows = await skeptic_arms.collect_rows(REPO_ROOT, 1, skeptic_arms.parse_arms(DEFAULT_ARMS)) assert len(collected) == 1 - assert len(rows) == len(ARMS) + assert len(rows) == 2 @pytest.mark.asyncio async def test_a_pass_with_no_findings_is_dropped_before_any_worktree( @@ -253,11 +299,66 @@ async def test_a_pass_with_no_findings_is_dropped_before_any_worktree( monkeypatch.setattr( skeptic_arms, "evidence_for_pr", lambda *_a: collected.append("x") or object() ) - rows = await skeptic_arms.collect_rows(REPO_ROOT, None) + rows = await skeptic_arms.collect_rows( + REPO_ROOT, None, skeptic_arms.parse_arms(DEFAULT_ARMS) + ) assert rows == [] assert collected == [] +class TestTheValidityGate: + """The only thing between a broken arm and a confirmed hypothesis.""" + + def test_a_quiet_arm_is_reported_as_a_problem(self) -> None: + """`judge_finding` turns every provider error into `unruled`. + + A spent quota, an upstream 429 and a model that cannot emit JSON all + arrive as the same row shape as a skeptic that declined to refute — and + "refutes nothing" is what #246 predicts for a same-vendor skeptic. So a + broken arm would *confirm* the hypothesis if this did not fire. + """ + rows: list[dict[str, Any]] = [ + {"arm": "openrouter", "findings": 10, "unruled": 4}, + {"arm": "gemini", "findings": 10, "unruled": 0}, + ] + problems = skeptic_arms.validity_problems(rows) + assert len(problems) == 1 + assert "openrouter" in problems[0] + assert "4 of 10" in problems[0] + + def test_a_healthy_pair_of_arms_raises_nothing(self) -> None: + rows: list[dict[str, Any]] = [ + {"arm": "openrouter", "findings": 100, "unruled": 2}, + {"arm": "gemini", "findings": 100, "unruled": 0}, + ] + assert skeptic_arms.validity_problems(rows) == [] + + def test_main_refuses_to_print_a_comparison_it_cannot_read( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + """Exit 2 and no table — and the rows are still written. + + Refused rather than footnoted: a caveat under a table does not stop the + table being read, and this table's whole content is how much each arm + refuted. The rows are kept because a refused run nobody can diagnose is + a second failure. + """ + + async def _rows( + _root: Path, _limit: int | None, _arms: dict[str, str | None] + ) -> list[dict[str, Any]]: + return [{"arm": "openrouter", "findings": 10, "unruled": 9, "finder_model": "x"}] + + out = tmp_path / "arms.jsonl" + monkeypatch.setattr(skeptic_arms, "collect_rows", _rows) + monkeypatch.setattr(sys, "argv", ["skeptic_arms.py", "--out", str(out)]) + assert skeptic_arms.main() == 2 + captured = capsys.readouterr() + assert "may be silence" in captured.err + assert "(pass, arm) results" not in captured.out + assert out.read_text(encoding="utf-8").strip() + + def test_main_writes_the_rows_it_collected( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: @@ -275,7 +376,9 @@ def test_main_writes_the_rows_it_collected( "killed_gt": [], } - async def _rows(_root: Path, _limit: int | None) -> list[dict[str, Any]]: + async def _rows( + _root: Path, _limit: int | None, _arms: dict[str, str | None] + ) -> list[dict[str, Any]]: return [row] out = tmp_path / "nested" / "arms.jsonl"