From 59674993f698ed13820c5ce75d35cc247f35c125 Mon Sep 17 00:00:00 2001 From: zaebee Date: Mon, 17 Aug 2026 13:37:36 +0000 Subject: [PATCH 1/4] feat(guardian): an OpenRouter provider, so #246 has a third vendor to compare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two providers this repository speaks to are the two #246 is comparing, and the local mistral key returns 402. Ollama would answer it for nothing but needs a server this project does not have. OpenRouter reaches a third vendor over an OpenAI-shaped API, which is what the comparison was blocked on. Three things in the provider are measurements rather than choices. max_tokens is generous because a first probe capped output at 600 and read three capable reasoning models as incapable: they spend 709 to 2,243 tokens reasoning before the answer, so the truncated reasoning came back where JSON was expected. Hitting the budget now raises instead of returning that prefix, because the limit is this file's choice and attributing it to the model was the error. A 429 raises and carries OpenRouter's body. The free tier gives no warning — /api/v1/auth/key reports no limit and completions carry no rate-limit headers — and the body is what distinguishes an account's daily allowance from one upstream's shared pool, which is why z-ai/glm-5.2:free was rejected as an arm. An error object arriving with HTTP 200 is reported rather than KeyError'd; that is how the account's allowed-providers setting surfaces, and the remedy is in the message. skeptic_arms gains a per-arm model (--arms gemini,openrouter=vendor/model) and a validity gate. The gate is not a tolerance: judge_finding contains provider errors per finding and returns None, so a spent quota, a 429 and unparseable JSON all arrive as 'unruled' — the same row shape as a skeptic that refuted nothing, which is what #246 predicts for a same-vendor skeptic. A broken arm would have confirmed the hypothesis. It fired on the first real run: the free nemotron arm left 96 of 135 findings unruled, all after the first 39, which with today's probes puts the free tier at 50 requests per day. Measured, not recalled. Co-Authored-By: Claude Opus 5 --- scripts/skeptic_arms.py | 110 ++++++++++-- src/cgis/guardian/providers/base.py | 2 +- src/cgis/guardian/providers/openrouter.py | 187 +++++++++++++++++++++ src/cgis/guardian/review_fingerprint.py | 4 +- src/cgis/guardian/runner.py | 55 +++++- tests/unit/test_guardian_openrouter.py | 193 ++++++++++++++++++++++ tests/unit/test_guardian_runner.py | 15 +- tests/unit/test_skeptic_arms.py | 127 ++++++++++++-- 8 files changed, 656 insertions(+), 37 deletions(-) create mode 100644 src/cgis/guardian/providers/openrouter.py create mode 100644 tests/unit/test_guardian_openrouter.py diff --git a/scripts/skeptic_arms.py b/scripts/skeptic_arms.py index cec3f1b0..4837ab15 100644 --- a/scripts/skeptic_arms.py +++ b/scripts/skeptic_arms.py @@ -66,10 +66,17 @@ REPO_ROOT = Path(__file__).resolve().parent.parent BENCH_DIR = REPO_ROOT / "benchmarks" / "guardian" -#: The two arms. Named rather than derived, because "the opposite of the primary" -#: is exactly the rule under test — deriving the arm from the finder would build -#: the hypothesis into the instrument. -ARMS = ("gemini", "mistral") +#: The default pair. Named rather than derived, because "the opposite of the +#: primary" is exactly the rule under test — deriving an arm from the finder +#: would build the hypothesis into the instrument. +#: +#: `--arms` overrides it, and each arm may name its own model +#: (`openrouter=nvidia/nemotron-3-super-120b-a12b:free`). Per arm rather than +#: from the environment: `GUARDIAN_SKEPTIC_MODEL` applies to whichever provider +#: is built, so a single value would hand one arm the other's model — one arm +#: running something that does not exist, reported as a difference between +#: vendors. +DEFAULT_ARMS = "gemini,mistral" class NoArmError(RuntimeError): @@ -80,17 +87,36 @@ class NoArmError(RuntimeError): """ -def arm_provider(name: str, env: Mapping[str, str]) -> tuple[BaseProvider, str]: +def parse_arms(spec: str) -> dict[str, str | None]: + """`gemini,openrouter=vendor/model:free` → {arm: model or None}. + + A model per arm, never one shared. The environment's + `GUARDIAN_SKEPTIC_MODEL` is dropped in `arm_provider` for the same reason. + """ + arms: dict[str, str | None] = {} + for item in spec.split(","): + name, _, model = item.strip().partition("=") + if name: + arms[name] = model or None + return arms + + +def arm_provider( + name: str, env: Mapping[str, str], model: str | None = None +) -> tuple[BaseProvider, str]: """The skeptic for one arm, or a refusal naming the missing key. - `GUARDIAN_SKEPTIC_MODEL` is dropped rather than passed through. - `build_skeptic_provider` applies it to whichever provider it builds, so an - environment holding the production value (`gemini-2.5-flash`) would hand - that model name to the mistral arm — one arm running a model that does not - exist, while the other ran the intended one. Each arm takes its provider's - own default, and the models used are recorded on every row. + The environment's `GUARDIAN_SKEPTIC_MODEL` is dropped and `model` used + instead. `build_skeptic_provider` applies that variable to whichever + provider it builds, so an environment holding the production value + (`gemini-2.5-flash`) would hand that name to a mistral or openrouter arm — + one arm running a model that does not exist while the other ran the intended + one, and the difference reported as a comparison between vendors. The models + actually used are recorded on every row. """ per_arm = {k: v for k, v in env.items() if k != "GUARDIAN_SKEPTIC_MODEL"} + if model: + per_arm["GUARDIAN_SKEPTIC_MODEL"] = model built = build_skeptic_provider({**per_arm, "GUARDIAN_SKEPTIC": name}, primary="none") if built is None: _msg = ( @@ -162,7 +188,9 @@ async def judge_one( } -async def collect_rows(repo_root: Path, limit: int | None) -> list[dict[str, Any]]: +async def collect_rows( + repo_root: Path, limit: int | None, arm_spec: dict[str, str | None] +) -> list[dict[str, Any]]: """Both arms over every frozen pass; one row per (pass, arm). Returns the rows rather than writing them. The write is the caller's, and @@ -170,7 +198,7 @@ async def collect_rows(repo_root: Path, limit: int | None) -> list[dict[str, Any the worktree collection above avoids with `to_thread`, and here there is nothing to gain by being in the loop at all. """ - arms = {name: arm_provider(name, os.environ) for name in ARMS} + arms = {name: arm_provider(name, os.environ, model) for name, model in arm_spec.items()} models = finder_models(BENCH_DIR / "results.jsonl") with tempfile.TemporaryDirectory(prefix="arms-") as tmp: @@ -241,6 +269,43 @@ async def collect_rows(repo_root: Path, limit: int | None) -> list[dict[str, Any UNKNOWN_VENDOR = "unknown" +#: The share of findings an arm may leave unruled before its numbers are refused. +#: +#: Not a tolerance — a validity gate, and it is the only thing standing between +#: this experiment and a wrong conclusion. `judge_finding` contains provider +#: errors per finding and returns None, so a spent quota, an upstream 429 and a +#: model that cannot produce JSON all arrive as "unruled". An arm that answered +#: nothing and an arm that refuted nothing are the same row shape, and "refutes +#: nothing" is precisely what #246 predicts for a same-vendor skeptic. Reported +#: without this check, a broken arm would confirm the hypothesis. +MAX_UNRULED_RATE = 0.05 + + +class ArmTooQuietError(RuntimeError): + """Raised when an arm left too many findings unruled to be scored. + + Refused rather than footnoted. A caveat under a table does not stop the + table being read, and this table's whole content is how much each skeptic + refuted. + """ + + +def validity_problems(rows: list[dict[str, Any]]) -> list[str]: + """Arms whose unruled share is too high to interpret, with the numbers.""" + totals: dict[str, list[int]] = defaultdict(lambda: [0, 0]) + for row in rows: + seen = totals[str(row["arm"])] + seen[0] += int(row.get("unruled") or 0) + seen[1] += int(row["findings"]) + return [ + f"arm {arm!r} left {unruled} of {findings} findings unruled " + f"({unruled / findings:.0%} > {MAX_UNRULED_RATE:.0%}); its numbers cannot be read as " + f"leniency because they may be silence." + for arm, (unruled, findings) in sorted(totals.items()) + if findings and unruled / findings > MAX_UNRULED_RATE + ] + + def _vendor(model: str) -> str: """The vendor behind a model name, or `UNKNOWN_VENDOR`. @@ -276,7 +341,7 @@ def report(rows: list[dict[str, Any]]) -> None: """The comparison the issue asks for, split by which vendor found the findings.""" print(f"\n{len(rows)} (pass, arm) results\n") print( - f"{'finder':9} {'skeptic':9} {'kind':7} {'passes':>6} {'refuted':>8} " + f"{'finder':10} {'skeptic':11} {'kind':8} {'passes':>6} {'refuted':>8} " f"{'of':>5} {'killed GT':>10}" ) groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) @@ -287,7 +352,7 @@ def report(rows: list[dict[str, Any]]) -> None: refuted = sum(int(r["refuted"]) for r in group) total = sum(int(r["findings"]) for r in group) killed = sum(len(r["killed_gt"]) for r in group) - print(f"{finder:9} {arm:9} {kind:7} {len(group):6} {refuted:8} {total:5} {killed:10}") + print(f"{finder:10} {arm:11} {kind:8} {len(group):6} {refuted:8} {total:5} {killed:10}") unattributed = sorted( {str(r["finder_model"]) for r in rows if _vendor(str(r["finder_model"])) == UNKNOWN_VENDOR} ) @@ -307,13 +372,26 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", type=Path, default=REPO_ROOT / ".guardian-arms.jsonl") parser.add_argument("--repo-root", type=Path, default=REPO_ROOT) + parser.add_argument( + "--arms", + default=DEFAULT_ARMS, + help="comma-separated arms, each optionally name=model (see DEFAULT_ARMS)", + ) parser.add_argument( "--limit", type=int, default=None, help="judge only the first N passes (a smoke run)" ) args = parser.parse_args() - rows = asyncio.run(collect_rows(args.repo_root, args.limit)) + rows = asyncio.run(collect_rows(args.repo_root, args.limit, parse_arms(args.arms))) args.out.parent.mkdir(parents=True, exist_ok=True) + # Written before the gate: the rows are evidence either way, and a refused + # run whose data was discarded cannot be diagnosed. args.out.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") + problems = validity_problems(rows) + if problems: + for problem in problems: + print(problem, file=sys.stderr) + print(f"Rows kept at {args.out}; no comparison printed.", file=sys.stderr) + return 2 report(rows) return 0 diff --git a/src/cgis/guardian/providers/base.py b/src/cgis/guardian/providers/base.py index 1752ffbd..b64cded9 100644 --- a/src/cgis/guardian/providers/base.py +++ b/src/cgis/guardian/providers/base.py @@ -86,7 +86,7 @@ def __init_subclass__(cls, **kwargs: object) -> None: return _msg = ( f"{cls.__name__} must declare `name: ClassVar[str]` — one of the " - 'GUARDIAN_PROVIDER values: "gemini", "mistral", "ollama". Add ' + 'GUARDIAN_PROVIDER values: "gemini", "mistral", "ollama", "openrouter". Add ' f'`name: ClassVar[str] = "..."` as the first line of ' f"{cls.__name__}'s class body." ) diff --git a/src/cgis/guardian/providers/openrouter.py b/src/cgis/guardian/providers/openrouter.py new file mode 100644 index 00000000..4885b6f4 --- /dev/null +++ b/src/cgis/guardian/providers/openrouter.py @@ -0,0 +1,187 @@ +"""OpenRouter provider — an OpenAI-compatible gateway to many vendors (#246). + +#246 needs a skeptic from a vendor other than the finder's, and the two the +repository already speaks to are the two it is comparing. Ollama would answer +that at no API cost, but it needs a server this project does not have. OpenRouter +reaches a third vendor over an OpenAI-shaped HTTP API with a free tier, which is +what the comparison was blocked on. + +**Reasoning models set the token budget here.** The free models measured on +2026-08-17 spend most of their completion on reasoning before emitting the +answer: 709 tokens for `nemotron-3-nano-30b`, 2,133 for `nemotron-3-super-120b`, +2,243 for `cohere/north-mini-code`. A first probe capped output at 600 and read +the truncated reasoning as "this model cannot produce JSON" — the model was fine +and the measurement was not, which is why `DEFAULT_MAX_TOKENS` is generous and +why exceeding it is reported rather than returned as a short answer. + +Free-tier quota is not discoverable in advance: `/api/v1/auth/key` returns no +limit for a free key and completions carry no rate-limit headers. Exhaustion +arrives as HTTP 429 mid-run, and a 429 that degrades to "no judgement" would +look exactly like a skeptic that refutes nothing — the very hypothesis under +test. So a 429 raises, loudly, rather than being smoothed into an empty answer. +""" + +import json +from typing import Any, ClassVar + +import httpx +from pydantic import BaseModel + +from cgis.guardian.providers.base import BaseProvider, ProviderUsage + +#: OpenAI-compatible completions endpoint. +OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" + +#: Wall-clock budget for one call. Measured latencies on the free tier span +#: 6.9s to 98.8s for a single judgement, so a timeout tuned to the fast models +#: would fail the slow ones by construction rather than by fault. +DEFAULT_REQUEST_TIMEOUT = 300.0 + +#: Completion budget for one call. Well above the 2,243 tokens the most verbose +#: model measured needed, because the cost of being wrong is asymmetric: a +#: budget that is too high wastes nothing on the free tier, and one that is too +#: low silently returns reasoning where a verdict was expected. +DEFAULT_MAX_TOKENS = 8000 + +_TOO_MANY_REQUESTS = 429 + + +class OpenRouterQuotaError(RuntimeError): + """Raised on HTTP 429 — the free tier's daily allowance is spent. + + Its own type, and never swallowed. Every other transport failure can be + retried into a judgement; this one cannot, and a run that treated it as + "the skeptic had nothing to say" would report exhausted quota as a lenient + skeptic. That is the exact shape of the result #246 is testing for, so it + must not be reachable by accident. + """ + + +class OpenRouterProvider(BaseProvider): + """A chat model reached through OpenRouter's OpenAI-compatible API.""" + + name: ClassVar[str] = "openrouter" + + def __init__( + self, + api_key: str, + model_name: str, + timeout: float = DEFAULT_REQUEST_TIMEOUT, + max_tokens: int = DEFAULT_MAX_TOKENS, + temperature: float | None = None, + ) -> None: + """Store the key, model, timeout and generation budget. + + `temperature` is None by default and then not sent at all, leaving the + model's own value in place — the rule the Mistral and Ollama providers + already follow, so an unset knob never becomes a value this project + invented and could not describe afterwards. + """ + super().__init__() + self._api_key = api_key + self._model_name = model_name + self._timeout = timeout + self._max_tokens = max_tokens + self._temperature = temperature + + async def _post(self, payload: dict[str, Any]) -> str: + """One completion call; return the message content. + + `response_format` is passed through when the caller asks for a schema. + Not every model honours it — the ones measured here comply through the + prompt rather than the parameter — so the caller still parses + defensively; sending it costs nothing and helps where it is supported. + """ + headers = {"Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json"} + async with httpx.AsyncClient(timeout=self._timeout) as client: + response = await client.post(OPENROUTER_URL, headers=headers, json=payload) + if response.status_code == _TOO_MANY_REQUESTS: + # The body is carried, not summarised. OpenRouter says whether the + # limit was per-minute on one upstream or the account's daily + # allowance, and those have different remedies — wait, or stop. An + # exception that dropped it would leave the operator guessing, the + # same defect as a CalledProcessError that swallows stderr. + _msg = ( + f"OpenRouter returned 429 for {self._model_name}: {response.text.strip()[:400]}. " + f"A run that continued would record unanswered findings, which is " + f"indistinguishable from a skeptic that refutes nothing." + ) + raise OpenRouterQuotaError(_msg) + response.raise_for_status() + body = response.json() + # An error object with HTTP 200: OpenRouter reports upstream refusals + # this way, and `.json()["choices"]` would raise KeyError with nothing + # in the message about which provider declined or why. + if "choices" not in body: + _msg = f"OpenRouter returned no choices for {self._model_name}: {body}" + raise RuntimeError(_msg) + choice = body["choices"][0] + usage = body.get("usage") or {} + self._record_usage( + ProviderUsage( + prompt_tokens=int(usage.get("prompt_tokens") or 0), + completion_tokens=int(usage.get("completion_tokens") or 0), + ) + ) + if choice.get("finish_reason") == "length": + # Reported, not returned. A model cut off mid-reasoning yields prose + # where JSON was asked for, and the caller would record that as an + # unparseable answer — attributing to the model a limit this file + # chose. Measured the hard way: a 600-token probe read three capable + # models as incapable. + _msg = ( + f"{self._model_name} hit the {self._max_tokens}-token budget before finishing. " + f"Raise max_tokens; the reply so far is reasoning, not an answer." + ) + raise RuntimeError(_msg) + return str(choice["message"].get("content") or "") + + def _payload(self, system_prompt: str, user_prompt: str) -> dict[str, Any]: + """The request body common to both generation modes.""" + payload: dict[str, Any] = { + "model": self._model_name, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + "max_tokens": self._max_tokens, + } + # `is not None`, not truthiness: 0.0 is the one temperature that states + # something, and a falsy test would drop exactly that value. + if self._temperature is not None: + payload["temperature"] = self._temperature + return payload + + async def generate_content(self, system_prompt: str, user_prompt: str) -> str: + """Free-text generation.""" + return await self._retry(lambda: self._post(self._payload(system_prompt, user_prompt))) + + async def generate_structured( + self, system_prompt: str, user_prompt: str, schema: type[BaseModel] + ) -> str: + """Generation asked to conform to `schema`; returns raw JSON text.""" + payload = self._payload(system_prompt, user_prompt) + payload["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": schema.__name__, + "strict": True, + "schema": schema.model_json_schema(), + }, + } + return await self._retry(lambda: self._post(payload)) + + +def parse_or_raise(text: str, schema: type[BaseModel]) -> BaseModel: + """Validate `text` against `schema`, tolerating a fenced code block. + + Models that answer through the prompt rather than through + `response_format` often wrap the object in ```json fences. Stripping them + here keeps that from being counted as an unparseable answer, which would + inflate exactly the "unruled" rate an experiment reads as leniency. + """ + stripped = text.strip() + if stripped.startswith("```"): + stripped = stripped.split("\n", 1)[-1] + stripped = stripped.rsplit("```", 1)[0] + return schema.model_validate(json.loads(stripped)) diff --git a/src/cgis/guardian/review_fingerprint.py b/src/cgis/guardian/review_fingerprint.py index 2506e3b0..c1b18f0c 100644 --- a/src/cgis/guardian/review_fingerprint.py +++ b/src/cgis/guardian/review_fingerprint.py @@ -149,13 +149,13 @@ def walk_closure(read: ReadFile, active_providers: frozenset[str]) -> list[str]: `active_providers` is the union of the finder's and the skeptic's provider names. Unselected providers are pruned **during** the walk, not filtered - afterwards: `runner.py` imports all three at module level, so a full + afterwards: `runner.py` imports every one of them at module level, so a full traversal minus a set would still have reached what an unselected provider imports. """ known = frozenset( name - for name in ("gemini", "mistral", "ollama") + for name in ("gemini", "mistral", "ollama", "openrouter") if _module_path(f"{_PROVIDER_PACKAGE}.{name}", read) is not None ) unknown = active_providers - known diff --git a/src/cgis/guardian/runner.py b/src/cgis/guardian/runner.py index 76856a1d..a4a9f594 100644 --- a/src/cgis/guardian/runner.py +++ b/src/cgis/guardian/runner.py @@ -23,6 +23,7 @@ DEFAULT_OLLAMA_NUM_PREDICT, OllamaProvider, ) +from cgis.guardian.providers.openrouter import OpenRouterProvider from cgis.guardian.recording import save_finder_recording from cgis.guardian.render import render_report @@ -200,6 +201,32 @@ def _build_gemini(env: Mapping[str, str], model_override: str | None) -> tuple[B return GeminiProvider(api_key=key, model_name=model), model +def _build_openrouter( + env: Mapping[str, str], model_override: str | None +) -> tuple[BaseProvider, str]: + """Construct an OpenRouterProvider; OPENROUTER_API_KEY and a model are required. + + No default model on purpose. OpenRouter fronts hundreds of models whose + availability and cost differ by orders of magnitude, and a default would + silently pick one nobody chose — the opposite of what this provider exists + for, which is naming a third vendor explicitly (#246). + """ + key = env.get("OPENROUTER_API_KEY") + if not key: + _msg = "OPENROUTER_API_KEY must be set when GUARDIAN_PROVIDER=openrouter" + raise RuntimeError(_msg) + if not model_override: + _msg = ( + "GUARDIAN_MODEL must name an OpenRouter model (e.g. " + "'nvidia/nemotron-3-super-120b-a12b:free'); there is no default." + ) + raise RuntimeError(_msg) + return ( + OpenRouterProvider(api_key=key, model_name=model_override, temperature=temperature(env)), + model_override, + ) + + def _build_ollama(env: Mapping[str, str], model_override: str | None) -> tuple[BaseProvider, str]: """Construct an OllamaProvider; GUARDIAN_MODEL names the model (no API key).""" if not model_override: @@ -257,10 +284,18 @@ def build_provider(env: Mapping[str, str]) -> tuple[BaseProvider, str]: model_override = _model_from_env(env, "GUARDIAN_MODEL") provider_name = env.get("GUARDIAN_PROVIDER", "").lower() or _autodetect_provider(env) - builders = {"mistral": _build_mistral, "gemini": _build_gemini, "ollama": _build_ollama} + builders = { + "mistral": _build_mistral, + "gemini": _build_gemini, + "ollama": _build_ollama, + "openrouter": _build_openrouter, + } builder = builders.get(provider_name) if builder is None: - _msg = f"Unknown GUARDIAN_PROVIDER={provider_name!r}. Use 'mistral', 'gemini', or 'ollama'." + _msg = ( + f"Unknown GUARDIAN_PROVIDER={provider_name!r}. " + f"Use one of: {', '.join(sorted(builders))}." + ) raise RuntimeError(_msg) return builder(env, model_override) @@ -289,7 +324,7 @@ def build_skeptic_provider( choice = env.get("GUARDIAN_SKEPTIC", "").lower() if choice == "off": return None - if choice not in ("", "gemini", "mistral", "ollama"): + if choice not in ("", "gemini", "mistral", "ollama", "openrouter"): log.warning("Unknown GUARDIAN_SKEPTIC; skeptic disabled.", value=choice) return None name = choice or ("mistral" if primary == "gemini" else "gemini") @@ -311,6 +346,20 @@ def build_skeptic_provider( num_predict=_ollama_num_predict(env), ) return provider, model + if name == "openrouter": + key = env.get("OPENROUTER_API_KEY") + if not key or not model_override: + log.warning( + "Skeptic disabled: an openrouter skeptic needs OPENROUTER_API_KEY and " + "GUARDIAN_SKEPTIC_MODEL; there is no default model." + ) + return None + return ( + OpenRouterProvider( + api_key=key, model_name=model_override, temperature=temperature(env) + ), + model_override, + ) if name == "mistral": key = env.get("MISTRAL_API_KEY") if not key: diff --git a/tests/unit/test_guardian_openrouter.py b/tests/unit/test_guardian_openrouter.py new file mode 100644 index 00000000..4910dacd --- /dev/null +++ b/tests/unit/test_guardian_openrouter.py @@ -0,0 +1,193 @@ +"""The OpenRouter provider (#246), with the network stubbed. + +Every branch here exists because failing it quietly would corrupt the +experiment the provider was added for. A spent quota, a truncated reasoning +model and an upstream refusal all produce "no verdict", and a skeptic that +produces no verdict is indistinguishable from one that refutes nothing — which +is the hypothesis #246 tests. +""" + +import json +from typing import Any + +import httpx +import pytest +from pydantic import BaseModel + +from cgis.guardian.providers import openrouter as mod +from cgis.guardian.providers.openrouter import ( + OpenRouterProvider, + OpenRouterQuotaError, + parse_or_raise, +) + + +class _Judgement(BaseModel): + verdict: str + rationale: str + + +def _reply( + content: str = '{"verdict": "refuted", "rationale": "r"}', + *, + status: int = 200, + finish: str = "stop", + usage: dict[str, int] | None = None, + body: dict[str, Any] | None = None, +) -> httpx.Response: + payload = body or { + "choices": [{"message": {"content": content}, "finish_reason": finish}], + "usage": usage or {"prompt_tokens": 11, "completion_tokens": 7}, + } + return httpx.Response(status, json=payload, request=httpx.Request("POST", mod.OPENROUTER_URL)) + + +def _serve(monkeypatch: pytest.MonkeyPatch, response: httpx.Response) -> list[dict[str, Any]]: + """Route every POST to `response`; return the list of request bodies sent.""" + sent: list[dict[str, Any]] = [] + + class _Client: + def __init__(self, **_kw: object) -> None: + pass + + async def __aenter__(self) -> "_Client": + return self + + async def __aexit__(self, *_exc: object) -> None: + return None + + async def post(self, _url: str, **kwargs: object) -> httpx.Response: + sent.append(kwargs["json"]) # type: ignore[arg-type] + return response + + monkeypatch.setattr(mod.httpx, "AsyncClient", _Client) + return sent + + +def _provider( + max_tokens: int = mod.DEFAULT_MAX_TOKENS, temperature: float | None = None +) -> OpenRouterProvider: + return OpenRouterProvider( + api_key="k", + model_name="vendor/model:free", + max_tokens=max_tokens, + temperature=temperature, + ) + + +class TestTheFailuresThatWouldLookLikeLeniency: + """Each of these returns "no verdict", and each must say why.""" + + @pytest.mark.asyncio + async def test_a_spent_quota_raises_rather_than_returning_nothing( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """429 is not an answer. The free tier gives no warning before it. + + `/api/v1/auth/key` reports no limit for a free key and completions carry + no rate-limit headers, so exhaustion is only ever discovered by hitting + it — mid-run, after some findings have been judged and some have not. + """ + _serve(monkeypatch, _reply(status=429)) + provider = _provider() + with pytest.raises(OpenRouterQuotaError, match="refutes nothing"): + await provider.generate_content("s", "u") + + @pytest.mark.asyncio + async def test_hitting_the_token_budget_raises_rather_than_returning_reasoning( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The mistake that read three capable models as incapable. + + A 600-token probe cut these reasoning models off mid-thought and + returned the reasoning as the answer, which parses as nothing. The limit + was this project's choice; attributing it to the model was the error. + """ + _serve(monkeypatch, _reply(content="We need to assess the claim", finish="length")) + provider = _provider(max_tokens=600) + with pytest.raises(RuntimeError, match="reasoning, not an answer"): + await provider.generate_content("s", "u") + + @pytest.mark.asyncio + async def test_an_error_object_with_http_200_is_reported( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """OpenRouter reports an upstream refusal this way, with a 200. + + `body["choices"]` would raise KeyError and lose the message naming which + provider declined — which is how the account's allowed-providers setting + surfaces, and it is a one-click fix nobody can apply without the text. + """ + refusal = {"error": {"message": "No allowed providers are available"}} + _serve(monkeypatch, _reply(body=refusal)) + provider = _provider() + with pytest.raises(RuntimeError, match="No allowed providers"): + await provider.generate_content("s", "u") + + +class TestTheRequest: + """What goes on the wire, and what deliberately does not.""" + + @pytest.mark.asyncio + async def test_an_unset_temperature_is_not_sent(self, monkeypatch: pytest.MonkeyPatch) -> None: + """An unset knob must not become a value this project invented.""" + sent = _serve(monkeypatch, _reply()) + await _provider().generate_content("s", "u") + assert "temperature" not in sent[0] + + @pytest.mark.asyncio + async def test_a_zero_temperature_is_sent(self, monkeypatch: pytest.MonkeyPatch) -> None: + """0.0 is the one temperature that states something, and it is falsy.""" + sent = _serve(monkeypatch, _reply()) + await _provider(temperature=0.0).generate_content("s", "u") + assert sent[0]["temperature"] == 0.0 + + @pytest.mark.asyncio + async def test_structured_generation_carries_the_schema( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + sent = _serve(monkeypatch, _reply()) + await _provider().generate_structured("s", "u", _Judgement) + fmt = sent[0]["response_format"] + assert fmt["type"] == "json_schema" + assert fmt["json_schema"]["name"] == "_Judgement" + assert "verdict" in fmt["json_schema"]["schema"]["properties"] + + @pytest.mark.asyncio + async def test_usage_is_recorded(self, monkeypatch: pytest.MonkeyPatch) -> None: + _serve(monkeypatch, _reply(usage={"prompt_tokens": 100, "completion_tokens": 20})) + provider = _provider() + await provider.generate_content("s", "u") + assert provider.last_usage.total_tokens == 120 + assert provider.cumulative_usage.total_tokens == 120 + + @pytest.mark.asyncio + async def test_a_reply_with_no_usage_is_not_a_crash( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Some upstreams omit it; losing the token count must not lose the answer.""" + _serve(monkeypatch, _reply(body={"choices": [{"message": {"content": "ok"}}]})) + assert await _provider().generate_content("s", "u") == "ok" + + +class TestFencedJson: + """Models that answer through the prompt rather than `response_format`.""" + + def test_a_bare_object_parses(self) -> None: + parsed = parse_or_raise('{"verdict": "refuted", "rationale": "r"}', _Judgement) + assert parsed.verdict == "refuted" # type: ignore[attr-defined] + + def test_a_fenced_object_parses(self) -> None: + """```json fences would otherwise count as an unparseable answer. + + That inflates the "unruled" rate, and a high unruled rate is read as a + lenient skeptic — the exact confusion this whole file guards against. + """ + text = '```json\n{"verdict": "confirmed", "rationale": "r"}\n```' + parsed = parse_or_raise(text, _Judgement) + assert parsed.verdict == "confirmed" # type: ignore[attr-defined] + + def test_prose_still_raises(self) -> None: + """Tolerating fences must not become tolerating anything.""" + with pytest.raises(json.JSONDecodeError): + parse_or_raise("I think the claim is correct.", _Judgement) diff --git a/tests/unit/test_guardian_runner.py b/tests/unit/test_guardian_runner.py index cf1f8ba9..b46edd8a 100644 --- a/tests/unit/test_guardian_runner.py +++ b/tests/unit/test_guardian_runner.py @@ -195,10 +195,19 @@ def test_build_provider_ollama_num_ctx_from_env() -> None: assert provider._num_ctx == 8192 # noqa: SLF001 # white-box: ctx wiring -def test_build_provider_unknown_lists_ollama() -> None: - """A typo in GUARDIAN_PROVIDER names all three valid providers.""" - with pytest.raises(RuntimeError, match="'mistral', 'gemini', or 'ollama'"): +def test_build_provider_unknown_lists_every_provider_there_is() -> None: + """A typo in GUARDIAN_PROVIDER names the valid ones — all of them. + + The message is built from the builder table rather than written out, so it + cannot list three providers while four exist. This test used to assert the + literal `'mistral', 'gemini', or 'ollama'` and went stale the moment + openrouter arrived (#246), which is the failure mode the derivation closes. + """ + with pytest.raises(RuntimeError) as raised: build_provider({"GUARDIAN_PROVIDER": "anthropic", "GEMINI_API_KEY": "g"}) + message = str(raised.value) + for name in ("mistral", "gemini", "ollama", "openrouter"): + assert name in message def test_build_skeptic_provider_ollama_cross_model() -> None: diff --git a/tests/unit/test_skeptic_arms.py b/tests/unit/test_skeptic_arms.py index ff20fb21..618f4ed8 100644 --- a/tests/unit/test_skeptic_arms.py +++ b/tests/unit/test_skeptic_arms.py @@ -19,7 +19,15 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "scripts")) import skeptic_arms -from skeptic_arms import ARMS, NoArmError, _vendor, arm_provider, finder_models, judge_one, report +from skeptic_arms import ( + DEFAULT_ARMS, + NoArmError, + _vendor, + arm_provider, + finder_models, + judge_one, + report, +) REPO_ROOT = Path(__file__).resolve().parent.parent.parent BENCH_DIR = REPO_ROOT / "benchmarks" / "guardian" @@ -50,9 +58,9 @@ def _capture(env: dict[str, str], **_kw: object) -> tuple[StubProvider, str]: return StubProvider([]), "m" monkeypatch.setattr(skeptic_arms, "build_skeptic_provider", _capture) - for name in ARMS: + for name in DEFAULT_ARMS.split(","): arm_provider(name, {}) - assert seen == list(ARMS) + assert seen == DEFAULT_ARMS.split(",") def test_a_skeptic_model_in_the_environment_reaches_neither_arm( self, monkeypatch: pytest.MonkeyPatch @@ -77,6 +85,38 @@ def _capture(env: dict[str, str], **_kw: object) -> tuple[StubProvider, str]: assert "GUARDIAN_SKEPTIC_MODEL" not in envs[0] assert envs[0]["X"] == "keep" + def test_an_arms_own_model_replaces_the_environments( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Per arm, because one arm may need a model the other cannot use. + + `openrouter` has no default model by design — it fronts hundreds — so it + must be told one, and telling it through the environment would tell the + other arm too. + """ + envs: list[dict[str, str]] = [] + + def _capture(env: dict[str, str], **_kw: object) -> tuple[StubProvider, str]: + envs.append(dict(env)) + return StubProvider([]), "m" + + monkeypatch.setattr(skeptic_arms, "build_skeptic_provider", _capture) + arm_provider("openrouter", {"GUARDIAN_SKEPTIC_MODEL": "gemini-2.5-flash"}, "vendor/m:free") + assert envs[0]["GUARDIAN_SKEPTIC_MODEL"] == "vendor/m:free" + + @pytest.mark.parametrize( + ("spec", "expected"), + [ + ("gemini,mistral", {"gemini": None, "mistral": None}), + ("gemini,openrouter=vendor/m:free", {"gemini": None, "openrouter": "vendor/m:free"}), + (" gemini , mistral ", {"gemini": None, "mistral": None}), + ("gemini,", {"gemini": None}), + ], + ) + def test_the_arm_spec_parses(self, spec: str, expected: dict[str, str | None]) -> None: + """A model may carry `/` and `:` — the split is on the first `=` only.""" + assert skeptic_arms.parse_arms(spec) == expected + class TestTheVendorSplit: """Both orientations are in the corpus, and that is what the split rests on.""" @@ -186,7 +226,9 @@ def _wire( path.write_text("{}", encoding="utf-8") collected: list[str] = [] monkeypatch.setattr( - skeptic_arms, "arm_provider", lambda name, _env: (StubProvider([]), f"{name}-model") + skeptic_arms, + "arm_provider", + lambda name, _env, _model=None: (StubProvider([]), f"{name}-model"), ) monkeypatch.setattr(skeptic_arms, "finder_models", lambda _p: {}) monkeypatch.setattr(skeptic_arms, "build", lambda *_a: paths) @@ -216,9 +258,11 @@ async def test_evidence_is_collected_once_per_pr_not_once_per_pass( and differs only in how long it takes — invisible in the output. """ collected = self._wire(monkeypatch, tmp_path, ["143@a", "143@b", "143@c"]) - rows = await skeptic_arms.collect_rows(REPO_ROOT, None) + rows = await skeptic_arms.collect_rows( + REPO_ROOT, None, skeptic_arms.parse_arms(DEFAULT_ARMS) + ) assert len(collected) == 1 - assert len(rows) == 3 * len(ARMS) + assert len(rows) == 3 * 2 @pytest.mark.asyncio async def test_each_pass_is_judged_by_every_arm( @@ -226,11 +270,13 @@ async def test_each_pass_is_judged_by_every_arm( ) -> None: """Pairing is the whole design: the same findings, both skeptics.""" self._wire(monkeypatch, tmp_path, ["143@a", "144@b"]) - rows = await skeptic_arms.collect_rows(REPO_ROOT, None) + rows = await skeptic_arms.collect_rows( + REPO_ROOT, None, skeptic_arms.parse_arms(DEFAULT_ARMS) + ) by_pass: dict[str, set[str]] = {} for row in rows: by_pass.setdefault(str(row["pass"]), set()).add(str(row["arm"])) - assert by_pass == {"143@a": set(ARMS), "144@b": set(ARMS)} + assert by_pass == {"143@a": {"gemini", "mistral"}, "144@b": {"gemini", "mistral"}} @pytest.mark.asyncio async def test_limit_trims_the_work_before_a_worktree_is_built( @@ -238,9 +284,9 @@ async def test_limit_trims_the_work_before_a_worktree_is_built( ) -> None: """`--limit` is the smoke run; it must cost one PR, not all of them.""" collected = self._wire(monkeypatch, tmp_path, ["143@a", "144@b"]) - rows = await skeptic_arms.collect_rows(REPO_ROOT, 1) + rows = await skeptic_arms.collect_rows(REPO_ROOT, 1, skeptic_arms.parse_arms(DEFAULT_ARMS)) assert len(collected) == 1 - assert len(rows) == len(ARMS) + assert len(rows) == 2 @pytest.mark.asyncio async def test_a_pass_with_no_findings_is_dropped_before_any_worktree( @@ -253,11 +299,66 @@ async def test_a_pass_with_no_findings_is_dropped_before_any_worktree( monkeypatch.setattr( skeptic_arms, "evidence_for_pr", lambda *_a: collected.append("x") or object() ) - rows = await skeptic_arms.collect_rows(REPO_ROOT, None) + rows = await skeptic_arms.collect_rows( + REPO_ROOT, None, skeptic_arms.parse_arms(DEFAULT_ARMS) + ) assert rows == [] assert collected == [] +class TestTheValidityGate: + """The only thing between a broken arm and a confirmed hypothesis.""" + + def test_a_quiet_arm_is_reported_as_a_problem(self) -> None: + """`judge_finding` turns every provider error into `unruled`. + + A spent quota, an upstream 429 and a model that cannot emit JSON all + arrive as the same row shape as a skeptic that declined to refute — and + "refutes nothing" is what #246 predicts for a same-vendor skeptic. So a + broken arm would *confirm* the hypothesis if this did not fire. + """ + rows: list[dict[str, Any]] = [ + {"arm": "openrouter", "findings": 10, "unruled": 4}, + {"arm": "gemini", "findings": 10, "unruled": 0}, + ] + problems = skeptic_arms.validity_problems(rows) + assert len(problems) == 1 + assert "openrouter" in problems[0] + assert "4 of 10" in problems[0] + + def test_a_healthy_pair_of_arms_raises_nothing(self) -> None: + rows: list[dict[str, Any]] = [ + {"arm": "openrouter", "findings": 100, "unruled": 2}, + {"arm": "gemini", "findings": 100, "unruled": 0}, + ] + assert skeptic_arms.validity_problems(rows) == [] + + def test_main_refuses_to_print_a_comparison_it_cannot_read( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + """Exit 2 and no table — and the rows are still written. + + Refused rather than footnoted: a caveat under a table does not stop the + table being read, and this table's whole content is how much each arm + refuted. The rows are kept because a refused run nobody can diagnose is + a second failure. + """ + + async def _rows( + _root: Path, _limit: int | None, _arms: dict[str, str | None] + ) -> list[dict[str, Any]]: + return [{"arm": "openrouter", "findings": 10, "unruled": 9, "finder_model": "x"}] + + out = tmp_path / "arms.jsonl" + monkeypatch.setattr(skeptic_arms, "collect_rows", _rows) + monkeypatch.setattr(sys, "argv", ["skeptic_arms.py", "--out", str(out)]) + assert skeptic_arms.main() == 2 + captured = capsys.readouterr() + assert "may be silence" in captured.err + assert "(pass, arm) results" not in captured.out + assert out.read_text(encoding="utf-8").strip() + + def test_main_writes_the_rows_it_collected( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: @@ -275,7 +376,9 @@ def test_main_writes_the_rows_it_collected( "killed_gt": [], } - async def _rows(_root: Path, _limit: int | None) -> list[dict[str, Any]]: + async def _rows( + _root: Path, _limit: int | None, _arms: dict[str, str | None] + ) -> list[dict[str, Any]]: return [row] out = tmp_path / "nested" / "arms.jsonl" From f83492a35f27e55e662e1e7a5abbef8f6df7805a Mon Sep 17 00:00:00 2001 From: zaebee Date: Mon, 17 Aug 2026 14:02:02 +0000 Subject: [PATCH 2/4] fix(guardian): state whether the model thinks, and say why a judgement failed (#246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two faults, found by running the arm rather than by reading about it. qwen3.7-plus puts its chain of thought in a separate reasoning field and fills content only at the end. On the larger prompts it spent the entire 8,000-token budget thinking and returned content: null — 118 of 135 findings unruled, caught by the validity gate. Raising the budget would have fixed the symptom and broken the experiment: gemini-2.5-flash, the arm it is compared against, is not doing extended thinking, so the two arms would have differed in a dimension nobody chose. The provider now sends reasoning explicitly in both directions, off by default, so a run can state what it did rather than inherit it and be unable to describe itself afterwards. The second fault cost half an hour because judge_finding logged only "Skeptic judgement failed; finding stays unruled" — 118 identical lines with no type and no message. A spent free quota, an HTTP 429, a 402 and unparseable JSON all land in that one except, and they have four different remedies. It now carries the type and the message. Containing the error per finding stays deliberate (#246 3.4); saying nothing about it was not a decision, it was an omission. What the two attempts measured, in benchmarks/guardian/experiments/ 246-cross-vendor/: the free tier allows 50 requests a day (39 answered in the run plus 10 in the day's probes, then a clean wall), and the paid account has $0 credits against $0.159 used, so 402. The comparison itself is still not run, and the gate is why no number was published for it. Co-Authored-By: Claude Opus 5 --- .../experiments/246-cross-vendor/README.md | 71 ++++++++++++++++++ .../free-nemotron-quota-cutoff.jsonl | 74 +++++++++++++++++++ src/cgis/guardian/providers/openrouter.py | 20 +++++ src/cgis/guardian/skeptic.py | 16 +++- tests/unit/test_guardian_openrouter.py | 24 +++++- 5 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 benchmarks/guardian/experiments/246-cross-vendor/README.md create mode 100644 benchmarks/guardian/experiments/246-cross-vendor/free-nemotron-quota-cutoff.jsonl diff --git a/benchmarks/guardian/experiments/246-cross-vendor/README.md b/benchmarks/guardian/experiments/246-cross-vendor/README.md new file mode 100644 index 00000000..54640525 --- /dev/null +++ b/benchmarks/guardian/experiments/246-cross-vendor/README.md @@ -0,0 +1,71 @@ +# Cross-vendor skeptic: still blocked, and now on measured limits (#246) + +Two attempts at the third arm, both stopped by the validity gate before any +number was reported. The gate is the deliverable here; the comparison is not. + +## Why a gate at all + +`judge_finding` contains provider errors per finding and returns None, so a +spent quota, an HTTP 429, a 402 and unparseable JSON all arrive as **the same +row shape as a skeptic that refused to refute anything** — which is exactly what +#246 predicts for a same-vendor skeptic. A broken arm would confirm the +hypothesis. `MAX_UNRULED_RATE` refuses to print a comparison above 5% unruled, +keeps the rows, and exits 2. + +It fired on both attempts. Neither produced a number that could be read. + +## Attempt 1 — free tier. The quota is 50 requests per day. + +`nvidia/nemotron-3-super-120b-a12b:free`, the whole corpus: **96 of 135 findings +unruled**, and the pattern is a cutoff in time rather than a property of the +input. + +| PR | evidence | unruled | +|---|---|---:| +| 122 (first) | no | **0 / 29** | +| 140 | yes | 29 / 39 | +| 141, 142, 143, 144 | — | **100%** | + +39 answered, then silence. With the day's earlier probes (4 candidate calls, 6 +in a smoke run) that is 49 successful requests before the wall — the documented +free allowance is 50/day, and this is that number arrived at from the data. + +Rows: `free-nemotron-quota-cutoff.jsonl`. + +## Attempt 2 — paid. Two separate faults, one after the other. + +`qwen/qwen3.7-plus`, chosen because capability has to match: comparing +`gemini-2.5-flash` against a small free model measures strong-versus-weak and +calls it same-versus-cross. + +**First fault — thinking.** 118 of 135 unruled. `qwen3.7-plus` puts its chain of +thought in a separate `reasoning` field and fills `content` only at the end, so +on the larger prompts it spent the entire 8,000-token budget thinking and +returned `content: null`. Raising the budget would have fixed the symptom and +broken the experiment: the arm it is compared against is not doing extended +thinking, so the two arms would differ in a dimension nobody chose. The provider +now sends `reasoning: {"enabled": …}` explicitly in both directions, off by +default, so a run can state what it did. + +**Second fault — credits.** The re-run failed on HTTP **402**: the account has +`total_credits: $0` against `total_usage: $0.159`. The trial allowance is spent; +paid models are unavailable until it is topped up. + +The second fault took half an hour to identify because the warning read only +"Skeptic judgement failed; finding stays unruled" — 118 identical lines, no +type, no message. It now carries both. A quota, a truncated reasoning model and +a 402 have three different remedies and all three land in that one `except`. + +## To unblock + +1. **Top up OpenRouter.** The measured cost of the run with reasoning off is + ~$0.15: 135 findings × ~2,700 prompt tokens at \$0.32/M, plus short + completions at \$1.28/M. A dollar covers it several times over. +2. **Free tier across three days.** The input is a frozen recording, so the run + is deterministic and can be split — 50 requests a day, 135 needed. Fragile, + and it needs resume support the tool does not have. +3. A local ollama, which this issue already names as the no-cost option. + +Everything else is in place: the corpus with both finder orientations, the +recordings, per-arm models, the scoring, and the gate that stopped two wrong +answers from being published. diff --git a/benchmarks/guardian/experiments/246-cross-vendor/free-nemotron-quota-cutoff.jsonl b/benchmarks/guardian/experiments/246-cross-vendor/free-nemotron-quota-cutoff.jsonl new file mode 100644 index 00000000..1d69cabb --- /dev/null +++ b/benchmarks/guardian/experiments/246-cross-vendor/free-nemotron-quota-cutoff.jsonl @@ -0,0 +1,74 @@ +{"timestamp": "2026-08-17T13:17:43.517549+00:00", "pr": 122, "pass": "122@2026-06-10T20-22-31.032903+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 0, "confirmed": 3, "unruled": 0, "recall": 0.18181818181818182, "precision": 0.6666666666666666, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:18:05.186110+00:00", "pr": 122, "pass": "122@2026-06-10T20-22-31.032903+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.18181818181818182, "precision": 0.6666666666666666, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:18:15.602819+00:00", "pr": 122, "pass": "122@2026-06-10T20-22-59.496508+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:18:24.136220+00:00", "pr": 122, "pass": "122@2026-06-10T20-22-59.496508+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:18:36.000053+00:00", "pr": 122, "pass": "122@2026-06-10T20-23-51.775172+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.18181818181818182, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:18:50.719370+00:00", "pr": 122, "pass": "122@2026-06-10T20-23-51.775172+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 1, "confirmed": 1, "unruled": 0, "recall": 0.18181818181818182, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:19:01.976110+00:00", "pr": 122, "pass": "122@2026-06-10T20-32-35.483908+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 3, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:19:23.184478+00:00", "pr": 122, "pass": "122@2026-06-10T20-32-35.483908+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 2, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 3, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:19:36.855990+00:00", "pr": 122, "pass": "122@2026-06-10T20-33-10.513615+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 0, "confirmed": 3, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:20:22.191259+00:00", "pr": 122, "pass": "122@2026-06-10T20-33-10.513615+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:20:35.820094+00:00", "pr": 122, "pass": "122@2026-06-10T21-58-22.314534+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:21:08.068793+00:00", "pr": 122, "pass": "122@2026-06-10T21-58-22.314534+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": false, "findings": 4, "refuted": 1, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.09090909090909091, "precision": 0.3333333333333333, "noise": 2, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:21:15.458172+00:00", "pr": 122, "pass": "122@2026-06-11T00-11-53.683357+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:21:21.049843+00:00", "pr": 122, "pass": "122@2026-06-11T00-11-53.683357+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:21:29.468118+00:00", "pr": 122, "pass": "122@2026-06-11T00-12-40.927704+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:21:35.271913+00:00", "pr": 122, "pass": "122@2026-06-11T00-12-40.927704+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:21:46.009233+00:00", "pr": 122, "pass": "122@2026-06-11T00-14-36.809184+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.18181818181818182, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:21:58.953594+00:00", "pr": 122, "pass": "122@2026-06-11T00-14-36.809184+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.18181818181818182, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:22:06.863138+00:00", "pr": 122, "pass": "122@2026-06-11T00-32-51.894263+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:22:18.621710+00:00", "pr": 122, "pass": "122@2026-06-11T00-32-51.894263+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:22:34.978570+00:00", "pr": 122, "pass": "122@2026-06-11T00-36-53.749615+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 1, "uncertain": 0, "confirmed": 0, "unruled": 0, "recall": 0.0, "precision": 1.0, "noise": 0, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:23:29.360852+00:00", "pr": 122, "pass": "122@2026-06-11T00-36-53.749615+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:23:35.269160+00:00", "pr": 122, "pass": "122@2026-06-11T00-42-55.162925+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:23:43.563281+00:00", "pr": 122, "pass": "122@2026-06-11T00-42-55.162925+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:23:53.848653+00:00", "pr": 122, "pass": "122@2026-06-11T00-48-45.262981+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 2, "refuted": 1, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.09090909090909091, "precision": 1.0, "noise": 0, "killed_gt": ["file-container-regression"]} +{"timestamp": "2026-08-17T13:24:05.969279+00:00", "pr": 122, "pass": "122@2026-06-11T00-48-45.262981+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.18181818181818182, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:24:25.419473+00:00", "pr": 140, "pass": "140@2026-06-10T20-24-40.504167+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 3, "refuted": 0, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 3, "killed_gt": []} +{"timestamp": "2026-08-17T13:25:25.250544+00:00", "pr": 140, "pass": "140@2026-06-10T20-24-40.504167+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 3, "refuted": 0, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 3, "killed_gt": []} +{"timestamp": "2026-08-17T13:25:47.211803+00:00", "pr": 140, "pass": "140@2026-06-10T20-25-19.261202+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 5, "refuted": 2, "uncertain": 1, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 3, "killed_gt": []} +{"timestamp": "2026-08-17T13:26:24.777165+00:00", "pr": 140, "pass": "140@2026-06-10T20-25-19.261202+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 5, "refuted": 0, "uncertain": 3, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 5, "killed_gt": []} +{"timestamp": "2026-08-17T13:26:33.938777+00:00", "pr": 140, "pass": "140@2026-06-10T20-26-01.543236+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:08.103278+00:00", "pr": 140, "pass": "140@2026-06-10T20-26-01.543236+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:17.614733+00:00", "pr": 140, "pass": "140@2026-06-11T00-16-26.605409+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:17.769243+00:00", "pr": 140, "pass": "140@2026-06-11T00-16-26.605409+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:27.876729+00:00", "pr": 140, "pass": "140@2026-06-11T00-17-25.717492+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 3, "refuted": 0, "uncertain": 0, "confirmed": 3, "unruled": 0, "recall": 0.06666666666666667, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:28.068613+00:00", "pr": 140, "pass": "140@2026-06-11T00-17-25.717492+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 3, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 3, "recall": 0.06666666666666667, "precision": 0.3333333333333333, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:38.870523+00:00", "pr": 140, "pass": "140@2026-06-11T00-33-44.053747+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.06666666666666667, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:39.001629+00:00", "pr": 140, "pass": "140@2026-06-11T00-33-44.053747+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.06666666666666667, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:45.424281+00:00", "pr": 140, "pass": "140@2026-06-11T00-37-50.526658+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:45.544269+00:00", "pr": 140, "pass": "140@2026-06-11T00-37-50.526658+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:53.801567+00:00", "pr": 140, "pass": "140@2026-06-11T00-50-08.134629+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 1, "confirmed": 0, "unruled": 0, "recall": 0.06666666666666667, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:27:53.921093+00:00", "pr": 140, "pass": "140@2026-06-11T00-50-08.134629+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.06666666666666667, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:06.369511+00:00", "pr": 140, "pass": "140@2026-07-29T17-29-19.861365+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 22, "refuted": 10, "uncertain": 2, "confirmed": 10, "unruled": 0, "recall": 0.26666666666666666, "precision": 0.3333333333333333, "noise": 8, "killed_gt": ["count-routers-on-squared", "weights-keyerror-custom-yaml"]} +{"timestamp": "2026-08-17T13:29:07.917533+00:00", "pr": 140, "pass": "140@2026-07-29T17-29-19.861365+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 22, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 22, "recall": 0.4, "precision": 0.2727272727272727, "noise": 16, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:21.415626+00:00", "pr": 141, "pass": "141@2026-06-10T20-26-53.683221+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 1.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:21.527539+00:00", "pr": 141, "pass": "141@2026-06-10T20-26-53.683221+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 1.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:28.243544+00:00", "pr": 141, "pass": "141@2026-06-10T20-29-48.066882+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 1, "confirmed": 0, "unruled": 0, "recall": 1.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:28.360785+00:00", "pr": 141, "pass": "141@2026-06-10T20-29-48.066882+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 1.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:37.370316+00:00", "pr": 141, "pass": "141@2026-06-10T21-59-24.111065+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 1.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:37.495452+00:00", "pr": 141, "pass": "141@2026-06-10T21-59-24.111065+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 1.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:49.082379+00:00", "pr": 142, "pass": "142@2026-06-10T20-30-15.605690+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.5, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:49.243691+00:00", "pr": 142, "pass": "142@2026-06-10T20-30-15.605690+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.5, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:56.956875+00:00", "pr": 142, "pass": "142@2026-06-10T21-52-43.992582+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:29:57.130875+00:00", "pr": 142, "pass": "142@2026-06-10T21-52-43.992582+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 2, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:04.860920+00:00", "pr": 142, "pass": "142@2026-06-10T21-53-15.240002+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:05.012683+00:00", "pr": 142, "pass": "142@2026-06-10T21-53-15.240002+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": false, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:14.805023+00:00", "pr": 142, "pass": "142@2026-06-11T00-45-11.791674+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 3, "confirmed": 0, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:14.969320+00:00", "pr": 142, "pass": "142@2026-06-11T00-45-11.791674+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 3, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:23.741156+00:00", "pr": 142, "pass": "142@2026-06-11T00-50-52.772949+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 3, "refuted": 2, "uncertain": 1, "confirmed": 0, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:23.921669+00:00", "pr": 142, "pass": "142@2026-06-11T00-50-52.772949+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": false, "findings": 3, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 3, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:43.638312+00:00", "pr": 143, "pass": "143@2026-06-11T00-46-29.982484+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:30:43.909808+00:00", "pr": 143, "pass": "143@2026-06-11T00-46-29.982484+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:21.206192+00:00", "pr": 143, "pass": "143@2026-07-29T17-15-00.138863+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 28, "refuted": 8, "uncertain": 1, "confirmed": 19, "unruled": 0, "recall": 1.0, "precision": 0.3, "noise": 14, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:22.809114+00:00", "pr": 143, "pass": "143@2026-07-29T17-15-00.138863+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 28, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 28, "recall": 1.0, "precision": 0.21428571428571427, "noise": 22, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:44.357216+00:00", "pr": 144, "pass": "144@2026-06-10T20-32-21.181637+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 2, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:44.503817+00:00", "pr": 144, "pass": "144@2026-06-10T20-32-21.181637+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-2.5-flash", "evidence": true, "findings": 2, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 2, "recall": 0.0, "precision": 0.0, "noise": 2, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:58.741622+00:00", "pr": 144, "pass": "144@2026-06-11T00-25-01.536057+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 1, "uncertain": 0, "confirmed": 0, "unruled": 0, "recall": 0.0, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:32:58.913488+00:00", "pr": 144, "pass": "144@2026-06-11T00-25-01.536057+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:33:06.869381+00:00", "pr": 144, "pass": "144@2026-06-11T00-47-31.514757+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 1, "uncertain": 0, "confirmed": 0, "unruled": 0, "recall": 0.0, "precision": 1.0, "noise": 0, "killed_gt": []} +{"timestamp": "2026-08-17T13:33:06.990972+00:00", "pr": 144, "pass": "144@2026-06-11T00-47-31.514757+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:33:14.521992+00:00", "pr": 144, "pass": "144@2026-06-11T00-52-31.320750+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 1, "unruled": 0, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:33:14.623054+00:00", "pr": 144, "pass": "144@2026-06-11T00-52-31.320750+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "gemini-3.5-flash", "evidence": true, "findings": 1, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 1, "recall": 0.0, "precision": 0.0, "noise": 1, "killed_gt": []} +{"timestamp": "2026-08-17T13:34:10.907748+00:00", "pr": 144, "pass": "144@2026-07-29T17-28-33.271744+00-00", "arm": "gemini", "skeptic_model": "gemini-2.5-flash", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 20, "refuted": 7, "uncertain": 0, "confirmed": 13, "unruled": 0, "recall": 0.6, "precision": 0.23076923076923078, "noise": 5, "killed_gt": []} +{"timestamp": "2026-08-17T13:34:12.013311+00:00", "pr": 144, "pass": "144@2026-07-29T17-28-33.271744+00-00", "arm": "openrouter", "skeptic_model": "nvidia/nemotron-3-super-120b-a12b:free", "finder_model": "mistral-medium-latest", "evidence": true, "findings": 20, "refuted": 0, "uncertain": 0, "confirmed": 0, "unruled": 20, "recall": 0.6, "precision": 0.15, "noise": 8, "killed_gt": []} diff --git a/src/cgis/guardian/providers/openrouter.py b/src/cgis/guardian/providers/openrouter.py index 4885b6f4..3703da91 100644 --- a/src/cgis/guardian/providers/openrouter.py +++ b/src/cgis/guardian/providers/openrouter.py @@ -43,6 +43,22 @@ #: low silently returns reasoning where a verdict was expected. DEFAULT_MAX_TOKENS = 8000 +#: Whether the model is asked to think before answering. +#: +#: Off, and stated rather than left to the model. `qwen3.7-plus` puts its chain +#: of thought in a separate `reasoning` field and fills `content` only at the +#: end, so on the larger prompts it spent the whole budget thinking and returned +#: `content: null` — 118 of 135 findings unruled in the first paid run, which the +#: validity gate caught and refused to score. +#: +#: Raising the budget would have fixed the symptom and broken the experiment. +#: The arm this one is compared against, `gemini-2.5-flash`, is not doing +#: extended thinking, so leaving the model's default on would make the two arms +#: differ in a dimension nobody chose — the same reason `temperature` is not +#: invented here, arriving from the other side: the honest move is to set it and +#: say so, not to inherit it and be unable to describe the run afterwards. +DEFAULT_REASONING = False + _TOO_MANY_REQUESTS = 429 @@ -69,6 +85,7 @@ def __init__( timeout: float = DEFAULT_REQUEST_TIMEOUT, max_tokens: int = DEFAULT_MAX_TOKENS, temperature: float | None = None, + reasoning: bool = DEFAULT_REASONING, ) -> None: """Store the key, model, timeout and generation budget. @@ -83,6 +100,7 @@ def __init__( self._timeout = timeout self._max_tokens = max_tokens self._temperature = temperature + self._reasoning = reasoning async def _post(self, payload: dict[str, Any]) -> str: """One completion call; return the message content. @@ -145,6 +163,8 @@ def _payload(self, system_prompt: str, user_prompt: str) -> dict[str, Any]: {"role": "user", "content": user_prompt}, ], "max_tokens": self._max_tokens, + # Always sent, in both directions, so a run can state what it did. + "reasoning": {"enabled": self._reasoning}, } # `is not None`, not truthiness: 0.0 is the one temperature that states # something, and a falsy test would drop exactly that value. diff --git a/src/cgis/guardian/skeptic.py b/src/cgis/guardian/skeptic.py index c4fd75fb..4e034f9f 100644 --- a/src/cgis/guardian/skeptic.py +++ b/src/cgis/guardian/skeptic.py @@ -212,8 +212,20 @@ async def judge_finding( FindingJudgement, ) return FindingJudgement.model_validate_json(extract_json(raw)) - except Exception: - log.warning("Skeptic judgement failed; finding stays unruled.", file=finding.file) + except Exception as exc: + # The reason is logged, not just the fact. Containing the error per + # finding is deliberate (#246 §3.4) — one bad response must not cost the + # pass — but a warning that says only "failed" leaves an operator with + # 118 identical lines and no way to tell a spent quota from a truncated + # reasoning model from unparseable JSON. Each of those has a different + # fix, and all three arrive here. Diagnosing one cost half an hour of + # guessing before this line carried the type and the message. + log.warning( + "Skeptic judgement failed; finding stays unruled.", + file=finding.file, + error_type=type(exc).__name__, + error=str(exc)[:300], + ) return None diff --git a/tests/unit/test_guardian_openrouter.py b/tests/unit/test_guardian_openrouter.py index 4910dacd..f2cbb73e 100644 --- a/tests/unit/test_guardian_openrouter.py +++ b/tests/unit/test_guardian_openrouter.py @@ -65,13 +65,16 @@ async def post(self, _url: str, **kwargs: object) -> httpx.Response: def _provider( - max_tokens: int = mod.DEFAULT_MAX_TOKENS, temperature: float | None = None + max_tokens: int = mod.DEFAULT_MAX_TOKENS, + temperature: float | None = None, + reasoning: bool = mod.DEFAULT_REASONING, ) -> OpenRouterProvider: return OpenRouterProvider( api_key="k", model_name="vendor/model:free", max_tokens=max_tokens, temperature=temperature, + reasoning=reasoning, ) @@ -142,6 +145,25 @@ async def test_a_zero_temperature_is_sent(self, monkeypatch: pytest.MonkeyPatch) await _provider(temperature=0.0).generate_content("s", "u") assert sent[0]["temperature"] == 0.0 + @pytest.mark.asyncio + async def test_reasoning_is_stated_in_both_directions( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Sent whether on or off, so a run can describe what it did. + + Left unsent, `qwen3.7-plus` thinks by default, puts the thought in a + separate field and fills `content` only at the end — it spent the whole + budget reasoning and returned nothing on 118 of 135 findings. The arm it + is compared against is not thinking, so inheriting the default would + make the two arms differ in a dimension nobody chose. + """ + sent = _serve(monkeypatch, _reply()) + await _provider().generate_content("s", "u") + assert sent[0]["reasoning"] == {"enabled": False} + sent_on = _serve(monkeypatch, _reply()) + await _provider(reasoning=True).generate_content("s", "u") + assert sent_on[0]["reasoning"] == {"enabled": True} + @pytest.mark.asyncio async def test_structured_generation_carries_the_schema( self, monkeypatch: pytest.MonkeyPatch From 865c4ccd58bdbddca0377d391a6a9a40d77f1fb5 Mon Sep 17 00:00:00 2001 From: zaebee Date: Mon, 17 Aug 2026 17:38:12 +0000 Subject: [PATCH 3/4] fix(guardian): take gemini's two and split the skeptic builder (#246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An empty choices list was an IndexError. "choices" not in body passes when the key is present and holds [], which is what an upstream content filter returns, and body["choices"][0] then raised IndexError — an exception whose message says nothing about which provider declined. Now falsy-checked. A single-line code fence was unparseable. Splitting on a newline returned the whole string when there was none, so the backticks survived and json.loads failed — counting a good answer as unruled, which is the reading this whole file exists to prevent. Regexes now match the fence and its optional language tag. The suggested fix used lstrip with a character class; lstrip("json") removes any of j, o, s, n and would eat into the payload, so the shape of the fix is different from the one proposed while the finding is the same. SonarCloud separately put build_skeptic_provider at cognitive complexity 18 against a limit of 15, which openrouter pushed it over. Split into one builder per provider behind a table, matching build_provider. Each arm's requirements — which key, which default model, whether one exists at all — are per-provider facts and read better beside that provider than interleaved with the others. Coverage on the new provider is 100%; the openrouter paths through build_provider and build_skeptic_provider now have tests of their own rather than being exercised only through the experiment script. Co-Authored-By: Claude Opus 5 --- src/cgis/guardian/providers/openrouter.py | 27 +++- src/cgis/guardian/runner.py | 142 ++++++++++++++-------- tests/unit/test_guardian_openrouter.py | 31 +++++ tests/unit/test_guardian_runner.py | 58 +++++++++ 4 files changed, 198 insertions(+), 60 deletions(-) diff --git a/src/cgis/guardian/providers/openrouter.py b/src/cgis/guardian/providers/openrouter.py index 3703da91..93eda248 100644 --- a/src/cgis/guardian/providers/openrouter.py +++ b/src/cgis/guardian/providers/openrouter.py @@ -22,6 +22,7 @@ """ import json +import re from typing import Any, ClassVar import httpx @@ -130,7 +131,12 @@ async def _post(self, payload: dict[str, Any]) -> str: # An error object with HTTP 200: OpenRouter reports upstream refusals # this way, and `.json()["choices"]` would raise KeyError with nothing # in the message about which provider declined or why. - if "choices" not in body: + # `not body.get("choices")` covers both shapes: the key absent, and the + # key present holding an empty list. An upstream content filter returns + # the second, and `body["choices"][0]` would then raise IndexError with + # nothing in the message about which provider declined. Raised in review + # of #412. + if not body.get("choices"): _msg = f"OpenRouter returned no choices for {self._model_name}: {body}" raise RuntimeError(_msg) choice = body["choices"][0] @@ -192,6 +198,18 @@ async def generate_structured( return await self._retry(lambda: self._post(payload)) +#: An opening code fence with an optional language tag, and a closing one. +#: +#: Regexes rather than splitting on a newline: a single-line fence +#: (```json {"verdict": "confirmed"}```) has none, so the split returned the +#: whole string and left the backticks in place. Raised in review of #412. +#: Character-class stripping was the suggested fix and is a trap — +#: `lstrip("json")` removes any of `j`, `o`, `s`, `n` and would eat into the +#: payload; these match the fence and nothing else. +_FENCE_OPEN = re.compile(r"^```[A-Za-z0-9_+-]*\s*") +_FENCE_CLOSE = re.compile(r"\s*```$") + + def parse_or_raise(text: str, schema: type[BaseModel]) -> BaseModel: """Validate `text` against `schema`, tolerating a fenced code block. @@ -200,8 +218,5 @@ def parse_or_raise(text: str, schema: type[BaseModel]) -> BaseModel: here keeps that from being counted as an unparseable answer, which would inflate exactly the "unruled" rate an experiment reads as leniency. """ - stripped = text.strip() - if stripped.startswith("```"): - stripped = stripped.split("\n", 1)[-1] - stripped = stripped.rsplit("```", 1)[0] - return schema.model_validate(json.loads(stripped)) + stripped = _FENCE_OPEN.sub("", text.strip(), count=1) + return schema.model_validate(json.loads(_FENCE_CLOSE.sub("", stripped, count=1).strip())) diff --git a/src/cgis/guardian/runner.py b/src/cgis/guardian/runner.py index a4a9f594..8d5456f7 100644 --- a/src/cgis/guardian/runner.py +++ b/src/cgis/guardian/runner.py @@ -3,7 +3,7 @@ import asyncio import math import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from pathlib import Path from typing import Literal @@ -310,72 +310,106 @@ def _autodetect_provider(env: Mapping[str, str]) -> str: raise RuntimeError(_msg) +def _skeptic_ollama( + env: Mapping[str, str], model_override: str | None +) -> tuple[BaseProvider, str] | None: + """A local ollama skeptic; the model must be named, there is no default.""" + model = model_override or _model_from_env(env, "GUARDIAN_MODEL") + if not model: + log.warning( + "Skeptic disabled: set GUARDIAN_SKEPTIC_MODEL (or GUARDIAN_MODEL) " + "for an ollama skeptic." + ) + return None + provider = OllamaProvider( + model_name=model, + host=_ollama_host(env), + num_ctx=_ollama_num_ctx(env), + temperature=temperature(env), + penalties=_ollama_penalties(env), + num_predict=_ollama_num_predict(env), + ) + return provider, model + + +def _skeptic_openrouter( + env: Mapping[str, str], model_override: str | None +) -> tuple[BaseProvider, str] | None: + """An OpenRouter skeptic; both the key and an explicit model are required.""" + key = env.get("OPENROUTER_API_KEY") + if not key or not model_override: + log.warning( + "Skeptic disabled: an openrouter skeptic needs OPENROUTER_API_KEY and " + "GUARDIAN_SKEPTIC_MODEL; there is no default model." + ) + return None + provider = OpenRouterProvider( + api_key=key, model_name=model_override, temperature=temperature(env) + ) + return provider, model_override + + +def _skeptic_mistral( + env: Mapping[str, str], model_override: str | None +) -> tuple[BaseProvider, str] | None: + """A mistral skeptic, on the finder's temperature.""" + key = env.get("MISTRAL_API_KEY") + if not key: + log.warning("Skeptic disabled: MISTRAL_API_KEY not set.") + return None + model = model_override or DEFAULT_MISTRAL_MODEL + # Same temperature as the finder: the run registers one sampling setting, + # not one per pass. + return MistralProvider(api_key=key, model_name=model, temperature=temperature(env)), model + + +def _skeptic_gemini( + env: Mapping[str, str], model_override: str | None +) -> tuple[BaseProvider, str] | None: + """A gemini skeptic.""" + key = env.get("GEMINI_API_KEY") + if not key: + log.warning("Skeptic disabled: GEMINI_API_KEY not set.") + return None + model = model_override or DEFAULT_GEMINI_MODEL + return GeminiProvider(api_key=key, model_name=model), model + + +#: One builder per skeptic provider. A table rather than a chain of `if name ==` +#: blocks: the chain reached cognitive complexity 18 against a limit of 15 when +#: openrouter joined it (SonarCloud on #412), and each arm's requirements — which +#: key, which default model, whether one exists at all — are per-provider facts +#: that belong beside that provider rather than interleaved with the others. +_SKEPTIC_BUILDERS: dict[ + str, Callable[[Mapping[str, str], str | None], tuple[BaseProvider, str] | None] +] = { + "ollama": _skeptic_ollama, + "openrouter": _skeptic_openrouter, + "mistral": _skeptic_mistral, + "gemini": _skeptic_gemini, +} + + def build_skeptic_provider( env: Mapping[str, str], *, primary: str ) -> tuple[BaseProvider, str] | None: """Return (skeptic_provider, model) or None for single-pass (spec §5.5). Default skeptic = the provider opposite to the primary; GUARDIAN_SKEPTIC - overrides ('gemini'|'mistral'|'ollama'|'off'); GUARDIAN_SKEPTIC_MODEL overrides - the model, enabling same-provider/different-model pairs (incl. two distinct - local Ollama models — a cross-model skeptic with no API cost). A missing API - key / model degrades to None — a review never fails because of the skeptic. + overrides ('gemini'|'mistral'|'ollama'|'openrouter'|'off'); + GUARDIAN_SKEPTIC_MODEL overrides the model, enabling + same-provider/different-model pairs (incl. two distinct local Ollama models — + a cross-model skeptic with no API cost). A missing API key / model degrades + to None — a review never fails because of the skeptic. """ choice = env.get("GUARDIAN_SKEPTIC", "").lower() if choice == "off": return None - if choice not in ("", "gemini", "mistral", "ollama", "openrouter"): + if choice and choice not in _SKEPTIC_BUILDERS: log.warning("Unknown GUARDIAN_SKEPTIC; skeptic disabled.", value=choice) return None name = choice or ("mistral" if primary == "gemini" else "gemini") - model_override = _model_from_env(env, "GUARDIAN_SKEPTIC_MODEL") - if name == "ollama": - model = model_override or _model_from_env(env, "GUARDIAN_MODEL") - if not model: - log.warning( - "Skeptic disabled: set GUARDIAN_SKEPTIC_MODEL (or GUARDIAN_MODEL) " - "for an ollama skeptic." - ) - return None - provider = OllamaProvider( - model_name=model, - host=_ollama_host(env), - num_ctx=_ollama_num_ctx(env), - temperature=temperature(env), - penalties=_ollama_penalties(env), - num_predict=_ollama_num_predict(env), - ) - return provider, model - if name == "openrouter": - key = env.get("OPENROUTER_API_KEY") - if not key or not model_override: - log.warning( - "Skeptic disabled: an openrouter skeptic needs OPENROUTER_API_KEY and " - "GUARDIAN_SKEPTIC_MODEL; there is no default model." - ) - return None - return ( - OpenRouterProvider( - api_key=key, model_name=model_override, temperature=temperature(env) - ), - model_override, - ) - if name == "mistral": - key = env.get("MISTRAL_API_KEY") - if not key: - log.warning("Skeptic disabled: MISTRAL_API_KEY not set.") - return None - model = model_override or DEFAULT_MISTRAL_MODEL - # Same temperature as the finder: the run registers one sampling - # setting, not one per pass. - skeptic = MistralProvider(api_key=key, model_name=model, temperature=temperature(env)) - return skeptic, model - key = env.get("GEMINI_API_KEY") - if not key: - log.warning("Skeptic disabled: GEMINI_API_KEY not set.") - return None - model = model_override or DEFAULT_GEMINI_MODEL - return GeminiProvider(api_key=key, model_name=model), model + return _SKEPTIC_BUILDERS[name](env, _model_from_env(env, "GUARDIAN_SKEPTIC_MODEL")) def build_footer( diff --git a/tests/unit/test_guardian_openrouter.py b/tests/unit/test_guardian_openrouter.py index f2cbb73e..0c2f802a 100644 --- a/tests/unit/test_guardian_openrouter.py +++ b/tests/unit/test_guardian_openrouter.py @@ -127,6 +127,21 @@ async def test_an_error_object_with_http_200_is_reported( with pytest.raises(RuntimeError, match="No allowed providers"): await provider.generate_content("s", "u") + @pytest.mark.asyncio + async def test_an_empty_choices_list_is_reported_not_an_indexerror( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The key present and the list empty — an upstream content filter. + + Raised in review of #412. `"choices" not in body` passed this and + `body["choices"][0]` then raised IndexError, whose message says nothing + about which provider declined or why. + """ + _serve(monkeypatch, _reply(body={"choices": [], "usage": {}})) + provider = _provider() + with pytest.raises(RuntimeError, match="no choices"): + await provider.generate_content("s", "u") + class TestTheRequest: """What goes on the wire, and what deliberately does not.""" @@ -209,6 +224,22 @@ def test_a_fenced_object_parses(self) -> None: parsed = parse_or_raise(text, _Judgement) assert parsed.verdict == "confirmed" # type: ignore[attr-defined] + def test_a_single_line_fence_parses(self) -> None: + """No newline to split on — the shape the first version left broken. + + Raised in review of #412. Splitting on `\\n` returned the whole string + when there was none, so the backticks survived and `json.loads` failed, + counting a perfectly good answer as unruled. + """ + text = '```json {"verdict": "refuted", "rationale": "r"}```' + parsed = parse_or_raise(text, _Judgement) + assert parsed.verdict == "refuted" # type: ignore[attr-defined] + + def test_a_fence_with_no_language_tag_parses(self) -> None: + text = '```\n{"verdict": "uncertain", "rationale": "r"}```' + parsed = parse_or_raise(text, _Judgement) + assert parsed.verdict == "uncertain" # type: ignore[attr-defined] + def test_prose_still_raises(self) -> None: """Tolerating fences must not become tolerating anything.""" with pytest.raises(json.JSONDecodeError): diff --git a/tests/unit/test_guardian_runner.py b/tests/unit/test_guardian_runner.py index b46edd8a..a885df56 100644 --- a/tests/unit/test_guardian_runner.py +++ b/tests/unit/test_guardian_runner.py @@ -766,3 +766,61 @@ def test_zero_and_a_normal_value_still_pass(self) -> None: """The guard must not cost the settings it exists to protect.""" assert runner.temperature_setting({"GUARDIAN_TEMPERATURE": "0"}) == (0.0, "explicit") assert runner.temperature_setting({"GUARDIAN_TEMPERATURE": "0.7"}) == (0.7, "explicit") + + +def test_build_provider_openrouter_requires_an_explicit_model() -> None: + """No default model, on purpose: OpenRouter fronts hundreds of them. + + A default would silently pick one nobody chose, which is the opposite of + what this provider is for — naming a third vendor explicitly (#246). + """ + with pytest.raises(RuntimeError, match="GUARDIAN_MODEL must name an OpenRouter model"): + build_provider({"GUARDIAN_PROVIDER": "openrouter", "OPENROUTER_API_KEY": "k"}) + + +def test_build_provider_openrouter_requires_a_key() -> None: + env = {"GUARDIAN_PROVIDER": "openrouter", "GUARDIAN_MODEL": "vendor/m:free"} + with pytest.raises(RuntimeError, match="OPENROUTER_API_KEY must be set"): + build_provider(env) + + +def test_build_provider_openrouter_builds() -> None: + provider, model = build_provider( + { + "GUARDIAN_PROVIDER": "openrouter", + "OPENROUTER_API_KEY": "k", + "GUARDIAN_MODEL": "vendor/m:free", + } + ) + assert provider.name == "openrouter" + assert model == "vendor/m:free" + + +def test_build_skeptic_openrouter_needs_both_key_and_model() -> None: + """Degrades to None rather than raising: a review never fails on the skeptic.""" + assert ( + build_skeptic_provider( + {"GUARDIAN_SKEPTIC": "openrouter", "OPENROUTER_API_KEY": "k"}, primary="mistral" + ) + is None + ) + assert ( + build_skeptic_provider( + {"GUARDIAN_SKEPTIC": "openrouter", "GUARDIAN_SKEPTIC_MODEL": "v/m"}, primary="mistral" + ) + is None + ) + + +def test_build_skeptic_openrouter_builds() -> None: + built = build_skeptic_provider( + { + "GUARDIAN_SKEPTIC": "openrouter", + "OPENROUTER_API_KEY": "k", + "GUARDIAN_SKEPTIC_MODEL": "vendor/m:free", + }, + primary="mistral", + ) + assert built is not None + assert built[0].name == "openrouter" + assert built[1] == "vendor/m:free" From 92ca839eacd12bf7ea7327bf1b0db7b16fd119ad Mon Sep 17 00:00:00 2001 From: zaebee Date: Mon, 17 Aug 2026 17:42:32 +0000 Subject: [PATCH 4/4] fix(guardian): the closing fence needs no regex (python:S8786) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SonarCloud on #412: an unanchored `\s*```$` is quadratic on a long run of whitespace that never reaches a fence, because every start position retries the \s*. Model output is not input this module chooses, so unbounded cost on hostile shapes is a real property rather than a theoretical one — 200k spaces now parse in under a millisecond. str.endswith answers the same question in one comparison. The opening fence keeps its regex: anchored at ^, it is tried from a single position and cannot backtrack across the input. Co-Authored-By: Claude Opus 5 --- src/cgis/guardian/providers/openrouter.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/cgis/guardian/providers/openrouter.py b/src/cgis/guardian/providers/openrouter.py index 93eda248..122c0e46 100644 --- a/src/cgis/guardian/providers/openrouter.py +++ b/src/cgis/guardian/providers/openrouter.py @@ -200,14 +200,23 @@ async def generate_structured( #: An opening code fence with an optional language tag, and a closing one. #: -#: Regexes rather than splitting on a newline: a single-line fence +#: The opening code fence and its optional language tag. +#: +#: A regex rather than splitting on a newline: a single-line fence #: (```json {"verdict": "confirmed"}```) has none, so the split returned the #: whole string and left the backticks in place. Raised in review of #412. #: Character-class stripping was the suggested fix and is a trap — #: `lstrip("json")` removes any of `j`, `o`, `s`, `n` and would eat into the -#: payload; these match the fence and nothing else. +#: payload; this matches the fence and nothing else. +#: +#: Anchored at `^`, so it is tried from one position and its `\s*` cannot +#: backtrack across the input. The closing fence gets no regex at all: an +#: unanchored `\s*```$` is quadratic on a long run of whitespace that never +#: reaches a fence (SonarCloud python:S8786 on #412), and model output is not +#: input this module chooses. `str.endswith` answers the same question in one +#: comparison. +FENCE = "```" _FENCE_OPEN = re.compile(r"^```[A-Za-z0-9_+-]*\s*") -_FENCE_CLOSE = re.compile(r"\s*```$") def parse_or_raise(text: str, schema: type[BaseModel]) -> BaseModel: @@ -218,5 +227,7 @@ def parse_or_raise(text: str, schema: type[BaseModel]) -> BaseModel: here keeps that from being counted as an unparseable answer, which would inflate exactly the "unruled" rate an experiment reads as leniency. """ - stripped = _FENCE_OPEN.sub("", text.strip(), count=1) - return schema.model_validate(json.loads(_FENCE_CLOSE.sub("", stripped, count=1).strip())) + stripped = _FENCE_OPEN.sub("", text.strip(), count=1).strip() + if stripped.endswith(FENCE): + stripped = stripped[: -len(FENCE)].strip() + return schema.model_validate(json.loads(stripped))