feat(scorecard): sweep every model config into one committed table (G4)

MODELS.md

@@ -1,4 +1,16 @@
-# Model-independence proof (PLAN D3)
+# Model independence — notes and history
+
+**The live scorecard is [`benchmarks/RESULTS.md`](benchmarks/RESULTS.md)**, regenerated by
+`vpcopilot bench <repo> --all-configs` across every `config/agents*.yaml`. Numbers live there so
+there is one place to update; this file keeps the qualitative findings — what swapping a model
+actually surfaced — which no table captures.
+
+See also the per-target comparisons in [`benchmarks/`](benchmarks/), e.g.
+[`compare-vampi-three-way.md`](benchmarks/compare-vampi-three-way.md).
+
+---
+
+## The original proof (PLAN D3, 2026-07-02) — historical
 
 Same code, same prompts, same answer key — only `config/agents.yaml` changed. Both providers
 ran the full pipeline (discover → verify → triage → generate → remediate) with valid

ROADMAP.md

@@ -173,6 +173,20 @@ Live validation on the LB stays. This runs before it, not instead of it.
     row per `config/agents*.yaml`, scoring discovery, verify precision and recall, triage
     accuracy, bonus finds, noise, and wall time; re-running with the same seed and model
     reproduces the score; `MODELS.md` links the table rather than restating numbers.
+  - **BUILT 2026-07-27, not yet run.** `scorecard.py` + `vpcopilot bench <repo> --all-configs`
+    sweep every `config/agents*.yaml` into `out-<tag>` and write `benchmarks/RESULTS.md`; a config
+    that fails gets a row carrying its error rather than vanishing from the table (one dead
+    provider must not cost the other three runs). `bench.py` now scores **verify precision** — the
+    false-positive filter rate, the real residue of the old `BACKLOG.md` item — and
+    `metrics.json` carries `discovery.duplicates_dropped`, the other half of it. `MODELS.md` links
+    the table instead of restating numbers. 12 tests.
+    **The box stays unchecked until a real four-way run produces `benchmarks/RESULTS.md`** — that
+    spends live Gemini and OpenAI quota and is the maintainer's call to start.
+  - **Reproducibility: the acceptance criterion is not achievable as written.** "Re-running with
+    the same seed and model reproduces the score" cannot hold here — Anthropic accepts no seed at
+    all and none of the four guarantee determinism. Rather than fake it, `RESULTS.md` states the
+    run date and says scores vary between runs, and tells the reader to re-run before reading
+    anything into a small difference.
   - **Fourth provider (decided 2026-07-27): Gemini.** `config/agents.gemini.yaml` is committed and
     the console's model switcher already picks it up (`_config_tag` maps `gemini`). The four are
     Claude (`agents.yaml`), OpenAI (`agents.openai.yaml`), Gemini, and the local Ollama model

src/vpcopilot/bench.py

@@ -30,6 +30,16 @@
 }
 
 
+def _precision(*, used: int, verified: int) -> float:
+    """Verify precision — the false-positive filter rate the old `BACKLOG.md` item asked for.
+
+    Real over reported: a verified finding that matched a key entry or a bonus entry is real,
+    everything else the verifier let through is noise. Distinct from `discovery_recall`, which asks
+    whether the KNOWN vulns were found — a run can ace one and fail the other, which is exactly why
+    both belong in the scorecard."""
+    return round(used / verified, 2) if verified else 0.0
+
+
 def _tail(path: str) -> str:
     return path.split("/api/", 1)[-1] if "/api/" in path else path
 
@@ -101,6 +111,13 @@ def _triage_ok(exp, f) -> bool:
     found = sum(r["found"] for r in rows)
     triage_correct = sum(1 for r in rows if r["triage_ok"])
     noise = [f["id"] for f in verified if f["id"] not in used]
+    metrics = {}
+    mp = out / "metrics.json"
+    if mp.exists():
+        try:
+            metrics = json.loads(mp.read_text())
+        except json.JSONDecodeError:
+            metrics = {}
     score = {
         "expected": n,
         "found": found,
@@ -110,5 +127,9 @@ def _triage_ok(exp, f) -> bool:
         "bonus_found": bonus_found,
         "bonus_total": len(bonus),
         "noise": len(noise),
+        "verified": len(verified),
+        "verify_precision": _precision(used=len(used), verified=len(verified)),
+        "duplicates_dropped": (metrics.get("discovery") or {}).get("duplicates_dropped", 0),
+        "wall_time_s": (metrics.get("timing_s") or {}).get("total", 0.0),
     }
     return {"rows": rows, "score": score, "noise": noise}

src/vpcopilot/cli.py

@@ -65,10 +65,33 @@ def bench(
     out: str = typer.Option("out", help="output directory"),
     config: str = typer.Option(None, "--config", help="path to agents.yaml"),
     rescore: bool = typer.Option(False, "--rescore", help="score the existing out/ without re-scanning"),
+    all_configs: bool = typer.Option(False, "--all-configs",
+                                     help="score EVERY config/agents*.yaml and write benchmarks/RESULTS.md"),
+    target: str = typer.Option(None, "--target", help="label for the scorecard (default: the repo path)"),
     min_confidence: float = typer.Option(0.5, "--min-confidence", help="drop verified findings below this confidence"),
     concurrency: int = typer.Option(8, "--concurrency", help="parallel workers for discover/verify"),
 ):
     """Run the scan and SCORE it against the answer key (discovery, triage, cure)."""
+    if all_configs:
+        from .scorecard import run_all_configs, write_results
+        res = run_all_configs(repo, key=key, min_confidence=min_confidence, concurrency=concurrency,
+                              rescore=rescore, log=lambda m: rprint(f"[dim]{m}[/dim]"))
+        t = Table(title="model scorecard")
+        for c in ["config", "model", "recall", "precision", "triage", "noise", "wall"]:
+            t.add_column(c)
+        for tag in sorted(res):
+            r = res[tag]
+            if r.get("error"):
+                t.add_row(tag, r.get("model", "?"), "[red]failed[/red]", "", "", "", "")
+                continue
+            sc = r["score"]
+            t.add_row(tag, r["model"], f"{sc['discovery_recall']:.2f}", f"{sc['verify_precision']:.2f}",
+                      f"{sc['triage_accuracy']:.2f}", str(sc["noise"]), f"{sc['wall_time_s']:.0f}s")
+        rprint(t)
+        path = write_results(res, target=target or repo, key=key)
+        rprint(f"wrote [bold]{path}[/bold]")
+        return
+
     from .bench import run_bench
 
     res = run_bench(repo, key, out_dir=out, config_path=config, scan=not rescore,

src/vpcopilot/pipeline.py

@@ -32,9 +32,12 @@ def _vclass(f):
     return f.vuln_class.value if hasattr(f.vuln_class, "value") else f.vuln_class
 
 
-def _dedup_findings(findings, log):
+def _dedup_findings(findings, log, counter: dict | None = None):
     """A6: collapse duplicate findings for one vuln — keyed on (file, vuln_class, endpoint-or-line);
-    keeps the highest-severity representative so one vuln yields one band-aid + one code-fix PR."""
+    keeps the highest-severity representative so one vuln yields one band-aid + one code-fix PR.
+
+    `counter` receives the number dropped, so the count reaches `metrics.json` instead of only the
+    log — the residue the old `BACKLOG.md` per-stage-metrics item left behind."""
     kept, seen = [], {}
     for f in sorted(findings, key=lambda f: _SEV_RANK.get(_sev(f), 9)):
         key = (f.file, _vclass(f), (getattr(f, "endpoint", "") or f"L{f.line}"))
@@ -43,6 +46,8 @@ def _dedup_findings(findings, log):
             continue
         seen[key] = f.id
         kept.append(f)
+    if counter is not None:
+        counter["duplicates_dropped"] = len(findings) - len(kept)
     return kept
 
 
@@ -67,6 +72,7 @@ def run_pipeline(
         if n:
             log(f"  ⚠ {n} file(s) skipped ({reason}) — raise --max-files/--max-bytes to include them")
     t0, started = time.perf_counter(), runmeta.utc_now()
+    dedup_counter: dict = {}
 
     # Ground endpoints in the app's DECLARED routes (OpenAPI spec / framework registrations) so a
     # weaker model looks a finding's path up instead of hallucinating it — and warn loudly if none.
@@ -164,7 +170,7 @@ def _threshold(f):
         from .agents import probe as probe_agent
 
         # A6: collapse duplicate findings so one vuln -> one band-aid + one code-fix PR
-        verified = _dedup_findings(verified, log)
+        verified = _dedup_findings(verified, log, dedup_counter)
         by_id = {f.id: f for f in verified}
 
         # 3) triage — band-aid coverage per finding. Chunk the batch so a big app (dozens of
@@ -261,7 +267,8 @@ def _remediate(f):
     metrics = {
         "timing_s": {"discover": round(discover_s, 2), "verify": round(verify_s, 2),
                      "synthesize": round(synth_s, 2), "total": round(time.perf_counter() - t0, 2)},
-        "discovery": {"files": len(files), "skipped_files": len(skipped), "candidates": len(findings)},
+        "discovery": {"files": len(files), "skipped_files": len(skipped), "candidates": len(findings),
+                      "duplicates_dropped": dedup_counter.get("duplicates_dropped", 0)},
         "verify": {"candidates": len(findings), "verified": len(verified), "refuted": refuted,
                    "dropped_low_confidence": dropped,
                    "confirm_rate": round(len(verified) / len(findings), 2) if findings else 0.0,

src/vpcopilot/scorecard.py

@@ -0,0 +1,145 @@
+"""G4 — the published model scorecard.
+
+`vpcopilot bench --all-configs` runs the same scan and the same answer key through every
+`config/agents*.yaml` and writes one committed `benchmarks/RESULTS.md`. That table is the evidence
+for the headline claim: swap the model in config, change no code, and the pipeline still works.
+
+Two things this deliberately does NOT do.
+
+It does not claim reproducibility. The roadmap's acceptance asked that re-running with the same seed
+reproduce the score; that is not achievable across these four providers — Anthropic accepts no seed
+at all, and none of them guarantee determinism even with one. The table states the run date and
+says scores move between runs, which is the honest version of that criterion.
+
+It does not report token cost. Nothing in `harness.py` counts tokens, and the decision of
+2026-07-27 was that a cost column is not worth threading through every agent call."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Column order is the reading order of a model comparison: did it find the vulns, was what it
+# reported real, did it route them correctly, what did it cost in time.
+COLUMNS = [
+    ("recall", "discovery recall", lambda s: f"{s.get('found', 0)}/{s.get('expected', 0)}"
+     f" = {s.get('discovery_recall', 0):.2f}"),
+    ("precision", "verify precision", lambda s: f"{s.get('verify_precision', 0):.2f}"
+     f" ({s.get('verified', 0) - s.get('noise', 0)}/{s.get('verified', 0)})"),
+    ("triage", "triage accuracy", lambda s: f"{s.get('triage_correct', 0)}/{s.get('found', 0)}"
+     f" = {s.get('triage_accuracy', 0):.2f}"),
+    ("bonus", "bonus finds", lambda s: f"{s.get('bonus_found', 0)}/{s.get('bonus_total', 0)}"),
+    ("noise", "noise", lambda s: str(s.get("noise", 0))),
+    ("dupes", "dupes dropped", lambda s: str(s.get("duplicates_dropped", 0))),
+    ("wall", "wall time", lambda s: f"{s.get('wall_time_s', 0):.0f}s"),
+]
+
+
+def build_results(results: dict, *, target: str, key: str, seed: int | None = None) -> str:
+    """One row per config. A config whose run errored still gets a row carrying the error — a table
+    that silently omits the provider that crashed reads as if it was never tried."""
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    head = [
+        "# Model scorecard",
+        "",
+        f"Same code, same prompts, same answer key — only `config/agents.yaml` changed. Target: "
+        f"**{target}** · key: `{key}` · recorded {ts}.",
+        "",
+        "Regenerate with:",
+        "",
+        "```sh",
+        f"vpcopilot bench <repo> --all-configs --key {key}",
+        "```",
+        "",
+    ]
+    if not results:
+        return "\n".join(head + ["_No configs scored — no `config/agents*.yaml` produced a run._", ""])
+
+    header = "| config | model | " + " | ".join(label for _, label, _ in COLUMNS) + " |"
+    sep = "|---" * (len(COLUMNS) + 2) + "|"
+    rows = []
+    for tag in sorted(results):
+        r = results[tag]
+        model = f"`{r.get('model', '?')}`"
+        if r.get("error"):
+            rows.append(f"| `{tag}` | {model} | " +
+                        " | ".join(["—"] * (len(COLUMNS) - 1)) +
+                        f" | **failed:** {r['error']} |")
+            continue
+        s = r.get("score") or {}
+        rows.append(f"| `{tag}` | {model} | " + " | ".join(fn(s) for _, _, fn in COLUMNS) + " |")
+
+    notes = [
+        "",
+        "## Reading this table",
+        "",
+        "- **discovery recall** — of the vulns the answer key labels, how many were found.",
+        "- **verify precision** — of everything the run reported, how much was real. Recall and "
+        "precision move independently: a run can find every labelled vuln and still bury them in "
+        "false positives.",
+        "- **triage accuracy** — of the vulns found, how many were routed to an acceptable control.",
+        "- **bonus finds** — real vulns beyond the core key, credited rather than counted as noise.",
+        "- **dupes dropped** — duplicate findings for one vuln, collapsed so one vuln yields one "
+        "band-aid and one code-fix PR.",
+        "",
+        "## What this table is not",
+        "",
+        f"- **Not reproducible run to run.** {'A seed of ' + str(seed) + ' was requested, but ' if seed is not None else ''}"
+        "two of these providers accept no seed at all and none guarantee determinism, so scores "
+        "vary between runs. Treat a small difference as noise; re-run before reading anything into "
+        "one. The run date above is part of the result.",
+        "- **Not a cost comparison.** Token cost is not measured — nothing in the harness counts "
+        "tokens, and a cost column was judged not worth threading through every agent call.",
+        "- **Not a general model ranking.** It is one target, one answer key, one prompt set. It "
+        "says which model drives *this* pipeline well.",
+        "",
+    ]
+    return "\n".join(head + [header, sep] + rows + notes)
+
+
+def write_results(results: dict, *, target: str, key: str, seed: int | None = None,
+                  dest_dir: str = "benchmarks") -> str:
+    d = Path(dest_dir)
+    d.mkdir(parents=True, exist_ok=True)
+    p = d / "RESULTS.md"
+    p.write_text(build_results(results, target=target, key=key, seed=seed))
+    return str(p)
+
+
+def run_all_configs(repo: str, *, key: str = "bench/answer_key.yaml", config_dir: str = "config",
+                    out_prefix: str = "out", min_confidence: float = 0.5, concurrency: int = 8,
+                    seed: int | None = None, rescore: bool = False,
+                    log=print) -> dict:
+    """Score every `config/agents*.yaml` against one target, into `out-<tag>` per config.
+
+    A config that fails is recorded and the sweep continues: one dead provider must not cost you
+    the other three runs' worth of tokens. `rescore=True` scores the existing out dirs without
+    re-scanning — the cheap path for iterating on the table itself."""
+    from .bench import run_bench
+    from .config import load_config
+
+    results: dict = {}
+    for cfg in sorted(Path(config_dir).glob("agents*.yaml")):
+        parts = cfg.name.split(".")
+        tag = parts[1] if len(parts) == 3 and parts[0] == "agents" else "default"
+        try:
+            model = load_config(str(cfg)).defaults.model
+        except Exception as e:  # noqa: BLE001
+            results[tag] = {"model": "?", "error": f"unreadable config: {e}"}
+            log(f"  {tag}: unreadable config — {e}")
+            continue
+        if tag == "default":  # agents.yaml -> name it by its provider, as the console does
+            tag = {"anthropic": "claude", "openai": "openai", "ollama": "dgx",
+                   "gemini": "gemini"}.get(model.split("/")[0], model.split("/")[0])
+        out_dir = f"{out_prefix}-{tag}"
+        log(f"[{tag}] {model} -> {out_dir}")
+        try:
+            res = run_bench(repo, key, out_dir=out_dir, config_path=str(cfg), scan=not rescore,
+                            min_confidence=min_confidence, concurrency=concurrency, log=log)
+            results[tag] = {"model": model, "score": res["score"], "out_dir": out_dir}
+            s = res["score"]
+            log(f"  {tag}: recall {s['discovery_recall']:.2f} · precision "
+                f"{s['verify_precision']:.2f} · triage {s['triage_accuracy']:.2f}")
+        except Exception as e:  # noqa: BLE001 — one dead provider must not abort the sweep
+            results[tag] = {"model": model, "error": str(e)[:200], "out_dir": out_dir}
+            log(f"  ⚠ {tag}: {e}")
+    return results