diff --git a/MODELS.md b/MODELS.md index bce1069..16a8e8b 100644 --- a/MODELS.md +++ b/MODELS.md @@ -1,4 +1,16 @@ -# Model-independence proof (PLAN D3) +# Model independence — notes and history + +**The live scorecard is [`benchmarks/RESULTS.md`](benchmarks/RESULTS.md)**, regenerated by +`vpcopilot bench --all-configs` across every `config/agents*.yaml`. Numbers live there so +there is one place to update; this file keeps the qualitative findings — what swapping a model +actually surfaced — which no table captures. + +See also the per-target comparisons in [`benchmarks/`](benchmarks/), e.g. +[`compare-vampi-three-way.md`](benchmarks/compare-vampi-three-way.md). + +--- + +## The original proof (PLAN D3, 2026-07-02) — historical Same code, same prompts, same answer key — only `config/agents.yaml` changed. Both providers ran the full pipeline (discover → verify → triage → generate → remediate) with valid diff --git a/ROADMAP.md b/ROADMAP.md index b54528c..2defe16 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -173,6 +173,20 @@ Live validation on the LB stays. This runs before it, not instead of it. row per `config/agents*.yaml`, scoring discovery, verify precision and recall, triage accuracy, bonus finds, noise, and wall time; re-running with the same seed and model reproduces the score; `MODELS.md` links the table rather than restating numbers. + - **BUILT 2026-07-27, not yet run.** `scorecard.py` + `vpcopilot bench --all-configs` + sweep every `config/agents*.yaml` into `out-` and write `benchmarks/RESULTS.md`; a config + that fails gets a row carrying its error rather than vanishing from the table (one dead + provider must not cost the other three runs). `bench.py` now scores **verify precision** — the + false-positive filter rate, the real residue of the old `BACKLOG.md` item — and + `metrics.json` carries `discovery.duplicates_dropped`, the other half of it. `MODELS.md` links + the table instead of restating numbers. 12 tests. + **The box stays unchecked until a real four-way run produces `benchmarks/RESULTS.md`** — that + spends live Gemini and OpenAI quota and is the maintainer's call to start. + - **Reproducibility: the acceptance criterion is not achievable as written.** "Re-running with + the same seed and model reproduces the score" cannot hold here — Anthropic accepts no seed at + all and none of the four guarantee determinism. Rather than fake it, `RESULTS.md` states the + run date and says scores vary between runs, and tells the reader to re-run before reading + anything into a small difference. - **Fourth provider (decided 2026-07-27): Gemini.** `config/agents.gemini.yaml` is committed and the console's model switcher already picks it up (`_config_tag` maps `gemini`). The four are Claude (`agents.yaml`), OpenAI (`agents.openai.yaml`), Gemini, and the local Ollama model diff --git a/src/vpcopilot/bench.py b/src/vpcopilot/bench.py index 6b6c57a..31aae07 100644 --- a/src/vpcopilot/bench.py +++ b/src/vpcopilot/bench.py @@ -30,6 +30,16 @@ } +def _precision(*, used: int, verified: int) -> float: + """Verify precision — the false-positive filter rate the old `BACKLOG.md` item asked for. + + Real over reported: a verified finding that matched a key entry or a bonus entry is real, + everything else the verifier let through is noise. Distinct from `discovery_recall`, which asks + whether the KNOWN vulns were found — a run can ace one and fail the other, which is exactly why + both belong in the scorecard.""" + return round(used / verified, 2) if verified else 0.0 + + def _tail(path: str) -> str: return path.split("/api/", 1)[-1] if "/api/" in path else path @@ -101,6 +111,13 @@ def _triage_ok(exp, f) -> bool: found = sum(r["found"] for r in rows) triage_correct = sum(1 for r in rows if r["triage_ok"]) noise = [f["id"] for f in verified if f["id"] not in used] + metrics = {} + mp = out / "metrics.json" + if mp.exists(): + try: + metrics = json.loads(mp.read_text()) + except json.JSONDecodeError: + metrics = {} score = { "expected": n, "found": found, @@ -110,5 +127,9 @@ def _triage_ok(exp, f) -> bool: "bonus_found": bonus_found, "bonus_total": len(bonus), "noise": len(noise), + "verified": len(verified), + "verify_precision": _precision(used=len(used), verified=len(verified)), + "duplicates_dropped": (metrics.get("discovery") or {}).get("duplicates_dropped", 0), + "wall_time_s": (metrics.get("timing_s") or {}).get("total", 0.0), } return {"rows": rows, "score": score, "noise": noise} diff --git a/src/vpcopilot/cli.py b/src/vpcopilot/cli.py index ec77c95..a5d67cb 100644 --- a/src/vpcopilot/cli.py +++ b/src/vpcopilot/cli.py @@ -65,10 +65,33 @@ def bench( out: str = typer.Option("out", help="output directory"), config: str = typer.Option(None, "--config", help="path to agents.yaml"), rescore: bool = typer.Option(False, "--rescore", help="score the existing out/ without re-scanning"), + all_configs: bool = typer.Option(False, "--all-configs", + help="score EVERY config/agents*.yaml and write benchmarks/RESULTS.md"), + target: str = typer.Option(None, "--target", help="label for the scorecard (default: the repo path)"), min_confidence: float = typer.Option(0.5, "--min-confidence", help="drop verified findings below this confidence"), concurrency: int = typer.Option(8, "--concurrency", help="parallel workers for discover/verify"), ): """Run the scan and SCORE it against the answer key (discovery, triage, cure).""" + if all_configs: + from .scorecard import run_all_configs, write_results + res = run_all_configs(repo, key=key, min_confidence=min_confidence, concurrency=concurrency, + rescore=rescore, log=lambda m: rprint(f"[dim]{m}[/dim]")) + t = Table(title="model scorecard") + for c in ["config", "model", "recall", "precision", "triage", "noise", "wall"]: + t.add_column(c) + for tag in sorted(res): + r = res[tag] + if r.get("error"): + t.add_row(tag, r.get("model", "?"), "[red]failed[/red]", "", "", "", "") + continue + sc = r["score"] + t.add_row(tag, r["model"], f"{sc['discovery_recall']:.2f}", f"{sc['verify_precision']:.2f}", + f"{sc['triage_accuracy']:.2f}", str(sc["noise"]), f"{sc['wall_time_s']:.0f}s") + rprint(t) + path = write_results(res, target=target or repo, key=key) + rprint(f"wrote [bold]{path}[/bold]") + return + from .bench import run_bench res = run_bench(repo, key, out_dir=out, config_path=config, scan=not rescore, diff --git a/src/vpcopilot/pipeline.py b/src/vpcopilot/pipeline.py index d69a8d4..d3a3ef5 100644 --- a/src/vpcopilot/pipeline.py +++ b/src/vpcopilot/pipeline.py @@ -32,9 +32,12 @@ def _vclass(f): return f.vuln_class.value if hasattr(f.vuln_class, "value") else f.vuln_class -def _dedup_findings(findings, log): +def _dedup_findings(findings, log, counter: dict | None = None): """A6: collapse duplicate findings for one vuln — keyed on (file, vuln_class, endpoint-or-line); - keeps the highest-severity representative so one vuln yields one band-aid + one code-fix PR.""" + keeps the highest-severity representative so one vuln yields one band-aid + one code-fix PR. + + `counter` receives the number dropped, so the count reaches `metrics.json` instead of only the + log — the residue the old `BACKLOG.md` per-stage-metrics item left behind.""" kept, seen = [], {} for f in sorted(findings, key=lambda f: _SEV_RANK.get(_sev(f), 9)): key = (f.file, _vclass(f), (getattr(f, "endpoint", "") or f"L{f.line}")) @@ -43,6 +46,8 @@ def _dedup_findings(findings, log): continue seen[key] = f.id kept.append(f) + if counter is not None: + counter["duplicates_dropped"] = len(findings) - len(kept) return kept @@ -67,6 +72,7 @@ def run_pipeline( if n: log(f" ⚠ {n} file(s) skipped ({reason}) — raise --max-files/--max-bytes to include them") t0, started = time.perf_counter(), runmeta.utc_now() + dedup_counter: dict = {} # Ground endpoints in the app's DECLARED routes (OpenAPI spec / framework registrations) so a # weaker model looks a finding's path up instead of hallucinating it — and warn loudly if none. @@ -164,7 +170,7 @@ def _threshold(f): from .agents import probe as probe_agent # A6: collapse duplicate findings so one vuln -> one band-aid + one code-fix PR - verified = _dedup_findings(verified, log) + verified = _dedup_findings(verified, log, dedup_counter) by_id = {f.id: f for f in verified} # 3) triage — band-aid coverage per finding. Chunk the batch so a big app (dozens of @@ -261,7 +267,8 @@ def _remediate(f): metrics = { "timing_s": {"discover": round(discover_s, 2), "verify": round(verify_s, 2), "synthesize": round(synth_s, 2), "total": round(time.perf_counter() - t0, 2)}, - "discovery": {"files": len(files), "skipped_files": len(skipped), "candidates": len(findings)}, + "discovery": {"files": len(files), "skipped_files": len(skipped), "candidates": len(findings), + "duplicates_dropped": dedup_counter.get("duplicates_dropped", 0)}, "verify": {"candidates": len(findings), "verified": len(verified), "refuted": refuted, "dropped_low_confidence": dropped, "confirm_rate": round(len(verified) / len(findings), 2) if findings else 0.0, diff --git a/src/vpcopilot/scorecard.py b/src/vpcopilot/scorecard.py new file mode 100644 index 0000000..051e208 --- /dev/null +++ b/src/vpcopilot/scorecard.py @@ -0,0 +1,145 @@ +"""G4 — the published model scorecard. + +`vpcopilot bench --all-configs` runs the same scan and the same answer key through every +`config/agents*.yaml` and writes one committed `benchmarks/RESULTS.md`. That table is the evidence +for the headline claim: swap the model in config, change no code, and the pipeline still works. + +Two things this deliberately does NOT do. + +It does not claim reproducibility. The roadmap's acceptance asked that re-running with the same seed +reproduce the score; that is not achievable across these four providers — Anthropic accepts no seed +at all, and none of them guarantee determinism even with one. The table states the run date and +says scores move between runs, which is the honest version of that criterion. + +It does not report token cost. Nothing in `harness.py` counts tokens, and the decision of +2026-07-27 was that a cost column is not worth threading through every agent call.""" +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path + +# Column order is the reading order of a model comparison: did it find the vulns, was what it +# reported real, did it route them correctly, what did it cost in time. +COLUMNS = [ + ("recall", "discovery recall", lambda s: f"{s.get('found', 0)}/{s.get('expected', 0)}" + f" = {s.get('discovery_recall', 0):.2f}"), + ("precision", "verify precision", lambda s: f"{s.get('verify_precision', 0):.2f}" + f" ({s.get('verified', 0) - s.get('noise', 0)}/{s.get('verified', 0)})"), + ("triage", "triage accuracy", lambda s: f"{s.get('triage_correct', 0)}/{s.get('found', 0)}" + f" = {s.get('triage_accuracy', 0):.2f}"), + ("bonus", "bonus finds", lambda s: f"{s.get('bonus_found', 0)}/{s.get('bonus_total', 0)}"), + ("noise", "noise", lambda s: str(s.get("noise", 0))), + ("dupes", "dupes dropped", lambda s: str(s.get("duplicates_dropped", 0))), + ("wall", "wall time", lambda s: f"{s.get('wall_time_s', 0):.0f}s"), +] + + +def build_results(results: dict, *, target: str, key: str, seed: int | None = None) -> str: + """One row per config. A config whose run errored still gets a row carrying the error — a table + that silently omits the provider that crashed reads as if it was never tried.""" + ts = datetime.now(timezone.utc).strftime("%Y-%m-%d") + head = [ + "# Model scorecard", + "", + f"Same code, same prompts, same answer key — only `config/agents.yaml` changed. Target: " + f"**{target}** · key: `{key}` · recorded {ts}.", + "", + "Regenerate with:", + "", + "```sh", + f"vpcopilot bench --all-configs --key {key}", + "```", + "", + ] + if not results: + return "\n".join(head + ["_No configs scored — no `config/agents*.yaml` produced a run._", ""]) + + header = "| config | model | " + " | ".join(label for _, label, _ in COLUMNS) + " |" + sep = "|---" * (len(COLUMNS) + 2) + "|" + rows = [] + for tag in sorted(results): + r = results[tag] + model = f"`{r.get('model', '?')}`" + if r.get("error"): + rows.append(f"| `{tag}` | {model} | " + + " | ".join(["—"] * (len(COLUMNS) - 1)) + + f" | **failed:** {r['error']} |") + continue + s = r.get("score") or {} + rows.append(f"| `{tag}` | {model} | " + " | ".join(fn(s) for _, _, fn in COLUMNS) + " |") + + notes = [ + "", + "## Reading this table", + "", + "- **discovery recall** — of the vulns the answer key labels, how many were found.", + "- **verify precision** — of everything the run reported, how much was real. Recall and " + "precision move independently: a run can find every labelled vuln and still bury them in " + "false positives.", + "- **triage accuracy** — of the vulns found, how many were routed to an acceptable control.", + "- **bonus finds** — real vulns beyond the core key, credited rather than counted as noise.", + "- **dupes dropped** — duplicate findings for one vuln, collapsed so one vuln yields one " + "band-aid and one code-fix PR.", + "", + "## What this table is not", + "", + f"- **Not reproducible run to run.** {'A seed of ' + str(seed) + ' was requested, but ' if seed is not None else ''}" + "two of these providers accept no seed at all and none guarantee determinism, so scores " + "vary between runs. Treat a small difference as noise; re-run before reading anything into " + "one. The run date above is part of the result.", + "- **Not a cost comparison.** Token cost is not measured — nothing in the harness counts " + "tokens, and a cost column was judged not worth threading through every agent call.", + "- **Not a general model ranking.** It is one target, one answer key, one prompt set. It " + "says which model drives *this* pipeline well.", + "", + ] + return "\n".join(head + [header, sep] + rows + notes) + + +def write_results(results: dict, *, target: str, key: str, seed: int | None = None, + dest_dir: str = "benchmarks") -> str: + d = Path(dest_dir) + d.mkdir(parents=True, exist_ok=True) + p = d / "RESULTS.md" + p.write_text(build_results(results, target=target, key=key, seed=seed)) + return str(p) + + +def run_all_configs(repo: str, *, key: str = "bench/answer_key.yaml", config_dir: str = "config", + out_prefix: str = "out", min_confidence: float = 0.5, concurrency: int = 8, + seed: int | None = None, rescore: bool = False, + log=print) -> dict: + """Score every `config/agents*.yaml` against one target, into `out-` per config. + + A config that fails is recorded and the sweep continues: one dead provider must not cost you + the other three runs' worth of tokens. `rescore=True` scores the existing out dirs without + re-scanning — the cheap path for iterating on the table itself.""" + from .bench import run_bench + from .config import load_config + + results: dict = {} + for cfg in sorted(Path(config_dir).glob("agents*.yaml")): + parts = cfg.name.split(".") + tag = parts[1] if len(parts) == 3 and parts[0] == "agents" else "default" + try: + model = load_config(str(cfg)).defaults.model + except Exception as e: # noqa: BLE001 + results[tag] = {"model": "?", "error": f"unreadable config: {e}"} + log(f" {tag}: unreadable config — {e}") + continue + if tag == "default": # agents.yaml -> name it by its provider, as the console does + tag = {"anthropic": "claude", "openai": "openai", "ollama": "dgx", + "gemini": "gemini"}.get(model.split("/")[0], model.split("/")[0]) + out_dir = f"{out_prefix}-{tag}" + log(f"[{tag}] {model} -> {out_dir}") + try: + res = run_bench(repo, key, out_dir=out_dir, config_path=str(cfg), scan=not rescore, + min_confidence=min_confidence, concurrency=concurrency, log=log) + results[tag] = {"model": model, "score": res["score"], "out_dir": out_dir} + s = res["score"] + log(f" {tag}: recall {s['discovery_recall']:.2f} · precision " + f"{s['verify_precision']:.2f} · triage {s['triage_accuracy']:.2f}") + except Exception as e: # noqa: BLE001 — one dead provider must not abort the sweep + results[tag] = {"model": model, "error": str(e)[:200], "out_dir": out_dir} + log(f" ⚠ {tag}: {e}") + return results diff --git a/tests/test_scorecard.py b/tests/test_scorecard.py new file mode 100644 index 0000000..a8bf0b3 --- /dev/null +++ b/tests/test_scorecard.py @@ -0,0 +1,148 @@ +"""G4 — the published model scorecard. + +`bench.py` scored recall and triage against the answer key. G4 adds the missing half of the old +BACKLOG ask (verify PRECISION — the false-positive filter rate), the discovery-duplicate counter, +and `scorecard.py`, which turns per-config bench results into one committed `benchmarks/RESULTS.md`. + +Offline: scores are hand-built dicts, so nothing here scans or calls a model.""" +import json + +from vpcopilot import scorecard + + +def _score(**over): + base = {"expected": 9, "found": 9, "discovery_recall": 1.0, "triage_correct": 9, + "triage_accuracy": 1.0, "bonus_found": 7, "bonus_total": 8, "noise": 0, + "verified": 16, "verify_precision": 1.0, "duplicates_dropped": 2, + "wall_time_s": 41.2} + base.update(over) + return base + + +# ---- verify precision: the residue from the old BACKLOG item ---- +def test_precision_counts_key_matches_and_bonus_as_real(tmp_path): + """Precision is real-over-verified: a finding that matched the key OR a bonus entry is real; + everything else the verifier let through is noise.""" + from vpcopilot.bench import _precision + assert _precision(used=9, verified=9) == 1.0 + assert _precision(used=8, verified=16) == 0.5 + assert _precision(used=0, verified=0) == 0.0 # nothing verified -> no claim, not a crash + + +def test_precision_and_recall_are_different_numbers(): + """Recall asks 'did we find the known vulns'; precision asks 'was what we reported real'. + A run can ace one and fail the other — which is the whole reason the item exists.""" + from vpcopilot.bench import _precision + assert _precision(used=9, verified=30) < 0.35 # perfect recall, poor precision + + +# ---- the scorecard table ---- +def test_results_md_has_one_row_per_config(): + md = scorecard.build_results({ + "claude": {"model": "anthropic/claude-opus-4-8", "score": _score()}, + "openai": {"model": "openai/gpt-4.1", "score": _score(found=6, discovery_recall=0.67)}, + "gemini": {"model": "gemini/gemini-3.1-pro-preview", "score": _score(noise=3, verify_precision=0.81)}, + "dgx": {"model": "openai/qwen3-coder:30b-a3b-q8_0", "score": _score(found=5, discovery_recall=0.56)}, + }, target="Nimbus vuln-lab", key="bench/answer_key.yaml") + rows = [ln for ln in md.splitlines() if ln.startswith("| `")] + assert len(rows) == 4 + for tag in ("claude", "openai", "gemini", "dgx"): + assert tag in md + assert "anthropic/claude-opus-4-8" in md and "qwen3-coder" in md + + +def test_results_md_reports_every_scored_dimension(): + md = scorecard.build_results({"claude": {"model": "m", "score": _score()}}, + target="t", key="k") + for header in ("recall", "precision", "triage", "bonus", "noise", "wall"): + assert header in md.lower(), header + + +def test_results_md_states_the_reproducibility_limit_rather_than_claiming_it(): + """The acceptance asked that a re-run reproduce the score. It cannot: two of the four providers + accept no seed at all, and none guarantee determinism. Saying so is the honest version.""" + md = scorecard.build_results({"claude": {"model": "m", "score": _score()}}, target="t", key="k") + low = md.lower() + assert "seed" in low + assert any(w in low for w in ("not deterministic", "does not guarantee", "vary", "variance")) + + +def test_results_md_records_what_produced_it(): + md = scorecard.build_results({"claude": {"model": "m", "score": _score()}}, + target="Nimbus vuln-lab", key="bench/answer_key.yaml") + assert "Nimbus vuln-lab" in md and "bench/answer_key.yaml" in md + assert "vpcopilot bench" in md # the command that regenerates it + + +def test_a_failed_config_is_reported_not_dropped(): + """A config whose run errored must appear with its error — a table that silently omits the + provider that crashed reads as if it was never tried.""" + md = scorecard.build_results({ + "claude": {"model": "m", "score": _score()}, + "gemini": {"model": "gemini/x", "error": "GeminiException 404: model retired"}, + }, target="t", key="k") + assert "gemini" in md and "404" in md + assert len([ln for ln in md.splitlines() if ln.startswith("| `")]) == 2 + + +def test_write_results_lands_in_benchmarks(tmp_path): + p = scorecard.write_results({"claude": {"model": "m", "score": _score()}}, + target="t", key="k", dest_dir=str(tmp_path)) + assert p.endswith("RESULTS.md") and "RESULTS.md" in json.dumps(p) + assert "claude" in open(p).read() + + +def test_empty_results_is_a_valid_table_not_a_crash(): + md = scorecard.build_results({}, target="t", key="k") + assert "no configs" in md.lower() or md.strip() + + +# ---- the sweep ---- +def test_every_config_is_scored_and_tagged(monkeypatch, tmp_path): + cfgs = tmp_path / "config" + cfgs.mkdir() + (cfgs / "agents.yaml").write_text("defaults:\n model: anthropic/claude-opus-4-8\nagents: {}\n") + (cfgs / "agents.openai.yaml").write_text("defaults:\n model: openai/gpt-4.1\nagents: {}\n") + (cfgs / "agents.gemini.yaml").write_text("defaults:\n model: gemini/g-3.1\nagents: {}\n") + seen = [] + monkeypatch.setattr(scorecard, "__name__", scorecard.__name__) + import vpcopilot.bench as bench + monkeypatch.setattr(bench, "run_bench", + lambda repo, key, **kw: seen.append(kw["out_dir"]) or {"score": _score()}) + res = scorecard.run_all_configs("repo", config_dir=str(cfgs), log=lambda m: None) + # agents.yaml is tagged by its provider, exactly as the console's switcher does + assert set(res) == {"claude", "openai", "gemini"} + assert set(seen) == {"out-claude", "out-openai", "out-gemini"} + assert res["openai"]["model"] == "openai/gpt-4.1" + + +def test_one_dead_provider_does_not_abort_the_sweep(monkeypatch, tmp_path): + """A four-way run is expensive; losing the other three because one provider 404s would be the + worst possible failure mode.""" + cfgs = tmp_path / "config" + cfgs.mkdir() + (cfgs / "agents.yaml").write_text("defaults:\n model: anthropic/claude-opus-4-8\nagents: {}\n") + (cfgs / "agents.gemini.yaml").write_text("defaults:\n model: gemini/retired\nagents: {}\n") + import vpcopilot.bench as bench + + def flaky(repo, key, **kw): + if "gemini" in kw["out_dir"]: + raise RuntimeError("GeminiException 404: model no longer available") + return {"score": _score()} + monkeypatch.setattr(bench, "run_bench", flaky) + res = scorecard.run_all_configs("repo", config_dir=str(cfgs), log=lambda m: None) + assert res["claude"]["score"]["discovery_recall"] == 1.0 + assert "404" in res["gemini"]["error"] and "score" not in res["gemini"] + assert "404" in scorecard.build_results(res, target="t", key="k") + + +def test_rescore_does_not_rerun_the_scan(monkeypatch, tmp_path): + cfgs = tmp_path / "config" + cfgs.mkdir() + (cfgs / "agents.yaml").write_text("defaults:\n model: anthropic/x\nagents: {}\n") + got = {} + import vpcopilot.bench as bench + monkeypatch.setattr(bench, "run_bench", + lambda repo, key, **kw: got.update(kw) or {"score": _score()}) + scorecard.run_all_configs("repo", config_dir=str(cfgs), rescore=True, log=lambda m: None) + assert got["scan"] is False