research: manual ground-truth check, adjudication and corrected hidden rates

research/finding_identity/gt_agreement.py

@@ -0,0 +1,248 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Score the manual ground-truth check: how often the harness's truth holds, per rater,
+and how well two raters agree.
+
+``python -m research.finding_identity.gt_agreement --human <labels> --agent <labels>``
+reads the key (``sample.csv`` from :mod:`gt_sample`) and each rater's labels, keyed by
+the blinded item id, and writes ``agreement.md`` next to the key.
+
+Labels: a JSON object ``{item: {"verdict": ...}}``, or a directory of ``<item>.json``
+files (as the labelling page's store exports them). A ``pair`` verdict of ``same`` and an
+``introduced`` verdict of ``new`` confirm the harness's truth; ``different`` and
+``existed`` contradict it; ``unclear`` counts for neither.
+
+``--adjudicated`` adds the final verdict per item (same format, each with a ``source``
+saying who settled it). With ``--results`` (the corpus replays the sample came from),
+the report also estimates each content scheme's hidden rate after removing the
+ground-truth errors: every hidden finding of those schemes lies in the
+``new_but_matched`` stratum, so its count is scaled by the share of that scheme's
+sampled matches that the adjudication confirms as truly new.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from collections import Counter
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+
+from .gt_sample import CONTENT_SCHEMES
+from .stats import wilson
+
+HERE = Path(__file__).parent
+KEY = HERE / "paper" / "ground_truth_check" / "sample.csv"
+
+CONFIRMS = {"same", "new"}
+CONTRADICTS = {"different", "existed"}
+
+
+def load_labels(path: Path) -> dict[str, str]:
+    """``{item: verdict}`` from a JSON file or a directory of per-item JSON files."""
+    if path.is_dir():
+        out = {}
+        for f in sorted(path.rglob("*.json")):
+            body = json.loads(f.read_text(encoding="utf-8"))
+            body = body.get("data", body) if isinstance(body, dict) else {}
+            if isinstance(body, dict) and isinstance(body.get("verdict"), str):
+                out[f.stem] = body["verdict"]
+        return out
+    raw = json.loads(path.read_text(encoding="utf-8"))
+    return {k: v["verdict"] for k, v in raw.items() if isinstance(v, dict) and "verdict" in v}
+
+
+def judgement(verdict: str | None) -> str:
+    """``confirms``, ``contradicts`` or ``unclear`` (also for a missing label)."""
+    if verdict in CONFIRMS:
+        return "confirms"
+    if verdict in CONTRADICTS:
+        return "contradicts"
+    return "unclear"
+
+
+def cohen_kappa(a: Sequence[str], b: Sequence[str]) -> float | None:
+    """Cohen's kappa for two equally long label sequences; ``None`` if undefined."""
+    n = len(a)
+    if n == 0 or n != len(b):
+        return None
+    observed = sum(x == y for x, y in zip(a, b, strict=True)) / n
+    ca, cb = Counter(a), Counter(b)
+    expected = sum(ca[k] * cb[k] for k in ca.keys() | cb.keys()) / (n * n)
+    if expected == 1:
+        return None
+    return (observed - expected) / (1 - expected)
+
+
+def precision_rows(
+    key: Sequence[Mapping[str, str]], labels: Mapping[str, str], group: str
+) -> list[tuple[str, int, int, int]]:
+    """Per value of *group* (and overall): (name, confirms, contradicts, unclear)."""
+    groups: dict[str, Counter[str]] = {}
+    for row in key:
+        j = judgement(labels.get(row["item"]))
+        for name in (row[group], "all"):
+            groups.setdefault(name, Counter())[j] += 1
+    names = [*sorted(n for n in groups if n != "all"), "all"]
+    return [
+        (n, groups[n]["confirms"], groups[n]["contradicts"], groups[n]["unclear"]) for n in names
+    ]
+
+
+def _kind(row: Mapping[str, str]) -> str:
+    return "introduced" if row["truth"] == "new" else "pair"
+
+
+def report(
+    key: Sequence[Mapping[str, str]],
+    raters: Mapping[str, Mapping[str, str]],
+    *,
+    header: bool = True,
+) -> str:
+    rows = [{**r, "kind": _kind(r)} for r in key]
+    out = (
+        [
+            "# Ground-truth check: results",
+            "",
+            "Generated by `research/finding_identity/gt_agreement.py`. *Holds* is the share of",
+            "decided items (unclear left out) where the rater confirms the harness's truth, with",
+            "a Wilson 95% interval.",
+            "",
+        ]
+        if header
+        else []
+    )
+    for name, labels in raters.items():
+        labelled = sum(1 for r in rows if r["item"] in labels)
+        out += [f"## {name}", "", f"{labelled} of {len(rows)} items labelled.", ""]
+        for group in ("kind", "stratum"):
+            out += [
+                f"| {group} | confirms | contradicts | unclear | holds | 95% CI |",
+                "|---|---:|---:|---:|---:|---|",
+            ]
+            for g, yes, no, unclear in precision_rows(rows, labels, group):
+                ci = wilson(yes, yes + no)
+                rate = f"{100 * yes / (yes + no):.1f}%" if yes + no else "n/a"
+                cis = f"[{100 * ci[0]:.1f}, {100 * ci[1]:.1f}]" if ci else "n/a"
+                out.append(f"| {g} | {yes} | {no} | {unclear} | {rate} | {cis} |")
+            out.append("")
+    names = list(raters)
+    if len(names) == 2:
+        both = [r for r in rows if r["item"] in raters[names[0]] and r["item"] in raters[names[1]]]
+        a = [judgement(raters[names[0]][r["item"]]) for r in both]
+        b = [judgement(raters[names[1]][r["item"]]) for r in both]
+        kappa = cohen_kappa(a, b)
+        agree = sum(x == y for x, y in zip(a, b, strict=True))
+        out += [
+            f"## Agreement between {names[0]} and {names[1]}",
+            "",
+            f"On the {len(both)} items both labelled (confirms / contradicts / unclear):",
+            f"{agree} agree ({100 * agree / len(both):.1f}%), Cohen's kappa "
+            + (f"{kappa:.2f}." if kappa is not None else "undefined.")
+            if both
+            else "No item is labelled by both.",
+            "",
+        ]
+        disputed = [r["item"] for r, x, y in zip(both, a, b, strict=True) if x != y]
+        if disputed:
+            out += ["Items to adjudicate: " + ", ".join(disputed) + ".", ""]
+    return "\n".join(out)
+
+
+def corrected_hidden(
+    key: Sequence[Mapping[str, str]], adjudicated: Mapping[str, str], results: Path
+) -> list[str]:
+    """Markdown lines: each content scheme's hidden rate before and after correction."""
+    hidden: Counter[str] = Counter()
+    introduced = 0
+    for f in sorted(results.glob("*/findings.jsonl")):
+        with f.open(encoding="utf-8") as fh:
+            for line in fh:
+                rec = json.loads(line)
+                if rec["truth"] != "new":
+                    continue
+                introduced += 1
+                for scheme in CONTENT_SCHEMES:
+                    hidden[scheme] += rec["outcomes"][scheme] == "false_known"
+    sample = [r for r in key if r["stratum"] == "new_but_matched"]
+    out = [
+        "## Hidden rates after correcting the ground truth",
+        "",
+        f"From `{results}`: {introduced:,} introduced findings. *Matched, sampled*: the",
+        "scheme's hidden findings among the sampled `new_but_matched` items; *truly new*:",
+        "those the adjudication confirms as new. The corrected rate scales the hidden",
+        "count by that share (range: its Wilson 95% interval).",
+        "",
+        "| scheme | hidden | matched, sampled | truly new | corrected hidden | range |",
+        "|---|---:|---:|---:|---:|---|",
+    ]
+    for scheme in CONTENT_SCHEMES:
+        rows = [r for r in sample if scheme in r["disagreeing_schemes"].split()]
+        new = sum(judgement(adjudicated.get(r["item"])) == "confirms" for r in rows)
+        ci = wilson(new, len(rows))
+        rate = hidden[scheme] / introduced if introduced else 0.0
+        if not rows or ci is None:
+            out.append(f"| `{scheme}` | {100 * rate:.2f}% | 0 | 0 | n/a | n/a |")
+            continue
+        corrected = rate * new / len(rows)
+        out.append(
+            f"| `{scheme}` | {100 * rate:.2f}% ({hidden[scheme]}) | {len(rows)} | {new} | "
+            f"{100 * corrected:.2f}% | [{100 * rate * ci[0]:.2f}, {100 * rate * ci[1]:.2f}] |"
+        )
+    return [*out, ""]
+
+
+def main(argv: list[str] | None = None) -> None:
+    ap = argparse.ArgumentParser(prog="python -m research.finding_identity.gt_agreement")
+    ap.add_argument("--key", type=Path, default=KEY, help="sample.csv from gt_sample")
+    ap.add_argument("--human", type=Path, help="the person's labels (file or directory)")
+    ap.add_argument(
+        "--agent",
+        type=Path,
+        action="append",
+        help="the agent rater's labels (file or directory); repeat it, and later files "
+        "override earlier ones, item by item",
+    )
+    ap.add_argument(
+        "--human-kind",
+        choices=["pair", "introduced"],
+        help="score only the person's labels of this item kind",
+    )
+    ap.add_argument("--adjudicated", type=Path, help="final verdicts, with their sources")
+    ap.add_argument("--results", type=Path, help="the replays the sample came from")
+    ap.add_argument("--out", type=Path, help="default: agreement.md next to the key")
+    args = ap.parse_args(argv)
+    with args.key.open(newline="", encoding="utf-8") as f:
+        key = list(csv.DictReader(f))
+    raters = {}
+    if args.human:
+        human = load_labels(args.human)
+        if args.human_kind:
+            kinds = {r["item"]: _kind(r) for r in key}
+            human = {k: v for k, v in human.items() if kinds.get(k) == args.human_kind}
+        raters["Human rater"] = human
+    if args.agent:
+        agent: dict[str, str] = {}
+        for path in args.agent:
+            agent.update(load_labels(path))
+        raters["Agent rater"] = agent
+    if not raters:
+        raise SystemExit("give --human and/or --agent")
+    text = report(key, raters)
+    if args.adjudicated:
+        final = load_labels(args.adjudicated)
+        raw = json.loads(args.adjudicated.read_text(encoding="utf-8"))
+        sources = Counter(str(v.get("source", "unknown")) for v in raw.values())
+        text += "\n" + report(key, {"Adjudicated": final}, header=False)
+        text += "\nSources of the adjudicated verdicts:\n\n"
+        text += "\n".join(f"- {n}: {src}" for src, n in sources.most_common()) + "\n"
+        if args.results:
+            text += "\n" + "\n".join(corrected_hidden(key, final, args.results))
+    out = args.out or args.key.parent / "agreement.md"
+    out.write_text(text.rstrip("\n") + "\n", encoding="utf-8")
+    print(f"wrote {out}")
+
+
+if __name__ == "__main__":
+    main()

research/finding_identity/gt_sample.py

@@ -16,7 +16,13 @@
 It writes ``sample.md``, one entry per finding with the code before and after the commit
 and a link to the commit, and ``sample.csv`` with an empty ``verdict`` column to fill in:
 ``same`` (the old and new finding are the same issue), ``different``, or ``unclear``.
-The sample is seeded, so rerunning it gives the same findings.
+
+For independent raters it also writes ``items.json``: the same findings, shuffled under
+neutral ids (``sample.csv``'s ``item`` column is the key), without the harness's answer,
+the stratum or the schemes' outcomes. A persisting finding becomes a ``pair`` item ("is
+the code flagged at the old line the same issue as at the new line?"), an introduced one
+an ``introduced`` item ("did this issue already exist before the commit?").
+The sample is seeded, so rerunning it gives the same findings and the same order.
 """
 
 from __future__ import annotations
@@ -30,7 +36,12 @@
 from pathlib import Path
 from typing import Any
 
+import codeguard.rules  # noqa: F401  (registers the built-in rules)
+from codeguard.engine.registry import REGISTRY
+
 from .corpus import load_corpus
+from .linemap import map_lines
+from .schemes import split_lines
 
 HERE = Path(__file__).parent
 
@@ -69,15 +80,42 @@ def _git(repo: Path, *args: str) -> str:
     return done.stdout.decode("utf-8", "replace") if done.returncode == 0 else ""
 
 
-def snippet(repo: Path, commit: str, path: str, line: int, context: int = 4) -> str:
-    """Lines around *line* of *path* at *commit*, numbered, the line itself marked."""
+def snippet_lines(
+    repo: Path, commit: str, path: str, line: int, context: int = 4
+) -> list[tuple[int, str]]:
+    """``(line number, text)`` for the lines around *line* of *path* at *commit*."""
     lines = _git(repo, "show", f"{commit}:{path}").split("\n")
     lo, hi = max(1, line - context), min(len(lines), line + context)
+    return [(n, lines[n - 1].rstrip()) for n in range(lo, hi + 1)]
+
+
+def snippet(repo: Path, commit: str, path: str, line: int, context: int = 4) -> str:
+    """Lines around *line* of *path* at *commit*, numbered, the line itself marked."""
     return "\n".join(
-        f"{'>' if n == line else ' '}{n:5d}  {lines[n - 1]}" for n in range(lo, hi + 1)
+        f"{'>' if n == line else ' '}{n:5d}  {text}"
+        for n, text in snippet_lines(repo, commit, path, line, context)
     )
 
 
+def corresponding_old_line(
+    old_lines: list[str], new_lines: list[str], line: int
+) -> tuple[int, bool]:
+    """The old line at the place of new line *line*, and whether *line* itself maps to it.
+
+    Uses the ground truth's line map: if *line* maps back to an old line (unchanged or
+    edited), that line; otherwise the line after the nearest old line that maps to a new
+    line above *line* (the point where the new code was inserted).
+    """
+    back = {new: old for old, (new, _) in map_lines(old_lines, new_lines).items()}
+    if line in back:
+        return back[line], True
+    above = [n for n in back if n < line]
+    if not above:
+        return 1, False
+    anchor = max(above)
+    return min(back[anchor] + 1, max(len(old_lines), 1)), False
+
+
 def diff_hunks(repo: Path, old: str, new: str, path: str, old_path: str, line: int) -> str:
     """The hunks of the commit's diff of this file that come within 6 lines of *line*."""
     paths = [path] if path == old_path else [old_path, path]
@@ -164,6 +202,12 @@ def write_sample(
         "",
     ]
     rows = []
+    items = []
+    titles = {r.id: r.title for r in REGISTRY.all()}
+    # Neutral item ids in a seeded order unrelated to the strata.
+    order = list(range(1, len(chosen) + 1))
+    random.Random(len(chosen)).shuffle(order)  # noqa: S311  (blinding order, not security)
+    item_of = {i: f"F{position:03d}" for position, i in enumerate(order, start=1)}
     for i, (stratum, label, rec, old) in enumerate(chosen, start=1):
         repo = repos / label
         new = rec["commit"]
@@ -191,9 +235,44 @@ def write_sample(
         if rec["truth"] == "new" or rec.get("truth_kind") == "edited":
             hunks = diff_hunks(repo, old, new, path, old_path, rec["line"])
             md += ["Diff near the line:", "", "```diff", hunks, "```", ""]
+        old_line = rec.get("old_line")
+        if old_line:
+            before_line, before_mapped = old_line, True
+        else:
+            # An introduced finding: show the old code at the same place, so a rater can
+            # see whether the issue was already there.
+            old_text = split_lines(_git(repo, "show", f"{old}:{old_path}"))
+            new_text = split_lines(_git(repo, "show", f"{new}:{path}"))
+            before_line, before_mapped = corresponding_old_line(old_text, new_text, rec["line"])
+        items.append(
+            {
+                "item": item_of[i],
+                "kind": "pair" if rec["truth"] == "known" else "introduced",
+                "repository": label,
+                "commit_url": url,
+                "rule": rec["rule"],
+                "rule_title": titles.get(rec["rule"], ""),
+                "path": path,
+                "old_path": old_path,
+                "old_line": old_line,
+                "line": rec["line"],
+                "before": snippet_lines(
+                    repo, old, old_path, before_line, context=4 if old_line else 6
+                ),
+                "before_line": before_line,
+                # True when the flagged line itself was already there before the commit.
+                "before_mapped": before_mapped,
+                "after": snippet_lines(repo, new, path, rec["line"]),
+                "diff": "\n".join(
+                    row.rstrip()
+                    for row in diff_hunks(repo, old, new, path, old_path, rec["line"]).split("\n")
+                ),
+            }
+        )
         rows.append(
             {
                 "id": i,
+                "item": item_of[i],
                 "stratum": stratum,
                 "repository": label,
                 "commit": new,
@@ -210,6 +289,8 @@ def write_sample(
     # Quoted source can end lines in spaces; the repository's hooks strip them.
     text = "\n".join(row.rstrip() for row in "\n".join(md).split("\n"))
     (out / "sample.md").write_text(text.rstrip("\n") + "\n", encoding="utf-8")
+    items.sort(key=lambda it: str(it["item"]))
+    (out / "items.json").write_text(json.dumps(items, indent=1) + "\n", encoding="utf-8")
     with (out / "sample.csv").open("w", newline="", encoding="utf-8") as f:
         w = csv.DictWriter(f, fieldnames=list(rows[0]))
         w.writeheader()