From 58e44fc7a93bc91e724ec6e306121f7a5e190cf3 Mon Sep 17 00:00:00 2001 From: Vichitra Attri Date: Wed, 14 Oct 2026 12:43:07 +0000 Subject: [PATCH 1/5] feat(research): record finding-identity totals per rule category Each replay now also scores the findings of every rule category (security, performance, quality) on their own and writes the results to summary.json under by_category, with the same totals and collisions blocks as the summary itself. Scheme keys include the rule ID and the ground truth only pairs findings of the same rule, so the categories add up exactly to the overall totals; a test checks this. The change is additive: existing keys and outputs are unchanged. Signed-off-by: Vichitra Attri --- research/finding_identity/categories.py | 36 +++++++++ research/finding_identity/replay.py | 55 +++++++++++++- research/finding_identity/scan.py | 14 ++++ .../finding_identity/tests/test_categories.py | 74 +++++++++++++++++++ 4 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 research/finding_identity/categories.py create mode 100644 research/finding_identity/tests/test_categories.py diff --git a/research/finding_identity/categories.py b/research/finding_identity/categories.py new file mode 100644 index 0000000..17f7dea --- /dev/null +++ b/research/finding_identity/categories.py @@ -0,0 +1,36 @@ +"""Rule categories (security, performance, quality) for per-category totals. + +Every scheme key includes the rule ID, and the ground truth pairs findings of the same +rule only, so a file's findings split exactly by category: scoring each category's +findings on their own gives counts that add up to the all-category counts. +""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Sequence +from functools import cache + +import codeguard.rules # noqa: F401 (registers the built-in rules) +from codeguard.engine.registry import REGISTRY + +from .matching import Observation + +UNKNOWN = "unknown" + + +@cache +def _categories() -> dict[str, str]: + return {r.id: r.category.value for r in REGISTRY.all()} + + +def category_of(rule_id: str) -> str: + """The category of *rule_id*, or ``"unknown"`` for a rule not in the registry.""" + return _categories().get(rule_id, UNKNOWN) + + +def split_by_category(observations: Sequence[Observation]) -> dict[str, list[Observation]]: + out: dict[str, list[Observation]] = defaultdict(list) + for o in observations: + out[category_of(o.rule)].append(o) + return dict(out) diff --git a/research/finding_identity/replay.py b/research/finding_identity/replay.py index 3cfe047..da8cbed 100644 --- a/research/finding_identity/replay.py +++ b/research/finding_identity/replay.py @@ -18,9 +18,10 @@ import codeguard +from .categories import split_by_category from .gitrepo import Change, GitRepo from .linemap import map_lines -from .matching import OUTCOMES, evaluate +from .matching import OUTCOMES, Observation, evaluate from .scan import Scanner, is_candidate GT_KEYS = ("old", "new", "known", "known_equal", "known_edited", "introduced", "removed") @@ -54,6 +55,7 @@ def _replay_change( schemes: Sequence[str], totals: dict[str, Any], records: list[dict[str, Any]] | None, + by_category: dict[str, dict[str, Any]] | None = None, ) -> None: old_ok, new_ok = is_candidate(ch.old_path), is_candidate(ch.new_path) if not (old_ok or new_ok): @@ -87,6 +89,53 @@ def _replay_change( for rec in result.records: rec.update(path=ch.new_path, old_path=ch.old_path, status=ch.status) records.append(rec) + if by_category is not None: + _add_by_category(by_category, old_obs, new_obs, line_map, schemes, ch.status) + + +def _add_by_category( + by_category: dict[str, dict[str, Any]], + old_obs: Sequence[Observation], + new_obs: Sequence[Observation], + line_map: dict[int, tuple[int, str]], + schemes: Sequence[str], + status: str, +) -> None: + """Score each rule category's findings on their own (they add up to the totals).""" + old_split, new_split = split_by_category(old_obs), split_by_category(new_obs) + for category in sorted(old_split.keys() | new_split.keys()): + result = evaluate( + old_split.get(category, []), new_split.get(category, []), line_map, schemes + ) + totals = by_category.setdefault(category, _empty_totals(schemes)) + for k, v in result.gt.items(): + totals["gt"][k] += v + for s, counts in result.by_scheme.items(): + for k, v in counts.items(): + totals["schemes"][s][k] += v + totals["files"] += 1 + if status == "R": + totals["renamed_files"] += 1 + + +def _category_summaries( + by_category: dict[str, dict[str, Any]], + scanner: Scanner, + schemes: Sequence[str], + categories: Sequence[str], +) -> dict[str, Any]: + """Per category, a ``{"totals", "collisions"}`` block shaped like the summary's.""" + names = sorted(set(categories) | by_category.keys() | scanner.category_observations.keys()) + return { + c: { + "totals": by_category.get(c, _empty_totals(schemes)), + "collisions": { + "observations": scanner.category_observations.get(c, 0), + "by_scheme": scanner.category_collisions.get(c, dict.fromkeys(schemes, 0)), + }, + } + for c in names + } def replay( @@ -116,6 +165,7 @@ def replay( "created": datetime.now(timezone.utc).isoformat(timespec="seconds"), } totals = _empty_totals(schemes) + category_totals: dict[str, dict[str, Any]] = {} started = time.monotonic() with ( (out_dir / "pairs.jsonl").open("w", encoding="utf-8") as pairs_out, @@ -125,7 +175,7 @@ def replay( pair_totals = _empty_totals(schemes) records: list[dict[str, Any]] | None = [] if details else None for ch in repo.changes(old, new): - _replay_change(repo, scanner, ch, schemes, pair_totals, records) + _replay_change(repo, scanner, ch, schemes, pair_totals, records, category_totals) _add(totals, pair_totals) pairs_out.write(json.dumps({"old": old, "new": new, **pair_totals}) + "\n") for rec in records or []: @@ -147,6 +197,7 @@ def replay( "observations": scanner.observations, "by_scheme": scanner.collision_totals, }, + "by_category": _category_summaries(category_totals, scanner, schemes, categories), } (out_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") return summary diff --git a/research/finding_identity/scan.py b/research/finding_identity/scan.py index 8733cf1..1f5cf38 100644 --- a/research/finding_identity/scan.py +++ b/research/finding_identity/scan.py @@ -12,6 +12,7 @@ from codeguard.lang.base import Language from codeguard.lang.registry import language_for_path +from .categories import split_by_category from .matching import Observation, collisions from .schemes import FileContext, keys_for @@ -51,6 +52,9 @@ def __init__(self, rule_ids: list[str], schemes: Sequence[str]) -> None: self.parse_failures = 0 self.observations = 0 self.collision_totals = dict.fromkeys(self._schemes, 0) + #: The same counts split by rule category (see :mod:`categories`). + self.category_observations: dict[str, int] = {} + self.category_collisions: dict[str, dict[str, int]] = {} def scan(self, cache_key: str, path: str, data: bytes) -> ScanResult | None: """Findings for *path* with contents *data*; ``None`` if it cannot be analysed.""" @@ -64,8 +68,18 @@ def scan(self, cache_key: str, path: str, data: bytes) -> ScanResult | None: self.observations += len(result.observations) for scheme, n in result.collisions.items(): self.collision_totals[scheme] += n + self._count_categories(result.observations) return result + def _count_categories(self, observations: Sequence[Observation]) -> None: + for category, obs in split_by_category(observations).items(): + seen = self.category_observations + seen[category] = seen.get(category, 0) + len(obs) + zeros = dict.fromkeys(self._schemes, 0) + totals = self.category_collisions.setdefault(category, zeros) + for scheme, n in collisions(obs, self._schemes).items(): + totals[scheme] += n + def _scan(self, path: str, data: bytes) -> ScanResult | None: lang = language_for_path(path) if lang is None or len(data) > MAX_BYTES: diff --git a/research/finding_identity/tests/test_categories.py b/research/finding_identity/tests/test_categories.py new file mode 100644 index 0000000..98bbd79 --- /dev/null +++ b/research/finding_identity/tests/test_categories.py @@ -0,0 +1,74 @@ +"""Per-category totals: each category is scored on its own and they add up.""" + +import json +from pathlib import Path + +from research.finding_identity.categories import UNKNOWN, category_of, split_by_category +from research.finding_identity.gitrepo import GitRepo +from research.finding_identity.matching import Observation +from research.finding_identity.replay import replay +from research.finding_identity.scan import Scanner +from research.finding_identity.schemes import ALL_SCHEMES +from research.finding_identity.tests.test_replay import _commit, _git + +V1 = "def handler(expr):\n try:\n return eval(expr)\n except:\n pass\n" +# Lines inserted above both findings. +V2 = "import os\n\n\n" + V1 +# eval() line edited; the bare except is untouched. +V3 = V2.replace("eval(expr)", "eval(expr.strip())") + + +def test_category_of() -> None: + assert category_of("CG-SEC-003") == "security" + assert category_of("CG-QUAL-002") == "quality" + assert category_of("NOT-A-RULE") == UNKNOWN + obs = [Observation("CG-SEC-003", 1, 0, {}), Observation("CG-QUAL-002", 2, 0, {})] + assert {k: len(v) for k, v in split_by_category(obs).items()} == { + "security": 1, + "quality": 1, + } + + +def test_replay_records_categories_that_add_up(tmp_path: Path) -> None: + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + _git(repo_dir, "init", "-q") + for text, msg in ((V1, "v1"), (V2, "v2"), (V3, "v3")): + _commit(repo_dir, text, msg) + + repo = GitRepo(repo_dir) + try: + summary = replay( + repo, + ref="HEAD", + commits=2, + scanner=Scanner(["CG-SEC-003", "CG-QUAL-002"], ALL_SCHEMES), + schemes=ALL_SCHEMES, + out_dir=tmp_path / "out", + label="tiny", + categories=["security", "quality", "performance"], + ) + finally: + repo.close() + + by_cat = summary["by_category"] + assert set(by_cat) == {"security", "quality", "performance"} + assert by_cat["performance"]["totals"]["gt"]["new"] == 0 + sec, qual = by_cat["security"], by_cat["quality"] + assert sec["totals"]["gt"]["known"] == 2 and qual["totals"]["gt"]["known"] == 2 + assert sec["totals"]["gt"]["known_edited"] == 1 and qual["totals"]["gt"]["known_edited"] == 0 + # line_text loses the edited eval() line, a security finding only. + assert sec["totals"]["schemes"]["line_text"]["false_new"] == 1 + assert qual["totals"]["schemes"]["line_text"]["false_new"] == 0 + + total = summary["totals"] + for key, value in total["gt"].items(): + assert value == sum(c["totals"]["gt"][key] for c in by_cat.values()) + for scheme, outcome_counts in total["schemes"].items(): + for key, value in outcome_counts.items(): + assert value == sum(c["totals"]["schemes"][scheme][key] for c in by_cat.values()) + assert summary["collisions"]["observations"] == sum( + c["collisions"]["observations"] for c in by_cat.values() + ) + on_disk = json.loads((tmp_path / "out" / "summary.json").read_text()) + assert on_disk["by_category"] == by_cat From b6a5905dc7bdfffc837d7a243aabd6cc26e866fc Mon Sep 17 00:00:00 2001 From: Vichitra Attri Date: Wed, 14 Oct 2026 12:43:07 +0000 Subject: [PATCH 2/5] feat(research): add corpus, batch runner and statistics for finding identity corpus.csv lists 36 open-source web applications (26 Python, 10 JavaScript/TypeScript), each pinned to a commit, with its licence, framework and why it qualifies. corpus.py pins rows to their branch head and re-checks the measurable selection criteria at the pinned commit from git alone: at least 200 first-parent commits in the two years before it, a recent pinned commit, a single OSI-approved root licence, and the declared back-end language as the majority of analysable files. The README records the criteria and the 24 rejected candidates with their reasons. `batch` replays every corpus repository at its pinned SHA in parallel, one replay process per repository. It resumes by skipping repositories whose summary already covers the pinned SHA and commit count, and logs failures to batch.jsonl and per-repository logs without stopping. `stats` reports, for all findings and for security findings, pooled rates with Wilson intervals, cluster-bootstrap intervals over repositories, macro-averaged rates, and the median and interquartile range of per-repository rates, as CSV and Markdown tables, with an optional box plot (matplotlib, listed in requirements.txt). Signed-off-by: Vichitra Attri --- research/finding_identity/.gitignore | 1 + research/finding_identity/README.md | 151 +++++- research/finding_identity/batch.py | 177 +++++++ research/finding_identity/cli.py | 8 +- research/finding_identity/commands.py | 134 +++++ research/finding_identity/corpus.csv | 37 ++ research/finding_identity/corpus.py | 311 ++++++++++++ research/finding_identity/requirements.txt | 3 + research/finding_identity/stats.py | 466 ++++++++++++++++++ research/finding_identity/tests/test_batch.py | 197 ++++++++ research/finding_identity/tests/test_stats.py | 245 +++++++++ 11 files changed, 1728 insertions(+), 2 deletions(-) create mode 100644 research/finding_identity/batch.py create mode 100644 research/finding_identity/commands.py create mode 100644 research/finding_identity/corpus.csv create mode 100644 research/finding_identity/corpus.py create mode 100644 research/finding_identity/requirements.txt create mode 100644 research/finding_identity/stats.py create mode 100644 research/finding_identity/tests/test_batch.py create mode 100644 research/finding_identity/tests/test_stats.py diff --git a/research/finding_identity/.gitignore b/research/finding_identity/.gitignore index 351a656..e7cf615 100644 --- a/research/finding_identity/.gitignore +++ b/research/finding_identity/.gitignore @@ -1,2 +1,3 @@ .repos/ results/ +tables/ diff --git a/research/finding_identity/README.md b/research/finding_identity/README.md index 6c915c5..4abc14d 100644 --- a/research/finding_identity/README.md +++ b/research/finding_identity/README.md @@ -70,7 +70,9 @@ python -m research.finding_identity replay https://github.com/DefectDojo/django- python -m research.finding_identity summarize research/finding_identity/results/* --csv rates.csv ``` -Clones go to `.repos/` and results to `results/