diff --git a/research/finding_identity/gt_agreement.py b/research/finding_identity/gt_agreement.py new file mode 100644 index 0000000..9aa616a --- /dev/null +++ b/research/finding_identity/gt_agreement.py @@ -0,0 +1,248 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Score the manual ground-truth check: how often the harness's truth holds, per rater, +and how well two raters agree. + +``python -m research.finding_identity.gt_agreement --human --agent `` +reads the key (``sample.csv`` from :mod:`gt_sample`) and each rater's labels, keyed by +the blinded item id, and writes ``agreement.md`` next to the key. + +Labels: a JSON object ``{item: {"verdict": ...}}``, or a directory of ``.json`` +files (as the labelling page's store exports them). A ``pair`` verdict of ``same`` and an +``introduced`` verdict of ``new`` confirm the harness's truth; ``different`` and +``existed`` contradict it; ``unclear`` counts for neither. + +``--adjudicated`` adds the final verdict per item (same format, each with a ``source`` +saying who settled it). With ``--results`` (the corpus replays the sample came from), +the report also estimates each content scheme's hidden rate after removing the +ground-truth errors: every hidden finding of those schemes lies in the +``new_but_matched`` stratum, so its count is scaled by the share of that scheme's +sampled matches that the adjudication confirms as truly new. +""" + +from __future__ import annotations + +import argparse +import csv +import json +from collections import Counter +from collections.abc import Mapping, Sequence +from pathlib import Path + +from .gt_sample import CONTENT_SCHEMES +from .stats import wilson + +HERE = Path(__file__).parent +KEY = HERE / "paper" / "ground_truth_check" / "sample.csv" + +CONFIRMS = {"same", "new"} +CONTRADICTS = {"different", "existed"} + + +def load_labels(path: Path) -> dict[str, str]: + """``{item: verdict}`` from a JSON file or a directory of per-item JSON files.""" + if path.is_dir(): + out = {} + for f in sorted(path.rglob("*.json")): + body = json.loads(f.read_text(encoding="utf-8")) + body = body.get("data", body) if isinstance(body, dict) else {} + if isinstance(body, dict) and isinstance(body.get("verdict"), str): + out[f.stem] = body["verdict"] + return out + raw = json.loads(path.read_text(encoding="utf-8")) + return {k: v["verdict"] for k, v in raw.items() if isinstance(v, dict) and "verdict" in v} + + +def judgement(verdict: str | None) -> str: + """``confirms``, ``contradicts`` or ``unclear`` (also for a missing label).""" + if verdict in CONFIRMS: + return "confirms" + if verdict in CONTRADICTS: + return "contradicts" + return "unclear" + + +def cohen_kappa(a: Sequence[str], b: Sequence[str]) -> float | None: + """Cohen's kappa for two equally long label sequences; ``None`` if undefined.""" + n = len(a) + if n == 0 or n != len(b): + return None + observed = sum(x == y for x, y in zip(a, b, strict=True)) / n + ca, cb = Counter(a), Counter(b) + expected = sum(ca[k] * cb[k] for k in ca.keys() | cb.keys()) / (n * n) + if expected == 1: + return None + return (observed - expected) / (1 - expected) + + +def precision_rows( + key: Sequence[Mapping[str, str]], labels: Mapping[str, str], group: str +) -> list[tuple[str, int, int, int]]: + """Per value of *group* (and overall): (name, confirms, contradicts, unclear).""" + groups: dict[str, Counter[str]] = {} + for row in key: + j = judgement(labels.get(row["item"])) + for name in (row[group], "all"): + groups.setdefault(name, Counter())[j] += 1 + names = [*sorted(n for n in groups if n != "all"), "all"] + return [ + (n, groups[n]["confirms"], groups[n]["contradicts"], groups[n]["unclear"]) for n in names + ] + + +def _kind(row: Mapping[str, str]) -> str: + return "introduced" if row["truth"] == "new" else "pair" + + +def report( + key: Sequence[Mapping[str, str]], + raters: Mapping[str, Mapping[str, str]], + *, + header: bool = True, +) -> str: + rows = [{**r, "kind": _kind(r)} for r in key] + out = ( + [ + "# Ground-truth check: results", + "", + "Generated by `research/finding_identity/gt_agreement.py`. *Holds* is the share of", + "decided items (unclear left out) where the rater confirms the harness's truth, with", + "a Wilson 95% interval.", + "", + ] + if header + else [] + ) + for name, labels in raters.items(): + labelled = sum(1 for r in rows if r["item"] in labels) + out += [f"## {name}", "", f"{labelled} of {len(rows)} items labelled.", ""] + for group in ("kind", "stratum"): + out += [ + f"| {group} | confirms | contradicts | unclear | holds | 95% CI |", + "|---|---:|---:|---:|---:|---|", + ] + for g, yes, no, unclear in precision_rows(rows, labels, group): + ci = wilson(yes, yes + no) + rate = f"{100 * yes / (yes + no):.1f}%" if yes + no else "n/a" + cis = f"[{100 * ci[0]:.1f}, {100 * ci[1]:.1f}]" if ci else "n/a" + out.append(f"| {g} | {yes} | {no} | {unclear} | {rate} | {cis} |") + out.append("") + names = list(raters) + if len(names) == 2: + both = [r for r in rows if r["item"] in raters[names[0]] and r["item"] in raters[names[1]]] + a = [judgement(raters[names[0]][r["item"]]) for r in both] + b = [judgement(raters[names[1]][r["item"]]) for r in both] + kappa = cohen_kappa(a, b) + agree = sum(x == y for x, y in zip(a, b, strict=True)) + out += [ + f"## Agreement between {names[0]} and {names[1]}", + "", + f"On the {len(both)} items both labelled (confirms / contradicts / unclear):", + f"{agree} agree ({100 * agree / len(both):.1f}%), Cohen's kappa " + + (f"{kappa:.2f}." if kappa is not None else "undefined.") + if both + else "No item is labelled by both.", + "", + ] + disputed = [r["item"] for r, x, y in zip(both, a, b, strict=True) if x != y] + if disputed: + out += ["Items to adjudicate: " + ", ".join(disputed) + ".", ""] + return "\n".join(out) + + +def corrected_hidden( + key: Sequence[Mapping[str, str]], adjudicated: Mapping[str, str], results: Path +) -> list[str]: + """Markdown lines: each content scheme's hidden rate before and after correction.""" + hidden: Counter[str] = Counter() + introduced = 0 + for f in sorted(results.glob("*/findings.jsonl")): + with f.open(encoding="utf-8") as fh: + for line in fh: + rec = json.loads(line) + if rec["truth"] != "new": + continue + introduced += 1 + for scheme in CONTENT_SCHEMES: + hidden[scheme] += rec["outcomes"][scheme] == "false_known" + sample = [r for r in key if r["stratum"] == "new_but_matched"] + out = [ + "## Hidden rates after correcting the ground truth", + "", + f"From `{results}`: {introduced:,} introduced findings. *Matched, sampled*: the", + "scheme's hidden findings among the sampled `new_but_matched` items; *truly new*:", + "those the adjudication confirms as new. The corrected rate scales the hidden", + "count by that share (range: its Wilson 95% interval).", + "", + "| scheme | hidden | matched, sampled | truly new | corrected hidden | range |", + "|---|---:|---:|---:|---:|---|", + ] + for scheme in CONTENT_SCHEMES: + rows = [r for r in sample if scheme in r["disagreeing_schemes"].split()] + new = sum(judgement(adjudicated.get(r["item"])) == "confirms" for r in rows) + ci = wilson(new, len(rows)) + rate = hidden[scheme] / introduced if introduced else 0.0 + if not rows or ci is None: + out.append(f"| `{scheme}` | {100 * rate:.2f}% | 0 | 0 | n/a | n/a |") + continue + corrected = rate * new / len(rows) + out.append( + f"| `{scheme}` | {100 * rate:.2f}% ({hidden[scheme]}) | {len(rows)} | {new} | " + f"{100 * corrected:.2f}% | [{100 * rate * ci[0]:.2f}, {100 * rate * ci[1]:.2f}] |" + ) + return [*out, ""] + + +def main(argv: list[str] | None = None) -> None: + ap = argparse.ArgumentParser(prog="python -m research.finding_identity.gt_agreement") + ap.add_argument("--key", type=Path, default=KEY, help="sample.csv from gt_sample") + ap.add_argument("--human", type=Path, help="the person's labels (file or directory)") + ap.add_argument( + "--agent", + type=Path, + action="append", + help="the agent rater's labels (file or directory); repeat it, and later files " + "override earlier ones, item by item", + ) + ap.add_argument( + "--human-kind", + choices=["pair", "introduced"], + help="score only the person's labels of this item kind", + ) + ap.add_argument("--adjudicated", type=Path, help="final verdicts, with their sources") + ap.add_argument("--results", type=Path, help="the replays the sample came from") + ap.add_argument("--out", type=Path, help="default: agreement.md next to the key") + args = ap.parse_args(argv) + with args.key.open(newline="", encoding="utf-8") as f: + key = list(csv.DictReader(f)) + raters = {} + if args.human: + human = load_labels(args.human) + if args.human_kind: + kinds = {r["item"]: _kind(r) for r in key} + human = {k: v for k, v in human.items() if kinds.get(k) == args.human_kind} + raters["Human rater"] = human + if args.agent: + agent: dict[str, str] = {} + for path in args.agent: + agent.update(load_labels(path)) + raters["Agent rater"] = agent + if not raters: + raise SystemExit("give --human and/or --agent") + text = report(key, raters) + if args.adjudicated: + final = load_labels(args.adjudicated) + raw = json.loads(args.adjudicated.read_text(encoding="utf-8")) + sources = Counter(str(v.get("source", "unknown")) for v in raw.values()) + text += "\n" + report(key, {"Adjudicated": final}, header=False) + text += "\nSources of the adjudicated verdicts:\n\n" + text += "\n".join(f"- {n}: {src}" for src, n in sources.most_common()) + "\n" + if args.results: + text += "\n" + "\n".join(corrected_hidden(key, final, args.results)) + out = args.out or args.key.parent / "agreement.md" + out.write_text(text.rstrip("\n") + "\n", encoding="utf-8") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/research/finding_identity/gt_sample.py b/research/finding_identity/gt_sample.py index ea2b22b..b16f0a5 100644 --- a/research/finding_identity/gt_sample.py +++ b/research/finding_identity/gt_sample.py @@ -16,7 +16,13 @@ It writes ``sample.md``, one entry per finding with the code before and after the commit and a link to the commit, and ``sample.csv`` with an empty ``verdict`` column to fill in: ``same`` (the old and new finding are the same issue), ``different``, or ``unclear``. -The sample is seeded, so rerunning it gives the same findings. + +For independent raters it also writes ``items.json``: the same findings, shuffled under +neutral ids (``sample.csv``'s ``item`` column is the key), without the harness's answer, +the stratum or the schemes' outcomes. A persisting finding becomes a ``pair`` item ("is +the code flagged at the old line the same issue as at the new line?"), an introduced one +an ``introduced`` item ("did this issue already exist before the commit?"). +The sample is seeded, so rerunning it gives the same findings and the same order. """ from __future__ import annotations @@ -30,7 +36,12 @@ from pathlib import Path from typing import Any +import codeguard.rules # noqa: F401 (registers the built-in rules) +from codeguard.engine.registry import REGISTRY + from .corpus import load_corpus +from .linemap import map_lines +from .schemes import split_lines HERE = Path(__file__).parent @@ -69,15 +80,42 @@ def _git(repo: Path, *args: str) -> str: return done.stdout.decode("utf-8", "replace") if done.returncode == 0 else "" -def snippet(repo: Path, commit: str, path: str, line: int, context: int = 4) -> str: - """Lines around *line* of *path* at *commit*, numbered, the line itself marked.""" +def snippet_lines( + repo: Path, commit: str, path: str, line: int, context: int = 4 +) -> list[tuple[int, str]]: + """``(line number, text)`` for the lines around *line* of *path* at *commit*.""" lines = _git(repo, "show", f"{commit}:{path}").split("\n") lo, hi = max(1, line - context), min(len(lines), line + context) + return [(n, lines[n - 1].rstrip()) for n in range(lo, hi + 1)] + + +def snippet(repo: Path, commit: str, path: str, line: int, context: int = 4) -> str: + """Lines around *line* of *path* at *commit*, numbered, the line itself marked.""" return "\n".join( - f"{'>' if n == line else ' '}{n:5d} {lines[n - 1]}" for n in range(lo, hi + 1) + f"{'>' if n == line else ' '}{n:5d} {text}" + for n, text in snippet_lines(repo, commit, path, line, context) ) +def corresponding_old_line( + old_lines: list[str], new_lines: list[str], line: int +) -> tuple[int, bool]: + """The old line at the place of new line *line*, and whether *line* itself maps to it. + + Uses the ground truth's line map: if *line* maps back to an old line (unchanged or + edited), that line; otherwise the line after the nearest old line that maps to a new + line above *line* (the point where the new code was inserted). + """ + back = {new: old for old, (new, _) in map_lines(old_lines, new_lines).items()} + if line in back: + return back[line], True + above = [n for n in back if n < line] + if not above: + return 1, False + anchor = max(above) + return min(back[anchor] + 1, max(len(old_lines), 1)), False + + def diff_hunks(repo: Path, old: str, new: str, path: str, old_path: str, line: int) -> str: """The hunks of the commit's diff of this file that come within 6 lines of *line*.""" paths = [path] if path == old_path else [old_path, path] @@ -164,6 +202,12 @@ def write_sample( "", ] rows = [] + items = [] + titles = {r.id: r.title for r in REGISTRY.all()} + # Neutral item ids in a seeded order unrelated to the strata. + order = list(range(1, len(chosen) + 1)) + random.Random(len(chosen)).shuffle(order) # noqa: S311 (blinding order, not security) + item_of = {i: f"F{position:03d}" for position, i in enumerate(order, start=1)} for i, (stratum, label, rec, old) in enumerate(chosen, start=1): repo = repos / label new = rec["commit"] @@ -191,9 +235,44 @@ def write_sample( if rec["truth"] == "new" or rec.get("truth_kind") == "edited": hunks = diff_hunks(repo, old, new, path, old_path, rec["line"]) md += ["Diff near the line:", "", "```diff", hunks, "```", ""] + old_line = rec.get("old_line") + if old_line: + before_line, before_mapped = old_line, True + else: + # An introduced finding: show the old code at the same place, so a rater can + # see whether the issue was already there. + old_text = split_lines(_git(repo, "show", f"{old}:{old_path}")) + new_text = split_lines(_git(repo, "show", f"{new}:{path}")) + before_line, before_mapped = corresponding_old_line(old_text, new_text, rec["line"]) + items.append( + { + "item": item_of[i], + "kind": "pair" if rec["truth"] == "known" else "introduced", + "repository": label, + "commit_url": url, + "rule": rec["rule"], + "rule_title": titles.get(rec["rule"], ""), + "path": path, + "old_path": old_path, + "old_line": old_line, + "line": rec["line"], + "before": snippet_lines( + repo, old, old_path, before_line, context=4 if old_line else 6 + ), + "before_line": before_line, + # True when the flagged line itself was already there before the commit. + "before_mapped": before_mapped, + "after": snippet_lines(repo, new, path, rec["line"]), + "diff": "\n".join( + row.rstrip() + for row in diff_hunks(repo, old, new, path, old_path, rec["line"]).split("\n") + ), + } + ) rows.append( { "id": i, + "item": item_of[i], "stratum": stratum, "repository": label, "commit": new, @@ -210,6 +289,8 @@ def write_sample( # Quoted source can end lines in spaces; the repository's hooks strip them. text = "\n".join(row.rstrip() for row in "\n".join(md).split("\n")) (out / "sample.md").write_text(text.rstrip("\n") + "\n", encoding="utf-8") + items.sort(key=lambda it: str(it["item"])) + (out / "items.json").write_text(json.dumps(items, indent=1) + "\n", encoding="utf-8") with (out / "sample.csv").open("w", newline="", encoding="utf-8") as f: w = csv.DictWriter(f, fieldnames=list(rows[0])) w.writeheader() diff --git a/research/finding_identity/paper/claims.md b/research/finding_identity/paper/claims.md index a23fe0e..6ae718a 100644 --- a/research/finding_identity/paper/claims.md +++ b/research/finding_identity/paper/claims.md @@ -47,7 +47,7 @@ commit pairs per repository, replayed on 2026-10-17. | C8 | Edited lines are where every scheme fails | false-new on the 142 edited-line findings: `line_text` 100, `github` 87.3, `codeguard` 74.6, `sonar` 63.4, `bandit` 23.9 (Bandit ignores location and code) | `pooled.csv`, `false_new_edited` | | C9 | **Set membership hides new findings** | Bandit's baseline on Bandit findings without B101: 29.2 (20.1–40.9) hidden, 320 of 1,097; with B101: 51.2 (42.5–65.2), 6,670/13,024; on CodeGuard findings: 20.0 (14.1–28.6), 394/1,974 | `table_rq1.tex`, `numbers.md` | | C9b | It is not a test-fixture effect | without B101, outside test code: 36.3% of 273 introduced findings in 22 repositories hidden (in test code: 26.8% of 824). Other schemes outside test code: `github` 1.8, `codeguard` 2.9, `sonar` 5.9. No interval: a sensitivity check (see note C) | note C | -| C10 | Every other scheme hides far fewer | on Bandit findings without B101: `line` 1.9, `codeguard` 2.5, `github` 3.3, `sonar` 4.5, `line_text` 6.0; on CodeGuard findings 0.8–1.9 | `table_rq1.tex` | +| C10 | Every other scheme hides far fewer | on Bandit findings without B101: `line` 1.9, `codeguard` 2.5, `github` 3.3, `sonar` 4.5, `line_text` 6.0; on CodeGuard findings 0.8–1.9. **Upper bounds**: see G4 | `table_rq1.tex` | | C11 | Schemes that survive more edits collide more | collision rate, CodeGuard findings: `github` 2.1 (1.1–3.5), `line_text` 17.4 (14.0–21.3), `codeguard` 18.6 (12.9–24.8), `sonar` 20.3 (16.5–24.4), `bandit` 54.2 (44.4–62.5) | `table_rq1.tex` | | C12 | The pooled rate hides a skewed spread; one repository dominates `codeguard`'s rate | Ghost: `codeguard` 19.9% of 687 persisting findings (renames from `.js` to `.ts`), `sonar` 1.0, `sonar_branch` 1.7. Per-repository median of `codeguard`: 0.48 | `per_repo.csv`, `numbers.md` | | C13 | Findings within a repository are correlated, so per-finding intervals are too narrow | `line_text` false-new 1.89: Wilson 1.73–2.08 against cluster bootstrap 0.80–3.51 | `pooled.csv` | @@ -56,6 +56,34 @@ Caution when pairing C11 with hidden rates: `codeguard`'s hidden rate (1.9, 0.9 `github`'s (0.8, 0.2–1.9) have overlapping intervals, so collisions cost "tends to hide more", not "hides more". +## Ground-truth check + +A manual check on 100 sampled findings; the procedure, including a flaw in its first +round, is in `ground_truth_check/README.md`, the scores in `ground_truth_check/agreement.md`. + +| # | Claim | Value | Source | +|---|---|---|---| +| G1 | The line-diff ground truth is reliable for persisting findings | holds for 59 of 60 sampled pairs: 98.3% (Wilson 91.1–99.7); human and LLM rater agree on 59 of 60 | `agreement.md` | +| G2 | It is weaker for introduced findings | holds for 30 of 40: 75.0% (59.8–85.8) | `agreement.md`, *Adjudicated* | +| G3 | Its errors sit where a scheme matched an "introduced" finding | `new_but_matched`: holds for 10 of 20, 50.0% (29.9–70.1); `agree_new`: 20 of 20. The errors are findings that persisted while the line diff lost them: the flagged line was edited past the similarity threshold (an initializer, a reworded message, a declaration joined onto one line) or moved out of a removed block | `agreement.md`, `adjudicated.json` | +| G4 | So the hidden rates of the content schemes are upper bounds, SonarQube's most | corrected hidden rate on CodeGuard findings: `sonar` 1.47 → 0.24 (range 0.07–0.66), `github` 0.81 → 0.41, `line_text` 1.17 → 0.87, `codeguard` 1.87 → 1.31. SonarQube's line-hash and code-move passes follow lines the line diff loses: 10 of the 12 sampled findings it "hid" were persisting | `agreement.md`, *Hidden rates after correcting* | +| G5 | Bandit's hidden rate barely moves | of its 394 hidden CodeGuard findings, 339 are in `agree_new`, where the ground truth held 20 of 20; at most the other 55 could be ground-truth errors (20.0% would fall to no less than about 17%) | `agreement.md`; stratum counts from `results/corpus-v3` | +| G6 | Threshold rules make "same finding" a question about the analyzer, not the line | three sampled complexity and nesting findings sit on an unchanged `def` line yet are new: the commit pushed the function over the limit elsewhere (verified by re-running the rule on both versions) | `adjudicated.json` (F036, F065, F076) | + +How to describe the check: the human rater labelled the 60 pairs and confirmed each of +the 14 harness errors the LLM rater flagged; the 27 introduced findings the LLM rater +confirmed as new had no human check. Report the LLM rater as an LLM. Cohen's kappa is +not informative here (the human rater confirmed every pair, so chance agreement is +total); report raw agreement instead. + +Scope: the check sampled CodeGuard findings only. Bandit's own findings (C9's 29.2%, +C10's Bandit-findings column) were not checked, so say that their hidden rates rest on +the same line diff without a manual check. + +Caution on G4: the corrected rates scale each scheme's hidden count by the share of its +sampled matches that are truly new, from 6 to 12 sampled findings per scheme. Quote them +as estimates with their ranges, and keep the uncorrected rates in Table II. + ## RQ2: controlled changes Setup: ten transformations, each applied to one target finding in each of 40 sampled @@ -94,8 +122,6 @@ D. **R4** counts scored `rename_function` cases by rule from `results/transform/*/cases.jsonl` (`target_outcomes.line_text == "false_new"`). E. **Still to do before submission.** -- Manual check of the ground truth on a stratified sample (for Threats): a sample file - for labelling can be generated from `findings.jsonl`. - Verify every related-work reference in `outline.md` before citing it. - Decide whether `sonar` or `sonar_branch` is "SonarQube" in the abstract. Branch mode is what compares a long-lived baseline on the main branch; pull-request mode is what diff --git a/research/finding_identity/paper/ground_truth_check/README.md b/research/finding_identity/paper/ground_truth_check/README.md new file mode 100644 index 0000000..45e9f94 --- /dev/null +++ b/research/finding_identity/paper/ground_truth_check/README.md @@ -0,0 +1,72 @@ +# Ground-truth check + +A manual check of the harness's ground truth (the line diff of `linemap.py`) on a +seeded, stratified sample of 100 CodeGuard findings from the corpus replays +(`results/corpus-v3`), drawn by `gt_sample.py`: 20 in each of five strata. + +| Stratum | What the harness says | Sampled from | +|---|---|---| +| `edited` | persisting, on an edited line | persisting findings whose line was edited | +| `known_but_new` | persisting, unchanged line, same path | persisting findings that SonarQube or CodeGuard report as new | +| `new_but_matched` | introduced | introduced findings that a content scheme matched to an old one (57 in the corpus) | +| `agree_known` | persisting | every content scheme agrees (control) | +| `agree_new` | introduced | every content scheme agrees (control; 1,917 in the corpus) | + +## Files + +- `sample.csv`: the key. One row per finding with the stratum, the harness's answer + (`truth`) and the content schemes that disagree with it; `item` links to the blinded + items. +- `sample.md`: the same findings for reading, with the harness's answer shown. +- `items.json`: the blinded items the raters saw: shuffled under neutral ids, without the + harness's answer, the stratum or any scheme's outcome. +- `human_labels_round1.json`: the human rater's first pass, all 100 items. +- `agent_labels_round1.json`, `agent_labels_introduced_round2.json`: the agent rater's + labels, with a one-sentence reason each. +- `human_adjudication.json`: the human rater's answers on the 14 items the agent flagged + as harness errors. +- `adjudicated.json`: the final verdict per item and where it came from (`source`). +- `agreement.md`: the scores, from `gt_agreement.py`. + +## How it was done + +1. **Round 1.** The human rater labelled all 100 items on a labelling page, and an LLM + agent labelled them independently, reading only `items.json`. Neither saw the other's + labels or the harness's answer. +2. **A flaw in round 1.** For the 40 introduced items the page and `items.json` showed no + code from before the commit, only the diff, which was empty near the line for many of + them. Neither rater could judge whether those issues existed before. The human + rater's round-1 labels on introduced items are therefore not used; the 60 pair labels + are. +3. **Round 2.** `items.json` now gives each introduced item the old code at the same + place (found with the line map). The agent relabelled the 40 introduced items from it, + with instructions that no longer assumed an untouched line means an issue existed. +4. **Human confirmation.** The human rater checked each of the 14 items where the agent + said the harness was wrong (13 introduced items it called "existed" and one pair it + called "different"), seeing the agent's reason, and agreed on all 14. +5. **Rule re-runs.** Three of those 14 (F036, F065, F076) are whole-function rules + (complexity, nesting) whose flagged line was unchanged. Re-running CodeGuard on the + file before and after the commit shows the function was not reported before: the + commit pushed it over the limit, so the finding is new and the harness was right. + Both raters had judged from an excerpt that did not show the changed lines. + +The 27 introduced items the agent labelled "new" in round 2 had no human check. + +## Result + +The harness's ground truth holds for 59 of 60 pairs (98.3%, Wilson 95% CI 91.1–99.7) +and 30 of 40 introduced items (75.0%, 59.8–85.8). The errors concentrate in +`new_but_matched`: 10 of 20 (29.9–70.1) are findings that persisted while the line diff +lost them (a line edited past the similarity threshold, or moved out of a removed block), +so the scheme that matched them was right. `agreement.md` has the corrected hidden rates +per scheme. + +Regenerate the scores with: + +```bash +G=research/finding_identity/paper/ground_truth_check +python -m research.finding_identity.gt_agreement --human $G/human_labels_round1.json \ + --human-kind pair --agent $G/agent_labels_round1.json \ + --agent $G/agent_labels_introduced_round2.json --adjudicated $G/adjudicated.json \ + --results research/finding_identity/results/corpus-v3 +``` diff --git a/research/finding_identity/paper/ground_truth_check/adjudicated.json b/research/finding_identity/paper/ground_truth_check/adjudicated.json new file mode 100644 index 0000000..721fd33 --- /dev/null +++ b/research/finding_identity/paper/ground_truth_check/adjudicated.json @@ -0,0 +1,402 @@ +{ + "F001": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F002": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F003": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F004": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F005": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F006": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F007": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F008": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F009": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F010": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F011": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F012": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F013": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F014": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F015": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F016": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F017": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F018": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F019": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F020": { + "source": "human confirmed the agent rater", + "verdict": "existed" + }, + "F021": { + "source": "human confirmed the agent rater", + "verdict": "different" + }, + "F022": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F023": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F024": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F025": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F026": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F027": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F028": { + "source": "human confirmed the agent rater", + "verdict": "existed" + }, + "F029": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F030": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F031": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F032": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F033": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F034": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F035": { + "source": "human confirmed the agent rater", + "verdict": "existed" + }, + "F036": { + "source": "rule re-run: createUser() reaches cyclomatic complexity 16 (limit 15) only after the commit; CodeGuard did not report it before", + "verdict": "new" + }, + "F037": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F038": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F039": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F040": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F041": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F042": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F043": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F044": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F045": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F046": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F047": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F048": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F049": { + "source": "human confirmed the agent rater", + "verdict": "existed" + }, + "F050": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F051": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F052": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F053": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F054": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F055": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F056": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F057": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F058": { + "source": "human confirmed the agent rater", + "verdict": "existed" + }, + "F059": { + "source": "human confirmed the agent rater", + "verdict": "existed" + }, + "F060": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F061": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F062": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F063": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F064": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F065": { + "source": "rule re-run: the test function reaches cyclomatic complexity 18 (limit 15) only after the commit; CodeGuard did not report it before", + "verdict": "new" + }, + "F066": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F067": { + "source": "human confirmed the agent rater", + "verdict": "existed" + }, + "F068": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F069": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F070": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F071": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F072": { + "source": "human confirmed the agent rater", + "verdict": "existed" + }, + "F073": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F074": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F075": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F076": { + "source": "rule re-run: raise_for_response() reaches nesting depth 6 (limit 5) only after the commit; CodeGuard did not report it before", + "verdict": "new" + }, + "F077": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F078": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F079": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F080": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F081": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F082": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F083": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F084": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F085": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F086": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F087": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F088": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F089": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F090": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F091": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F092": { + "source": "human confirmed the agent rater", + "verdict": "existed" + }, + "F093": { + "source": "human confirmed the agent rater", + "verdict": "existed" + }, + "F094": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F095": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F096": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F097": { + "source": "human and agent rater agree", + "verdict": "same" + }, + "F098": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F099": { + "source": "agent rater only (round 2)", + "verdict": "new" + }, + "F100": { + "source": "human and agent rater agree", + "verdict": "same" + } +} diff --git a/research/finding_identity/paper/ground_truth_check/agent_labels_introduced_round2.json b/research/finding_identity/paper/ground_truth_check/agent_labels_introduced_round2.json new file mode 100644 index 0000000..3fbc2d6 --- /dev/null +++ b/research/finding_identity/paper/ground_truth_check/agent_labels_introduced_round2.json @@ -0,0 +1,162 @@ +{ + "F001": { + "reason": "The flagged await-in-loop is in a newly added test; the older test's loop is a separate, still-present occurrence.", + "verdict": "new" + }, + "F007": { + "reason": "The loose comparison is in a newly added nested-row fetch callback that did not exist before.", + "verdict": "new" + }, + "F008": { + "reason": "The file is newly added, so the await-in-loop retry code is new.", + "verdict": "new" + }, + "F014": { + "reason": "'var sign' is a newly added declaration; the old block only declared 'var diff'.", + "verdict": "new" + }, + "F015": { + "reason": "The MOCK_PASS meetingPassword is in a test block added where the old file had only a comment.", + "verdict": "new" + }, + "F020": { + "reason": "'var loc' was already a var declaration; only its initializer changed.", + "verdict": "existed" + }, + "F024": { + "reason": "The credentials line is in a newly added _get method; the existing _post copy remains separately.", + "verdict": "new" + }, + "F027": { + "reason": "The f-string DROP TRIGGER execute is in a newly added test class.", + "verdict": "new" + }, + "F028": { + "reason": "The same errors.password string assignment existed before, only reworded and joined onto one line.", + "verdict": "existed" + }, + "F035": { + "reason": "The collectionType == \"REST\" ternary for root collections existed before and was only restructured.", + "verdict": "existed" + }, + "F036": { + "reason": "The createUser signature is unchanged, there is no diff near it and line numbers are identical, so nothing shows the complexity changed.", + "verdict": "existed" + }, + "F041": { + "reason": "'var card' is in the rewritten updatePermissions done-callback, which is new logic built around byNodegroup.", + "verdict": "new" + }, + "F042": { + "reason": "The file is newly added, so the f-string DROP TRIGGER statements appear for the first time.", + "verdict": "new" + }, + "F049": { + "reason": "The same await inside the for loop existed before, just inside the removed withChildren block (re-indented).", + "verdict": "existed" + }, + "F051": { + "reason": "The file is newly added, so the innerHTML template assignment is new.", + "verdict": "new" + }, + "F052": { + "reason": "'var Tokenizer' is part of the newly added select2 v4 data-adapter code.", + "verdict": "new" + }, + "F054": { + "reason": "The password tuple from create_non_staff_user is in a newly added test class.", + "verdict": "new" + }, + "F056": { + "reason": "The file is newly added, so the await nextTick() in the loop is new.", + "verdict": "new" + }, + "F058": { + "reason": "'var loc' was already a var declaration; only its initializer changed.", + "verdict": "existed" + }, + "F059": { + "reason": "The await personRepository.update call was already inside the same loop, only moved out of an if block.", + "verdict": "existed" + }, + "F065": { + "reason": "The added lines are straight-line awaits with no branches, so the complexity condition must already have held.", + "verdict": "existed" + }, + "F067": { + "reason": "The await readJson inside the for loop existed before; it was only reformatted with a type parameter.", + "verdict": "existed" + }, + "F069": { + "reason": "'var siteLanguage' is a newly added declaration.", + "verdict": "new" + }, + "F070": { + "reason": "The create_key call is in a newly added test method.", + "verdict": "new" + }, + "F072": { + "reason": "The same var declaration list existed before, only merged onto one line.", + "verdict": "existed" + }, + "F073": { + "reason": "The localCollections ternary with == is a new comparison; the old one in runDispatchWithOutSyncing is still present.", + "verdict": "new" + }, + "F074": { + "reason": "The file is newly added, so the await axios.post in the loop is new.", + "verdict": "new" + }, + "F076": { + "reason": "raise_for_response is unchanged with no diff nearby, so its nesting depth predates the commit.", + "verdict": "existed" + }, + "F077": { + "reason": "'var config' is new; the old line 7 called selectize directly with no var declaration.", + "verdict": "new" + }, + "F078": { + "reason": "The file is a newly added translation file.", + "verdict": "new" + }, + "F081": { + "reason": "The 'starfleet' == comparison is a newly added branch; the azure comparison it displaced still exists below.", + "verdict": "new" + }, + "F083": { + "reason": "The file is newly added, so preview_command and its complexity are new.", + "verdict": "new" + }, + "F086": { + "reason": "The file is newly added, so the await inside the loop is new.", + "verdict": "new" + }, + "F092": { + "reason": "The admin seed user with the hardcoded password looks relocated: about 247 lines were removed above it and the block reads as existing seed code, although the old copy is not shown.", + "verdict": "existed" + }, + "F093": { + "reason": "The same password-message assertion existed before, only reworded.", + "verdict": "existed" + }, + "F094": { + "reason": "start_wasm_invocation, with its long parameter list, is a newly added function.", + "verdict": "new" + }, + "F095": { + "reason": "'var self = this;' is a newly added line.", + "verdict": "new" + }, + "F096": { + "reason": "The mock access/refresh tokens are in a newly added office365 test.", + "verdict": "new" + }, + "F098": { + "reason": "'var perms' is in the newly rewritten updatePermissions callback.", + "verdict": "new" + }, + "F099": { + "reason": "'var lastSyncedCardSelection' is a newly added declaration.", + "verdict": "new" + } +} diff --git a/research/finding_identity/paper/ground_truth_check/agent_labels_round1.json b/research/finding_identity/paper/ground_truth_check/agent_labels_round1.json new file mode 100644 index 0000000..f09548e --- /dev/null +++ b/research/finding_identity/paper/ground_truth_check/agent_labels_round1.json @@ -0,0 +1,402 @@ +{ + "F001": { + "verdict": "new", + "reason": "The await-in-loop line is a + line in a newly added test." + }, + "F002": { + "verdict": "same", + "reason": "Same function definition, only shifted down; no diff near it." + }, + "F003": { + "verdict": "same", + "reason": "Identical += line, only shifted; no diff near it." + }, + "F004": { + "verdict": "same", + "reason": "Same referrer redirect moved into a new inner function." + }, + "F005": { + "verdict": "same", + "reason": "Same loose-equality condition line, edited only in the role comparison." + }, + "F006": { + "verdict": "same", + "reason": "Same var self in the revert method, function shifted by the commit." + }, + "F007": { + "verdict": "new", + "reason": "The loose comparison is inside a large block of + lines adding nested-row loading." + }, + "F008": { + "verdict": "new", + "reason": "The whole file is added by this commit." + }, + "F009": { + "verdict": "same", + "reason": "Same open(temp_filepath) write, re-indented into a try block." + }, + "F010": { + "verdict": "same", + "reason": "Identical empty catch, only shifted; no diff near it." + }, + "F011": { + "verdict": "same", + "reason": "Same function at the same line, unchanged." + }, + "F012": { + "verdict": "same", + "reason": "The after-filter awaited count checks in the loop were collapsed into one awaited call replacing the flagged line." + }, + "F013": { + "verdict": "same", + "reason": "Same == comparison on the not_found error, file moved and statement rewritten as an if." + }, + "F014": { + "verdict": "new", + "reason": "var sign is a + line added in the new rounding logic." + }, + "F015": { + "verdict": "new", + "reason": "The MOCK_PASS line is within a block of + lines adding a new test." + }, + "F016": { + "verdict": "same", + "reason": "Same var me line at the same position, unchanged." + }, + "F017": { + "verdict": "same", + "reason": "Same var i line, only reformatted." + }, + "F018": { + "verdict": "same", + "reason": "Same function definition, only shifted." + }, + "F019": { + "verdict": "same", + "reason": "Same method definition, shifted by one line." + }, + "F020": { + "verdict": "existed", + "reason": "The var loc declaration existed before; only its initializer was edited." + }, + "F021": { + "verdict": "different", + "reason": "The old per-test document.body.innerHTML setup was replaced by a new shared helper writing to a created div." + }, + "F022": { + "verdict": "same", + "reason": "Same method definition, only shifted." + }, + "F023": { + "verdict": "same", + "reason": "Same feed function; only its signature lost a parameter." + }, + "F024": { + "verdict": "new", + "reason": "The credentials unpacking is in a newly added _get method (+ lines), distinct from the existing _post occurrence." + }, + "F025": { + "verdict": "same", + "reason": "Same hardcoded test password constant; only its value changed." + }, + "F026": { + "verdict": "same", + "reason": "Same new_secret literal, shifted by one line." + }, + "F027": { + "verdict": "new", + "reason": "The f-string execute is in a newly added test class." + }, + "F028": { + "verdict": "existed", + "reason": "The errors.password message assignment existed before, only reworded and joined onto one line." + }, + "F029": { + "verdict": "same", + "reason": "Same create_key line, shifted." + }, + "F030": { + "verdict": "same", + "reason": "Same var geoJSONString line, shifted." + }, + "F031": { + "verdict": "same", + "reason": "Same var title line at the same position." + }, + "F032": { + "verdict": "same", + "reason": "Same var parts line, shifted by one." + }, + "F033": { + "verdict": "same", + "reason": "Same hardcoded test password, shifted." + }, + "F034": { + "verdict": "same", + "reason": "Same test function definition at the same line." + }, + "F035": { + "verdict": "existed", + "reason": "The collection-branch collectionType == \"REST\" ternary was carried over from the moved file's rewritten subscription handler, which already had the same REST/GraphQL dispatch." + }, + "F036": { + "verdict": "existed", + "reason": "No diff near the createUser definition, so the function existed unchanged at this position." + }, + "F037": { + "verdict": "same", + "reason": "Same tokenNamePrefix literal in the same test, shifted." + }, + "F038": { + "verdict": "same", + "reason": "Same hardcoded test password, shifted." + }, + "F039": { + "verdict": "same", + "reason": "Same except/pass at the same line, unchanged." + }, + "F040": { + "verdict": "same", + "reason": "The Manufacture else-if comparison is an unchanged context line, shifted by one." + }, + "F041": { + "verdict": "new", + "reason": "The var card lookup is within the rewritten updatePermissions + lines." + }, + "F042": { + "verdict": "new", + "reason": "The whole file is added by this commit." + }, + "F043": { + "verdict": "same", + "reason": "Same data != \"null\" check, shifted." + }, + "F044": { + "verdict": "same", + "reason": "Same role comparison in the selection loop, edited to normalise nulls." + }, + "F045": { + "verdict": "same", + "reason": "Same merge of req.body into the checklist item, shifted by one." + }, + "F046": { + "verdict": "same", + "reason": "Same status != COMPLETE ternary, shifted." + }, + "F047": { + "verdict": "same", + "reason": "Same purchaseHourglass function; only a parameter was removed." + }, + "F048": { + "verdict": "same", + "reason": "Same hardcoded admin password at the same line." + }, + "F049": { + "verdict": "existed", + "reason": "The awaited recursive export in the loop was moved out of the removed withChildren block, de-indented." + }, + "F050": { + "verdict": "same", + "reason": "Same f-string INSERT in the test, shifted." + }, + "F051": { + "verdict": "new", + "reason": "The whole file is added by this commit." + }, + "F052": { + "verdict": "new", + "reason": "var Tokenizer is part of the new select2 data-adapter code added by the commit." + }, + "F053": { + "verdict": "same", + "reason": "Same var declaration of the tags widget, only renamed for tomselect." + }, + "F054": { + "verdict": "new", + "reason": "The line is in a newly added test class." + }, + "F055": { + "verdict": "same", + "reason": "Same awaited migration.migrate() in the migrations loop, carried into the rewritten runMigrations." + }, + "F056": { + "verdict": "new", + "reason": "The whole test file is added by this commit." + }, + "F057": { + "verdict": "same", + "reason": "Same MOCK_PASSWORD literal, shifted." + }, + "F058": { + "verdict": "existed", + "reason": "The var loc declaration existed before; only its initializer was edited." + }, + "F059": { + "verdict": "existed", + "reason": "The awaited update in the loop was moved out of the removed if-block, de-indented." + }, + "F060": { + "verdict": "same", + "reason": "Same innerHTML copy of the document container; only the element variable was renamed." + }, + "F061": { + "verdict": "same", + "reason": "Same MOCK_PASS literal in the same test, shifted." + }, + "F062": { + "verdict": "same", + "reason": "Same hardcoded test password, shifted." + }, + "F063": { + "verdict": "same", + "reason": "Same loose comparison against node.nodeid; only the loop variable was renamed." + }, + "F064": { + "verdict": "same", + "reason": "Same MOCK_PASS literal just after the rewritten hunk, shifted by the same offset as neighbouring lines." + }, + "F065": { + "verdict": "existed", + "reason": "The flagged function definition is an unchanged context line; the commit only added straight-line setup calls to an existing function." + }, + "F066": { + "verdict": "same", + "reason": "Same MOCK_PASS literal, shifted." + }, + "F067": { + "verdict": "existed", + "reason": "The awaited readJson in the loop existed before; the line was only edited to add a type parameter." + }, + "F068": { + "verdict": "same", + "reason": "Same password literal in an unchanged test, shifted by a new test inserted above." + }, + "F069": { + "verdict": "new", + "reason": "var siteLanguage is a + line added by the commit." + }, + "F070": { + "verdict": "new", + "reason": "The create_key line is in a newly added test." + }, + "F071": { + "verdict": "same", + "reason": "Same var selectedText line, shifted." + }, + "F072": { + "verdict": "existed", + "reason": "The same var declaration list existed before, only joined onto one line." + }, + "F073": { + "verdict": "new", + "reason": "The localCollections ternary with == is a newly added + line; the pre-existing comparison remains separately below." + }, + "F074": { + "verdict": "new", + "reason": "The whole file is added by this commit." + }, + "F075": { + "verdict": "same", + "reason": "Same f-string INSERT, shifted." + }, + "F076": { + "verdict": "existed", + "reason": "No diff near the flagged method, so it existed unchanged." + }, + "F077": { + "verdict": "new", + "reason": "var config is a + line added for the TomSelect setup." + }, + "F078": { + "verdict": "new", + "reason": "The whole translation file is added by this commit." + }, + "F079": { + "verdict": "same", + "reason": "Same f-string INSERT, shifted." + }, + "F080": { + "verdict": "same", + "reason": "Same MOCK_PASS literal, shifted." + }, + "F081": { + "verdict": "new", + "reason": "The starfleet comparison is a newly added branch; the old azure comparison moved to the following else-if." + }, + "F082": { + "verdict": "same", + "reason": "Same var siteLanguage in loadCannedResponses, shifted." + }, + "F083": { + "verdict": "new", + "reason": "The whole file is added by this commit." + }, + "F084": { + "verdict": "same", + "reason": "Same purchase function; only a parameter was removed." + }, + "F085": { + "verdict": "same", + "reason": "Same MOCK_PASSWORD literal, shifted." + }, + "F086": { + "verdict": "new", + "reason": "The whole file is added by this commit." + }, + "F087": { + "verdict": "same", + "reason": "Same awaited exists() check in the loop, edited only with a type cast." + }, + "F088": { + "verdict": "same", + "reason": "Same awaited call in the loop; only an argument was added." + }, + "F089": { + "verdict": "same", + "reason": "Same var matchingCard line, shifted." + }, + "F090": { + "verdict": "same", + "reason": "Same handleNoShowFee declaration, only whitespace changed." + }, + "F091": { + "verdict": "same", + "reason": "Same test method, shifted." + }, + "F092": { + "verdict": "unclear", + "reason": "The admin seed block is shown as + lines, but the hunk header implies ~250 lines were removed earlier and the excerpt cannot show whether this block was moved from there." + }, + "F093": { + "verdict": "existed", + "reason": "The expected password message existed before at the same place; only its text was edited." + }, + "F094": { + "verdict": "new", + "reason": "The function is in a large block of + lines added by the commit." + }, + "F095": { + "verdict": "new", + "reason": "var self = this is a + line added in initialize." + }, + "F096": { + "verdict": "new", + "reason": "The credential literal is in a newly added test." + }, + "F097": { + "verdict": "same", + "reason": "Same MOCK_PASS literal, shifted." + }, + "F098": { + "verdict": "new", + "reason": "var perms is part of the rewritten permission-loading + lines." + }, + "F099": { + "verdict": "new", + "reason": "var lastSyncedCardSelection is a + line added by the commit." + }, + "F100": { + "verdict": "same", + "reason": "Same function; its signature gained an optional parameter." + } +} diff --git a/research/finding_identity/paper/ground_truth_check/agreement.md b/research/finding_identity/paper/ground_truth_check/agreement.md new file mode 100644 index 0000000..9ce93ae --- /dev/null +++ b/research/finding_identity/paper/ground_truth_check/agreement.md @@ -0,0 +1,92 @@ +# Ground-truth check: results + +Generated by `research/finding_identity/gt_agreement.py`. *Holds* is the share of +decided items (unclear left out) where the rater confirms the harness's truth, with +a Wilson 95% interval. + +## Human rater + +60 of 100 items labelled. + +| kind | confirms | contradicts | unclear | holds | 95% CI | +|---|---:|---:|---:|---:|---| +| introduced | 0 | 0 | 40 | n/a | n/a | +| pair | 60 | 0 | 0 | 100.0% | [94.0, 100.0] | +| all | 60 | 0 | 40 | 100.0% | [94.0, 100.0] | + +| stratum | confirms | contradicts | unclear | holds | 95% CI | +|---|---:|---:|---:|---:|---| +| agree_known | 20 | 0 | 0 | 100.0% | [83.9, 100.0] | +| agree_new | 0 | 0 | 20 | n/a | n/a | +| edited | 20 | 0 | 0 | 100.0% | [83.9, 100.0] | +| known_but_new | 20 | 0 | 0 | 100.0% | [83.9, 100.0] | +| new_but_matched | 0 | 0 | 20 | n/a | n/a | +| all | 60 | 0 | 40 | 100.0% | [94.0, 100.0] | + +## Agent rater + +100 of 100 items labelled. + +| kind | confirms | contradicts | unclear | holds | 95% CI | +|---|---:|---:|---:|---:|---| +| introduced | 27 | 13 | 0 | 67.5% | [52.0, 79.9] | +| pair | 59 | 1 | 0 | 98.3% | [91.1, 99.7] | +| all | 86 | 14 | 0 | 86.0% | [77.9, 91.5] | + +| stratum | confirms | contradicts | unclear | holds | 95% CI | +|---|---:|---:|---:|---:|---| +| agree_known | 20 | 0 | 0 | 100.0% | [83.9, 100.0] | +| agree_new | 17 | 3 | 0 | 85.0% | [64.0, 94.8] | +| edited | 19 | 1 | 0 | 95.0% | [76.4, 99.1] | +| known_but_new | 20 | 0 | 0 | 100.0% | [83.9, 100.0] | +| new_but_matched | 10 | 10 | 0 | 50.0% | [29.9, 70.1] | +| all | 86 | 14 | 0 | 86.0% | [77.9, 91.5] | + +## Agreement between Human rater and Agent rater + +On the 60 items both labelled (confirms / contradicts / unclear): +59 agree (98.3%), Cohen's kappa 0.00. + +Items to adjudicate: F021. + +## Adjudicated + +100 of 100 items labelled. + +| kind | confirms | contradicts | unclear | holds | 95% CI | +|---|---:|---:|---:|---:|---| +| introduced | 30 | 10 | 0 | 75.0% | [59.8, 85.8] | +| pair | 59 | 1 | 0 | 98.3% | [91.1, 99.7] | +| all | 89 | 11 | 0 | 89.0% | [81.4, 93.7] | + +| stratum | confirms | contradicts | unclear | holds | 95% CI | +|---|---:|---:|---:|---:|---| +| agree_known | 20 | 0 | 0 | 100.0% | [83.9, 100.0] | +| agree_new | 20 | 0 | 0 | 100.0% | [83.9, 100.0] | +| edited | 19 | 1 | 0 | 95.0% | [76.4, 99.1] | +| known_but_new | 20 | 0 | 0 | 100.0% | [83.9, 100.0] | +| new_but_matched | 10 | 10 | 0 | 50.0% | [29.9, 70.1] | +| all | 89 | 11 | 0 | 89.0% | [81.4, 93.7] | + +Sources of the adjudicated verdicts: + +- 59: human and agent rater agree +- 27: agent rater only (round 2) +- 11: human confirmed the agent rater +- 1: rule re-run: createUser() reaches cyclomatic complexity 16 (limit 15) only after the commit; CodeGuard did not report it before +- 1: rule re-run: the test function reaches cyclomatic complexity 18 (limit 15) only after the commit; CodeGuard did not report it before +- 1: rule re-run: raise_for_response() reaches nesting depth 6 (limit 5) only after the commit; CodeGuard did not report it before + +## Hidden rates after correcting the ground truth + +From `research/finding_identity/results/corpus-v3`: 1,974 introduced findings. *Matched, sampled*: the +scheme's hidden findings among the sampled `new_but_matched` items; *truly new*: +those the adjudication confirms as new. The corrected rate scales the hidden +count by that share (range: its Wilson 95% interval). + +| scheme | hidden | matched, sampled | truly new | corrected hidden | range | +|---|---:|---:|---:|---:|---| +| `line_text` | 1.17% (23) | 8 | 6 | 0.87% | [0.48, 1.08] | +| `github` | 0.81% (16) | 6 | 3 | 0.41% | [0.15, 0.66] | +| `codeguard` | 1.87% (37) | 10 | 7 | 1.31% | [0.74, 1.67] | +| `sonar` | 1.47% (29) | 12 | 2 | 0.24% | [0.07, 0.66] | diff --git a/research/finding_identity/paper/ground_truth_check/human_adjudication.json b/research/finding_identity/paper/ground_truth_check/human_adjudication.json new file mode 100644 index 0000000..0dad20a --- /dev/null +++ b/research/finding_identity/paper/ground_truth_check/human_adjudication.json @@ -0,0 +1,72 @@ +{ + "F020": { + "at": "2026-10-18T08:07:24.387Z", + "note": "", + "verdict": "existed" + }, + "F021": { + "at": "2026-10-18T08:13:34.507Z", + "note": "", + "verdict": "different" + }, + "F028": { + "at": "2026-10-18T08:13:42.932Z", + "note": "", + "verdict": "existed" + }, + "F035": { + "at": "2026-10-18T08:13:52.453Z", + "note": "", + "verdict": "existed" + }, + "F036": { + "at": "2026-10-18T08:13:57.676Z", + "note": "", + "verdict": "existed" + }, + "F049": { + "at": "2026-10-18T08:14:01.349Z", + "note": "", + "verdict": "existed" + }, + "F058": { + "at": "2026-10-18T08:14:06.458Z", + "note": "", + "verdict": "existed" + }, + "F059": { + "at": "2026-10-18T08:14:15.091Z", + "note": "", + "verdict": "existed" + }, + "F065": { + "at": "2026-10-18T08:14:19.344Z", + "note": "", + "verdict": "existed" + }, + "F067": { + "at": "2026-10-18T08:14:23.883Z", + "note": "", + "verdict": "existed" + }, + "F072": { + "at": "2026-10-18T08:14:27.899Z", + "note": "", + "verdict": "existed" + }, + "F076": { + "at": "2026-10-18T08:14:44.112Z", + "note": "", + "verdict": "existed" + }, + "F092": { + "at": "2026-10-18T08:14:50.120Z", + "note": "", + "verdict": "existed" + }, + "F093": { + "at": "2026-10-18T08:14:54.128Z", + "note": "", + "verdict": "existed" + } +} diff --git a/research/finding_identity/paper/ground_truth_check/human_labels_round1.json b/research/finding_identity/paper/ground_truth_check/human_labels_round1.json new file mode 100644 index 0000000..737700a --- /dev/null +++ b/research/finding_identity/paper/ground_truth_check/human_labels_round1.json @@ -0,0 +1,502 @@ +{ + "F001": { + "at": "2026-10-18T07:49:38.103Z", + "note": "", + "verdict": "new" + }, + "F002": { + "at": "2026-10-18T07:49:49.965Z", + "note": "", + "verdict": "same" + }, + "F003": { + "at": "2026-10-18T07:49:58.592Z", + "note": "", + "verdict": "same" + }, + "F004": { + "at": "2026-10-18T07:50:04.375Z", + "note": "", + "verdict": "same" + }, + "F005": { + "at": "2026-10-18T07:50:14.428Z", + "note": "", + "verdict": "same" + }, + "F006": { + "at": "2026-10-18T07:50:17.342Z", + "note": "", + "verdict": "same" + }, + "F007": { + "at": "2026-10-18T07:50:21.309Z", + "note": "", + "verdict": "new" + }, + "F008": { + "at": "2026-10-18T07:50:24.190Z", + "note": "", + "verdict": "new" + }, + "F009": { + "at": "2026-10-18T07:50:27.072Z", + "note": "", + "verdict": "same" + }, + "F010": { + "at": "2026-10-18T07:50:30.324Z", + "note": "", + "verdict": "same" + }, + "F011": { + "at": "2026-10-18T07:50:45.302Z", + "note": "", + "verdict": "same" + }, + "F012": { + "at": "2026-10-18T07:59:20.216Z", + "note": "", + "verdict": "same" + }, + "F013": { + "at": "2026-10-18T07:59:52.453Z", + "note": "", + "verdict": "same" + }, + "F014": { + "at": "2026-10-18T07:51:02.148Z", + "note": "", + "verdict": "new" + }, + "F015": { + "at": "2026-10-18T07:51:05.824Z", + "note": "", + "verdict": "new" + }, + "F016": { + "at": "2026-10-18T07:51:16.383Z", + "note": "", + "verdict": "same" + }, + "F017": { + "at": "2026-10-18T07:51:21.067Z", + "note": "", + "verdict": "same" + }, + "F018": { + "at": "2026-10-18T07:51:26.705Z", + "note": "", + "verdict": "same" + }, + "F019": { + "at": "2026-10-18T07:51:30.399Z", + "note": "", + "verdict": "same" + }, + "F020": { + "at": "2026-10-18T07:51:32.718Z", + "note": "", + "verdict": "new" + }, + "F021": { + "at": "2026-10-18T08:00:22.268Z", + "note": "", + "verdict": "same" + }, + "F022": { + "at": "2026-10-18T07:51:38.505Z", + "note": "", + "verdict": "same" + }, + "F023": { + "at": "2026-10-18T08:00:00.319Z", + "note": "", + "verdict": "same" + }, + "F024": { + "at": "2026-10-18T07:51:44.346Z", + "note": "", + "verdict": "new" + }, + "F025": { + "at": "2026-10-18T07:58:57.385Z", + "note": "", + "verdict": "same" + }, + "F026": { + "at": "2026-10-18T07:52:22.931Z", + "note": "", + "verdict": "same" + }, + "F027": { + "at": "2026-10-18T07:52:25.543Z", + "note": "", + "verdict": "new" + }, + "F028": { + "at": "2026-10-18T07:52:29.402Z", + "note": "", + "verdict": "new" + }, + "F029": { + "at": "2026-10-18T07:52:34.048Z", + "note": "", + "verdict": "same" + }, + "F030": { + "at": "2026-10-18T07:55:10.450Z", + "note": "", + "verdict": "same" + }, + "F031": { + "at": "2026-10-18T07:55:13.225Z", + "note": "", + "verdict": "same" + }, + "F032": { + "at": "2026-10-18T07:55:16.876Z", + "note": "", + "verdict": "same" + }, + "F033": { + "at": "2026-10-18T07:55:20.111Z", + "note": "", + "verdict": "same" + }, + "F034": { + "at": "2026-10-18T07:55:22.852Z", + "note": "", + "verdict": "same" + }, + "F035": { + "at": "2026-10-18T07:55:25.612Z", + "note": "", + "verdict": "new" + }, + "F036": { + "at": "2026-10-18T07:55:28.515Z", + "note": "", + "verdict": "new" + }, + "F037": { + "at": "2026-10-18T07:55:31.484Z", + "note": "", + "verdict": "same" + }, + "F038": { + "at": "2026-10-18T07:55:34.026Z", + "note": "", + "verdict": "same" + }, + "F039": { + "at": "2026-10-18T07:55:36.322Z", + "note": "", + "verdict": "same" + }, + "F040": { + "at": "2026-10-18T07:55:38.244Z", + "note": "", + "verdict": "same" + }, + "F041": { + "at": "2026-10-18T07:55:40.240Z", + "note": "", + "verdict": "new" + }, + "F042": { + "at": "2026-10-18T07:55:43.995Z", + "note": "", + "verdict": "new" + }, + "F043": { + "at": "2026-10-18T07:55:46.554Z", + "note": "", + "verdict": "same" + }, + "F044": { + "at": "2026-10-18T07:59:56.234Z", + "note": "", + "verdict": "same" + }, + "F045": { + "at": "2026-10-18T07:56:10.897Z", + "note": "", + "verdict": "same" + }, + "F046": { + "at": "2026-10-18T07:56:15.109Z", + "note": "", + "verdict": "same" + }, + "F047": { + "at": "2026-10-18T07:56:21.186Z", + "note": "", + "verdict": "same" + }, + "F048": { + "at": "2026-10-18T07:56:23.266Z", + "note": "", + "verdict": "same" + }, + "F049": { + "at": "2026-10-18T07:56:25.130Z", + "note": "", + "verdict": "new" + }, + "F050": { + "at": "2026-10-18T07:56:28.576Z", + "note": "", + "verdict": "same" + }, + "F051": { + "at": "2026-10-18T07:56:30.801Z", + "note": "", + "verdict": "new" + }, + "F052": { + "at": "2026-10-18T07:56:33.433Z", + "note": "", + "verdict": "new" + }, + "F053": { + "at": "2026-10-18T07:56:35.687Z", + "note": "", + "verdict": "same" + }, + "F054": { + "at": "2026-10-18T07:56:37.699Z", + "note": "", + "verdict": "new" + }, + "F055": { + "at": "2026-10-18T07:56:51.178Z", + "note": "", + "verdict": "same" + }, + "F056": { + "at": "2026-10-18T07:56:52.930Z", + "note": "", + "verdict": "new" + }, + "F057": { + "at": "2026-10-18T07:56:57.418Z", + "note": "", + "verdict": "same" + }, + "F058": { + "at": "2026-10-18T07:56:59.088Z", + "note": "", + "verdict": "new" + }, + "F059": { + "at": "2026-10-18T07:57:00.632Z", + "note": "", + "verdict": "new" + }, + "F060": { + "at": "2026-10-18T07:57:03.925Z", + "note": "", + "verdict": "same" + }, + "F061": { + "at": "2026-10-18T07:57:06.352Z", + "note": "", + "verdict": "same" + }, + "F062": { + "at": "2026-10-18T07:57:09.762Z", + "note": "", + "verdict": "same" + }, + "F063": { + "at": "2026-10-18T07:57:15.830Z", + "note": "", + "verdict": "same" + }, + "F064": { + "at": "2026-10-18T07:57:18.197Z", + "note": "", + "verdict": "same" + }, + "F065": { + "at": "2026-10-18T07:57:20.076Z", + "note": "", + "verdict": "new" + }, + "F066": { + "at": "2026-10-18T07:57:23.018Z", + "note": "", + "verdict": "same" + }, + "F067": { + "at": "2026-10-18T07:57:25.796Z", + "note": "", + "verdict": "new" + }, + "F068": { + "at": "2026-10-18T07:57:29.469Z", + "note": "", + "verdict": "same" + }, + "F069": { + "at": "2026-10-18T07:57:31.315Z", + "note": "", + "verdict": "new" + }, + "F070": { + "at": "2026-10-18T07:57:35.283Z", + "note": "", + "verdict": "new" + }, + "F071": { + "at": "2026-10-18T07:57:37.872Z", + "note": "", + "verdict": "same" + }, + "F072": { + "at": "2026-10-18T07:57:40.101Z", + "note": "", + "verdict": "new" + }, + "F073": { + "at": "2026-10-18T07:57:43.534Z", + "note": "", + "verdict": "new" + }, + "F074": { + "at": "2026-10-18T07:57:46.582Z", + "note": "", + "verdict": "new" + }, + "F075": { + "at": "2026-10-18T07:57:48.938Z", + "note": "", + "verdict": "same" + }, + "F076": { + "at": "2026-10-18T07:57:50.569Z", + "note": "", + "verdict": "new" + }, + "F077": { + "at": "2026-10-18T07:57:52.416Z", + "note": "", + "verdict": "new" + }, + "F078": { + "at": "2026-10-18T07:57:54.322Z", + "note": "", + "verdict": "new" + }, + "F079": { + "at": "2026-10-18T07:57:55.968Z", + "note": "", + "verdict": "same" + }, + "F080": { + "at": "2026-10-18T07:57:57.323Z", + "note": "", + "verdict": "same" + }, + "F081": { + "at": "2026-10-18T07:57:59.024Z", + "note": "", + "verdict": "new" + }, + "F082": { + "at": "2026-10-18T07:58:00.678Z", + "note": "", + "verdict": "same" + }, + "F083": { + "at": "2026-10-18T07:58:02.103Z", + "note": "", + "verdict": "new" + }, + "F084": { + "at": "2026-10-18T07:58:07.685Z", + "note": "", + "verdict": "same" + }, + "F085": { + "at": "2026-10-18T07:58:09.927Z", + "note": "", + "verdict": "same" + }, + "F086": { + "at": "2026-10-18T07:58:17.789Z", + "note": "", + "verdict": "new" + }, + "F087": { + "at": "2026-10-18T07:58:19.499Z", + "note": "", + "verdict": "same" + }, + "F088": { + "at": "2026-10-18T07:58:21.445Z", + "note": "", + "verdict": "same" + }, + "F089": { + "at": "2026-10-18T07:58:23.591Z", + "note": "", + "verdict": "same" + }, + "F090": { + "at": "2026-10-18T07:58:25.287Z", + "note": "", + "verdict": "same" + }, + "F091": { + "at": "2026-10-18T07:58:27.208Z", + "note": "", + "verdict": "same" + }, + "F092": { + "at": "2026-10-18T07:58:28.498Z", + "note": "", + "verdict": "new" + }, + "F093": { + "at": "2026-10-18T07:58:32.837Z", + "note": "", + "verdict": "new" + }, + "F094": { + "at": "2026-10-18T07:58:37.738Z", + "note": "", + "verdict": "new" + }, + "F095": { + "at": "2026-10-18T07:58:39.234Z", + "note": "", + "verdict": "new" + }, + "F096": { + "at": "2026-10-18T07:58:41.648Z", + "note": "", + "verdict": "new" + }, + "F097": { + "at": "2026-10-18T07:58:45.100Z", + "note": "", + "verdict": "same" + }, + "F098": { + "at": "2026-10-18T07:58:47.519Z", + "note": "", + "verdict": "new" + }, + "F099": { + "at": "2026-10-18T07:58:48.748Z", + "note": "", + "verdict": "new" + }, + "F100": { + "at": "2026-10-18T07:58:50.869Z", + "note": "", + "verdict": "same" + } +} diff --git a/research/finding_identity/paper/ground_truth_check/items.json b/research/finding_identity/paper/ground_truth_check/items.json new file mode 100644 index 0000000..39bb3c8 --- /dev/null +++ b/research/finding_identity/paper/ground_truth_check/items.json @@ -0,0 +1,9290 @@ +[ + { + "item": "F001", + "kind": "introduced", + "repository": "ghost", + "commit_url": "https://github.com/TryGhost/Ghost/commit/7da91e095b2803c62bc67fe70f2c4f0d114846d9", + "rule": "CG-PERF-101", + "rule_title": "await inside a loop runs the iterations one after another", + "path": "tools/scripts/test/ci-path-filters.test.js", + "old_path": "tools/scripts/test/ci-path-filters.test.js", + "old_line": null, + "line": 69, + "before": [ + [ + 51, + " assert.ok(docs.includes('**/*.md'));" + ], + [ + 52, + " assert.ok(docs.includes('**/*.mdx'));" + ], + [ + 53, + " assert.ok(anyCode.includes('!**/*.md'));" + ], + [ + 54, + " assert.ok(anyCode.includes('!**/*.mdx'));" + ], + [ + 55, + " assert.ok(e2e.includes('!**/*.md'));" + ], + [ + 56, + " assert.ok(e2e.includes('!**/*.mdx'));" + ], + [ + 57, + " });" + ], + [ + 58, + "" + ], + [ + 59, + " it('skips code checks and E2E for ownership and docs changes, but keeps mixed changes', async () => {" + ], + [ + 60, + " for (const filterName of ['any-code', 'e2e']) {" + ], + [ + 61, + " const patterns = await getFilterPatterns(filterName);" + ], + [ + 62, + " const matches = (path) =>" + ], + [ + 63, + " patterns.every((pattern) => picomatch(pattern, { dot: true })(path));" + ] + ], + "before_line": 57, + "before_mapped": false, + "after": [ + [ + 65, + " const docs = await getFilterPatterns('docs');" + ], + [ + 66, + " assert.ok(docs.some((pattern) => picomatch(pattern, { dot: true })(inventory)));" + ], + [ + 67, + "" + ], + [ + 68, + " for (const filterName of ['core', 'any-code', 'e2e']) {" + ], + [ + 69, + " const patterns = await getFilterPatterns(filterName);" + ], + [ + 70, + " const positives = patterns.filter((pattern) => !pattern.startsWith('!'));" + ], + [ + 71, + " const exclusions = patterns.filter((pattern) => pattern.startsWith('!'));" + ], + [ + 72, + " const matches = (path) =>" + ], + [ + 73, + " (positives.length === 0 ||" + ] + ], + "diff": "@@ -56,6 +60,83 @@ describe('CI path filters', () => {\n assert.ok(e2e.includes('!**/*.mdx'));\n });\n\n+ it('treats the service inventory as docs and keeps code checks for mixed changes', async () => {\n+ const inventory = 'ghost/core/server/services/service-inventory.yaml';\n+ const docs = await getFilterPatterns('docs');\n+ assert.ok(docs.some((pattern) => picomatch(pattern, { dot: true })(inventory)));\n+\n+ for (const filterName of ['core', 'any-code', 'e2e']) {\n+ const patterns = await getFilterPatterns(filterName);\n+ const positives = patterns.filter((pattern) => !pattern.startsWith('!'));\n+ const exclusions = patterns.filter((pattern) => pattern.startsWith('!'));\n+ const matches = (path) =>\n+ (positives.length === 0 ||\n+ positives.some((pattern) => picomatch(pattern, { dot: true })(path))) &&\n+ exclusions.every((pattern) => picomatch(pattern, { dot: true })(path));\n+ const metadata = [inventory, 'ghost/core/server/services/README.md'];\n+\n+ assert.strictEqual(metadata.some(matches), false, filterName);\n+ for (const runtime of [\n+ 'ghost/core/server/services/identity-tokens/index.ts',\n+ 'ghost/core/server/services/other-metadata.yaml',\n+ '.github/workflows/ci.yml',\n+ ]) {\n+ assert.strictEqual([...metadata, runtime].some(matches), true, `${filterName}: ${runtime}`);\n+ }\n+ }\n+\n+ const anyCode = await getFilterPatterns('any-code');\n+ assert.ok(\n+ anyCode.every((pattern) =>\n+ picomatch(pattern, { dot: true })(\n+ 'ghost/test/unit/server/services/service-inventory.test.ts',\n+ ),\n+ ),\n+ 'Changes to the inventory validator must still run code checks',\n+ );\n+ });\n+\n... (45 more lines)" + }, + { + "item": "F002", + "kind": "pair", + "repository": "erpnext", + "commit_url": "https://github.com/frappe/erpnext/commit/028a9ed1d94ce99702dd65b32ca3aede52b52006", + "rule": "CG-QUAL-005", + "rule_title": "Function takes too many parameters", + "path": "erpnext/stock/doctype/pick_list/pick_list.py", + "old_path": "erpnext/stock/doctype/pick_list/pick_list.py", + "old_line": 1354, + "line": 1451, + "before": [ + [ + 1350, + "\titem_location_map[item_doc.item_code] = available_locations" + ], + [ + 1351, + "\treturn locations" + ], + [ + 1352, + "" + ], + [ + 1353, + "" + ], + [ + 1354, + "def get_available_item_locations(" + ], + [ + 1355, + "\titem_code," + ], + [ + 1356, + "\tfrom_warehouses," + ], + [ + 1357, + "\trequired_qty," + ], + [ + 1358, + "\tcompany," + ] + ], + "before_line": 1354, + "before_mapped": true, + "after": [ + [ + 1447, + "\titem_location_map[item_doc.item_code] = available_locations" + ], + [ + 1448, + "\treturn locations" + ], + [ + 1449, + "" + ], + [ + 1450, + "" + ], + [ + 1451, + "def get_available_item_locations(" + ], + [ + 1452, + "\titem_code," + ], + [ + 1453, + "\tfrom_warehouses," + ], + [ + 1454, + "\trequired_qty," + ], + [ + 1455, + "\tcompany," + ] + ], + "diff": "(no diff hunk near this line)" + }, + { + "item": "F003", + "kind": "pair", + "repository": "erpnext", + "commit_url": "https://github.com/frappe/erpnext/commit/750bf9d9b378fb92eab68733ae727214f3961c0a", + "rule": "CG-PERF-002", + "rule_title": "String built with += inside a loop", + "path": "erpnext/stock/utils.py", + "old_path": "erpnext/stock/utils.py", + "old_line": 159, + "line": 185, + "before": [ + [ + 155, + "\t\t\t\t\t\tfrappe.bold(field), frappe.bold(\"Inventory Dimension\")" + ], + [ + 156, + "\t\t\t\t\t)" + ], + [ + 157, + "\t\t\t\t)" + ], + [ + 158, + "\t\t\targs[field] = value" + ], + [ + 159, + "\t\t\textra_cond += f\" and {field} = %({field})s\"" + ], + [ + 160, + "" + ], + [ + 161, + "\tlast_entry = get_previous_sle(args, extra_cond=extra_cond)" + ], + [ + 162, + "" + ], + [ + 163, + "\tif with_valuation_rate:" + ] + ], + "before_line": 159, + "before_mapped": true, + "after": [ + [ + 181, + "\t\t\t\t\t\tfrappe.bold(field), frappe.bold(\"Inventory Dimension\")" + ], + [ + 182, + "\t\t\t\t\t)" + ], + [ + 183, + "\t\t\t\t)" + ], + [ + 184, + "\t\t\targs[field] = value" + ], + [ + 185, + "\t\t\textra_cond += f\" and {field} = %({field})s\"" + ], + [ + 186, + "" + ], + [ + 187, + "\tlast_entry = get_previous_sle(args, extra_cond=extra_cond)" + ], + [ + 188, + "" + ], + [ + 189, + "\tif with_valuation_rate:" + ] + ], + "diff": "(no diff hunk near this line)" + }, + { + "item": "F004", + "kind": "pair", + "repository": "kiwi", + "commit_url": "https://github.com/kiwitcms/Kiwi/commit/362febe93063899b8c7468cf7490c26028368a5c", + "rule": "CG-SEC-109", + "rule_title": "Redirect to a URL taken from the request", + "path": "tcms/testplans/static/testplans/js/clone.js", + "old_path": "tcms/testplans/static/testplans/js/clone.js", + "old_line": 110, + "line": 112, + "before": [ + [ + 106, + "" + ], + [ + 107, + " if (newPlanIds.length === 1) {" + ], + [ + 108, + " window.location.assign(`/plan/${newPlanIds[0]}/`)" + ], + [ + 109, + " } else {" + ], + [ + 110, + " window.location.assign(document.referrer || '/')" + ], + [ + 111, + " }" + ], + [ + 112, + " }" + ], + [ + 113, + "" + ], + [ + 114, + " if (!rows.length) {" + ] + ], + "before_line": 110, + "before_mapped": true, + "after": [ + [ + 108, + " const redirectToClonedPlan = function () {" + ], + [ + 109, + " if (isTree || newPlanIds.length === 1) {" + ], + [ + 110, + " window.location.assign(`/plan/${newPlanIds[0]}/`)" + ], + [ + 111, + " } else {" + ], + [ + 112, + " window.location.assign(document.referrer || '/')" + ], + [ + 113, + " }" + ], + [ + 114, + " }" + ], + [ + 115, + "" + ], + [ + 116, + " // in tree mode set the parent of every clone, which can only be" + ] + ], + "diff": "@@ -104,11 +105,44 @@ export function pageTestplansCloneReadyHandler () {\n return\n }\n\n- if (newPlanIds.length === 1) {\n- window.location.assign(`/plan/${newPlanIds[0]}/`)\n- } else {\n- window.location.assign(document.referrer || '/')\n+ const redirectToClonedPlan = function () {\n+ if (isTree || newPlanIds.length === 1) {\n+ window.location.assign(`/plan/${newPlanIds[0]}/`)\n+ } else {\n+ window.location.assign(document.referrer || '/')\n+ }\n }\n+\n+ // in tree mode set the parent of every clone, which can only be\n+ // done once all of the clones have been created.\n+ if (isTree) {\n+ let updatedCount = 0\n+\n+ const updateIsDone = function () {\n+ updatedCount += 1\n+\n+ if (updatedCount === rows.length) {\n+ redirectToClonedPlan()\n+ }\n+ }\n+\n+ rows.each(function () {\n+ const row = $(this)\n+\n+ jsonRPC(\n+ 'TestPlan.update',\n+ [\n+ clonedPlanIds[row.data('plan-id')],\n+ { parent: clonedPlanIds[row.attr('data-parent-id')] }\n+ ],\n+ updateIsDone\n+ )\n... (10 more lines)" + }, + { + "item": "F005", + "kind": "pair", + "repository": "pgadmin", + "commit_url": "https://github.com/pgadmin-org/pgadmin4/commit/5f56805162c117ac1de17368ef8fdc0b07221e2c", + "rule": "CG-QUAL-101", + "rule_title": "Loose equality (== / !=)", + "path": "web/pgadmin/tools/sqleditor/static/js/components/QueryToolComponent.jsx", + "old_path": "web/pgadmin/tools/sqleditor/static/js/components/QueryToolComponent.jsx", + "old_line": 813, + "line": 872, + "before": [ + [ + 809, + " let existIdx = _.findIndex(qtState.connection_list, (conn)=>{" + ], + [ + 810, + " conn.role= conn.role == ''? null :conn.role;" + ], + [ + 811, + " return(" + ], + [ + 812, + " conn.sid == connectionData.sid && conn.database_name == connectionData.database_name" + ], + [ + 813, + " && conn.user == connectionData.user && conn.role == connectionData.role" + ], + [ + 814, + " );" + ], + [ + 815, + " });" + ], + [ + 816, + " if(existIdx > -1) {" + ], + [ + 817, + " reject(new Error(gettext('Connection with this configuration already present.')));" + ] + ], + "before_line": 813, + "before_mapped": true, + "after": [ + [ + 868, + "" + ], + [ + 869, + " let existIdx = _.findIndex(qtState.connection_list, (conn)=>{" + ], + [ + 870, + " return(" + ], + [ + 871, + " conn.sid == connectionData.sid && conn.database_name == connectionData.database_name" + ], + [ + 872, + " && conn.user == connectionData.user && (conn.role || null) == (connectionData.role || null)" + ], + [ + 873, + " );" + ], + [ + 874, + " });" + ], + [ + 875, + " if(existIdx > -1) {" + ], + [ + 876, + " reject(new Error(gettext('Connection with this configuration already present.')));" + ] + ], + "diff": "@@ -807,10 +867,9 @@ export default function QueryToolComponent({params, pgWindow, pgAdmin, selectedN\n };\n\n let existIdx = _.findIndex(qtState.connection_list, (conn)=>{\n- conn.role= conn.role == ''? null :conn.role;\n return(\n conn.sid == connectionData.sid && conn.database_name == connectionData.database_name\n- && conn.user == connectionData.user && conn.role == connectionData.role\n+ && conn.user == connectionData.user && (conn.role || null) == (connectionData.role || null)\n );\n });\n if(existIdx > -1) {\n@@ -818,11 +877,12 @@ export default function QueryToolComponent({params, pgWindow, pgAdmin, selectedN\n return;\n }\n updateQueryToolConnection(connectionData, true)\n- .catch((err)=>{\n- reject(err instanceof Error ? err : Error(gettext('Something went wrong')));\n- }).then(()=>{\n+ .then(()=>{\n resolve();\n onClose();\n+ })\n+ .catch((err)=>{\n+ reject(err instanceof Error ? err : Error(gettext('Something went wrong')));\n });\n });\n }}" + }, + { + "item": "F006", + "kind": "pair", + "repository": "arches", + "commit_url": "https://github.com/archesproject/arches/commit/2e02ce6e61999d93c10b05d853ae08f1912fa294", + "rule": "CG-QUAL-104", + "rule_title": "var declaration", + "path": "arches/app/media/js/views/graph/permission-manager/permission-settings-form.js", + "old_path": "arches/app/media/js/views/graph/permission-manager/permission-settings-form.js", + "old_line": 128, + "line": 210, + "before": [ + [ + 124, + " });" + ], + [ + 125, + " }," + ], + [ + 126, + "" + ], + [ + 127, + " revert: function() {" + ], + [ + 128, + " var self = this;" + ], + [ + 129, + " var postData = {" + ], + [ + 130, + " 'selectedIdentities': this.selectedIdentities()," + ], + [ + 131, + " 'selectedCards': this.selectedCards()" + ], + [ + 132, + " };" + ] + ], + "before_line": 128, + "before_mapped": true, + "after": [ + [ + 206, + " });" + ], + [ + 207, + " }," + ], + [ + 208, + "" + ], + [ + 209, + " revert: function() {" + ], + [ + 210, + " var self = this;" + ], + [ + 211, + " if (!this.canSubmit()) {" + ], + [ + 212, + " return;" + ], + [ + 213, + " }" + ], + [ + 214, + " var confirm = arches.translations.graphDesignerPermissionsResetConfirm;" + ] + ], + "diff": "@@ -109,48 +188,64 @@ var PermissionSettingsForm = Backbone.View.extend({\n })\n };\n\n+ this.loading(true);\n $.ajax({\n type: 'POST',\n url: arches.urls.permission_data,\n- data: JSON.stringify(postData),\n- success: function(res) {\n+ data: JSON.stringify(postData)\n+ })\n+ .done(function() {\n+ self.showSuccess(arches.translations.graphDesignerPermissionsApplied);\n self.trigger('save');\n- self.clearUserPermissionCache();\n- // adds event to trigger dirty state in graph-designer\n- document.dispatchEvent(\n- new Event('permissionsSave')\n- );\n- }\n- });\n+ })\n+ .fail(function(response) {\n+ self.showError(response);\n+ })\n+ .always(function() {\n+ self.loading(false);\n+ });\n },\n\n revert: function() {\n+ var self = this;\n+ if (!this.canSubmit()) {\n+ return;\n+ }\n+ var confirm = arches.translations.graphDesignerPermissionsResetConfirm;\n+ this.alert(new AlertViewModel(\n+ 'ep-alert-red',\n+ confirm.title,\n... (53 more lines)" + }, + { + "item": "F007", + "kind": "introduced", + "repository": "inventree", + "commit_url": "https://github.com/inventree/InvenTree/commit/99893f3453c6a213a331c022e82cbafbde7ebd51", + "rule": "CG-QUAL-101", + "rule_title": "Loose equality (== / !=)", + "path": "src/frontend/src/components/tables/InvenTreeTable.tsx", + "old_path": "src/frontend/src/components/tables/InvenTreeTable.tsx", + "old_line": null, + "line": 940, + "before": [ + [ + 773, + "" + ], + [ + 774, + " // Update tableState.records when new data received" + ], + [ + 775, + " useEffect(() => {" + ], + [ + 776, + " tableState.setRecords(tableData ?? apiData ?? []);" + ], + [ + 777, + " }, [tableData, apiData]);" + ], + [ + 778, + "" + ], + [ + 779, + " const previewDrawer = usePreviewDrawerState();" + ], + [ + 780, + "" + ], + [ + 781, + " // Callback to display \"preview\" view for a row (if available)" + ], + [ + 782, + " const showRowPreview = useCallback(" + ], + [ + 783, + " (pk: string | number, targetUrl?: string) => {" + ], + [ + 784, + " if (tableProps.modelType && pk) {" + ], + [ + 785, + " previewDrawer.openPreview(" + ] + ], + "before_line": 779, + "before_mapped": false, + "after": [ + [ + 936, + " params: queryParams," + ], + [ + 937, + " timeout: 10 * 1000" + ], + [ + 938, + " })" + ], + [ + 939, + " .then((response) => {" + ], + [ + 940, + " if (generation != nestedGeneration.current) {" + ], + [ + 941, + " return;" + ], + [ + 942, + " }" + ], + [ + 943, + "" + ], + [ + 944, + " let results = response.data?.results ?? response.data ?? [];" + ] + ], + "diff": "@@ -776,6 +862,170 @@ export function InvenTreeTableInternal>({\n tableState.setRecords(tableData ?? apiData ?? []);\n }, [tableData, apiData]);\n\n+ // Serialized params, so that nested data is not discarded on every render\n+ const paramsKey: string = useMemo(\n+ () => JSON.stringify(tableProps.params ?? {}),\n+ [tableProps.params]\n+ );\n+\n+ // Discard nested data whenever the top-level table data is reloaded,\n+ // or the nesting configuration changes (as child rows depend on it)\n+ useEffect(() => {\n+ nestedGeneration.current += 1;\n+ // Keep the existing (empty) state object to avoid a needless re-render\n+ setNestedState((state) => (Object.keys(state).length > 0 ? {} : state));\n+ }, [\n+ props.nestedRows,\n+ url,\n+ tableData,\n+ apiData,\n+ paramsKey,\n+ pageSize,\n+ sortStatus,\n+ tableState.page,\n+ tableState.searchTerm,\n+ tableState.tableKey,\n+ tableState.filterSet.activeFilters,\n+ tableSearchParams\n+ ]);\n+\n+ // Expand or collapse a nested row, fetching child rows if required\n+ toggleNestedRow.current = (record: T) => {\n+ const nestedRows = props.nestedRows;\n+\n+ if (!nestedRows || !url) {\n+ return;\n+ }\n+\n+ const key = getRowKey(record);\n... (131 more lines)" + }, + { + "item": "F008", + "kind": "introduced", + "repository": "lnbits", + "commit_url": "https://github.com/lnbits/lnbits/commit/c0806ca59c90464e424f881a92efd8033fd9bd2b", + "rule": "CG-PERF-101", + "rule_title": "await inside a loop runs the iterations one after another", + "path": "tests/e2e/extension-helpers.ts", + "old_path": "tests/e2e/extension-helpers.ts", + "old_line": null, + "line": 540, + "before": [ + [ + 1, + "" + ] + ], + "before_line": 1, + "before_mapped": false, + "after": [ + [ + 536, + " await expect(tab).toBeVisible({timeout: 60_000})" + ], + [ + 537, + " for (let attempt = 0; attempt < 3; attempt += 1) {" + ], + [ + 538, + " await tab.click()" + ], + [ + 539, + " try {" + ], + [ + 540, + " await expect(tab).toHaveAttribute('aria-selected', 'true', {" + ], + [ + 541, + " timeout: 5_000" + ], + [ + 542, + " })" + ], + [ + 543, + " return" + ], + [ + 544, + " } catch (_error) {" + ] + ], + "diff": "@@ -0,0 +1,658 @@\n+import {expect, type Frame, type Locator, type Page} from '@playwright/test'\n+\n+import {\n+ browserJson,\n+ delay,\n+ isRecord,\n+ type LNbitsE2EServer,\n+ waitForResult\n+} from './fixtures'\n+\n+const INSTALLABLE_EXTENSION_REFRESH_TASK =\n+ 'refresh_installable_extensions_cache'\n+\n+export type ExtensionUnderTest = {\n+ extId: string\n+ manifestUrl: string\n+ name: string\n+ permissionTexts?: string[]\n+ repository: string\n+}\n+\n+export type E2EWallet = {\n+ adminkey: string\n+ id: string\n+ inkey: string\n+ name: string\n+}\n+\n+export async function login(\n+ page: Page,\n+ server: LNbitsE2EServer\n+): Promise {\n+ await page.goto('/')\n+ await page.locator('input[name=\"username\"]').fill(server.username)\n+ await page.locator('input[name=\"password\"]').fill(server.password)\n+ await page.getByRole('button', {name: /^login$/i}).click()\n+ await expect(page).toHaveURL(/\\/wallet\\/[^/]+$/)\n+ await dismissDisclaimer(page)\n+}\n... (620 more lines)" + }, + { + "item": "F009", + "kind": "pair", + "repository": "edx-platform", + "commit_url": "https://github.com/openedx/edx-platform/commit/b3f862e61aa84a5f8c027081b7d1a07a8619917d", + "rule": "CG-SEC-013", + "rule_title": "Request data used as a file path", + "path": "cms/djangoapps/contentstore/api/views/course_import.py", + "old_path": "cms/djangoapps/contentstore/api/views/course_import.py", + "old_line": 142, + "line": 145, + "before": [ + [ + 138, + " if not course_dir.isdir():" + ], + [ + 139, + " os.mkdir(course_dir)" + ], + [ + 140, + "" + ], + [ + 141, + " log.debug(f'importing course to {temp_filepath}')" + ], + [ + 142, + " with open(temp_filepath, \"wb+\") as temp_file:" + ], + [ + 143, + " for chunk in request.FILES['course_data'].chunks():" + ], + [ + 144, + " temp_file.write(chunk)" + ], + [ + 145, + "" + ], + [ + 146, + " log.info(\"Course import %s: Upload complete\", course_key)" + ] + ], + "before_line": 142, + "before_mapped": true, + "after": [ + [ + 141, + " os.makedirs(course_dir, exist_ok=True)" + ], + [ + 142, + "" + ], + [ + 143, + " try:" + ], + [ + 144, + " log.debug(f'importing course to {temp_filepath}')" + ], + [ + 145, + " with open(temp_filepath, \"wb+\") as temp_file:" + ], + [ + 146, + " for chunk in request.FILES['course_data'].chunks():" + ], + [ + 147, + " temp_file.write(chunk)" + ], + [ + 148, + "" + ], + [ + 149, + " log.info(\"Course import %s: Upload complete\", course_key)" + ] + ], + "diff": "@@ -131,22 +134,26 @@ class CourseImportView(CourseImportExportViewMixin, GenericAPIView):\n developer_message='Parameter in the wrong format',\n error_code='internal_error',\n )\n- course_dir = path(settings.GITHUB_REPO_ROOT) / base64.urlsafe_b64encode(\n- repr(course_key).encode('utf-8')\n- ).decode('utf-8')\n+ # Staging directory private to this upload, so that a concurrent import of\n+ # the same course cannot overwrite this archive or delete it on its way out.\n+ course_dir = course_import_working_dir(course_key, f'upload-{uuid4().hex}')\n temp_filepath = course_dir / filename\n- if not course_dir.isdir():\n- os.mkdir(course_dir)\n-\n- log.debug(f'importing course to {temp_filepath}')\n- with open(temp_filepath, \"wb+\") as temp_file:\n- for chunk in request.FILES['course_data'].chunks():\n- temp_file.write(chunk)\n-\n- log.info(\"Course import %s: Upload complete\", course_key)\n- with open(temp_filepath, 'rb') as local_file:\n- django_file = File(local_file)\n- storage_path = course_import_export_storage.save('olx_import/' + filename, django_file)\n+ os.makedirs(course_dir, exist_ok=True)\n+\n+ try:\n+ log.debug(f'importing course to {temp_filepath}')\n+ with open(temp_filepath, \"wb+\") as temp_file:\n+ for chunk in request.FILES['course_data'].chunks():\n+ temp_file.write(chunk)\n+\n+ log.info(\"Course import %s: Upload complete\", course_key)\n+ with open(temp_filepath, 'rb') as local_file:\n+ django_file = File(local_file)\n+ storage_path = course_import_export_storage.save('olx_import/' + filename, django_file)\n+ finally:\n+ # The archive now lives in storage; the import task downloads it into its\n+ # own working directory, so this staging copy is no longer needed.\n+ remove_course_import_working_dir(course_dir)\n\n... (3 more lines)" + }, + { + "item": "F010", + "kind": "pair", + "repository": "lnbits", + "commit_url": "https://github.com/lnbits/lnbits/commit/f5989201d9c21a4690334f91bbbb421214c989e7", + "rule": "CG-QUAL-102", + "rule_title": "Empty catch block", + "path": "lnbits/static/js/wasm-extension-component.js", + "old_path": "lnbits/static/js/wasm-extension-component.js", + "old_line": 904, + "line": 1126, + "before": [ + [ + 900, + " socket.addEventListener('message', event => {" + ], + [ + 901, + " let data = event.data" + ], + [ + 902, + " try {" + ], + [ + 903, + " data = JSON.parse(event.data)" + ], + [ + 904, + " } catch (_error) {}" + ], + [ + 905, + "" + ], + [ + 906, + " this.sendBridgeEvent({" + ], + [ + 907, + " event: 'payment.update'," + ], + [ + 908, + " subscriptionId," + ] + ], + "before_line": 904, + "before_mapped": true, + "after": [ + [ + 1122, + " socket.addEventListener('message', event => {" + ], + [ + 1123, + " let data = event.data" + ], + [ + 1124, + " try {" + ], + [ + 1125, + " data = JSON.parse(event.data)" + ], + [ + 1126, + " } catch (_error) {}" + ], + [ + 1127, + "" + ], + [ + 1128, + " this.sendBridgeEvent({" + ], + [ + 1129, + " event: 'payment.update'," + ], + [ + 1130, + " subscriptionId," + ] + ], + "diff": "(no diff hunk near this line)" + }, + { + "item": "F011", + "kind": "pair", + "repository": "indico", + "commit_url": "https://github.com/indico/indico/commit/1ab29e13596cacf234ce75f7c0d4be217010ba6d", + "rule": "CG-QUAL-003", + "rule_title": "Function is too complex", + "path": "indico/modules/events/registration/util.py", + "old_path": "indico/modules/events/registration/util.py", + "old_line": 474, + "line": 474, + "before": [ + [ + 470, + " return registration" + ], + [ + 471, + "" + ], + [ + 472, + "" + ], + [ + 473, + "@no_autoflush" + ], + [ + 474, + "def modify_registration(registration, data, management=False, notify_user=True):" + ], + [ + 475, + " from indico.modules.events.registration.tasks import delete_previous_registration_file" + ], + [ + 476, + "" + ], + [ + 477, + " user = session.user if session else None" + ], + [ + 478, + " old_data = snapshot_registration_data(registration)" + ] + ], + "before_line": 474, + "before_mapped": true, + "after": [ + [ + 470, + " return registration" + ], + [ + 471, + "" + ], + [ + 472, + "" + ], + [ + 473, + "@no_autoflush" + ], + [ + 474, + "def modify_registration(registration, data, management=False, notify_user=True):" + ], + [ + 475, + " from indico.modules.events.registration.tasks import delete_previous_registration_file" + ], + [ + 476, + "" + ], + [ + 477, + " user = session.user if session else None" + ], + [ + 478, + " old_data = snapshot_registration_data(registration)" + ] + ], + "diff": "(no diff hunk near this line)" + }, + { + "item": "F012", + "kind": "pair", + "repository": "documenso", + "commit_url": "https://github.com/documenso/documenso/commit/9c27ce6d185f7e25eec80439b931c4709723a2ae", + "rule": "CG-PERF-101", + "rule_title": "await inside a loop runs the iterations one after another", + "path": "packages/app-tests/e2e/teams/team-documents.spec.ts", + "old_path": "packages/app-tests/e2e/teams/team-documents.spec.ts", + "old_line": 37, + "line": 29, + "before": [ + [ + 33, + "" + ], + [ + 34, + " // Check counts after filtering." + ], + [ + 35, + " await checkDocumentTabCount(page, 'Inbox', 0);" + ], + [ + 36, + " await checkDocumentTabCount(page, 'Pending', 2);" + ], + [ + 37, + " await checkDocumentTabCount(page, 'Completed', 0);" + ], + [ + 38, + " await checkDocumentTabCount(page, 'Draft', 1);" + ], + [ + 39, + " await checkDocumentTabCount(page, 'All', 3);" + ], + [ + 40, + "" + ], + [ + 41, + " await apiSignout({ page });" + ] + ], + "before_line": 37, + "before_mapped": true, + "after": [ + [ + 25, + " // Apply filter." + ], + [ + 26, + " await toggleDocumentSenderFilter(page, teamMember2.name ?? '');" + ], + [ + 27, + "" + ], + [ + 28, + " // Check counts after filtering." + ], + [ + 29, + " await checkDocumentCounts(page, { inbox: 0, pending: 2, completed: 0, draft: 1, all: 3 });" + ], + [ + 30, + "" + ], + [ + 31, + " await apiSignout({ page });" + ], + [ + 32, + " }" + ], + [ + 33, + "});" + ] + ], + "diff": "@@ -20,23 +20,13 @@ test('[TEAMS]: check team documents count', async ({ page }) => {\n });\n\n // Check document counts.\n- await checkDocumentTabCount(page, 'Inbox', 0);\n- await checkDocumentTabCount(page, 'Pending', 2);\n- await checkDocumentTabCount(page, 'Completed', 1);\n- await checkDocumentTabCount(page, 'Draft', 2);\n- await checkDocumentTabCount(page, 'All', 5);\n+ await checkDocumentCounts(page, { inbox: 0, pending: 2, completed: 1, draft: 2, all: 5 });\n\n // Apply filter.\n- await page.locator('button').filter({ hasText: 'Sender: All' }).click();\n- await page.getByRole('option', { name: teamMember2.name ?? '' }).click();\n- await page.waitForURL(/senderIds/);\n+ await toggleDocumentSenderFilter(page, teamMember2.name ?? '');\n\n // Check counts after filtering.\n- await checkDocumentTabCount(page, 'Inbox', 0);\n- await checkDocumentTabCount(page, 'Pending', 2);\n- await checkDocumentTabCount(page, 'Completed', 0);\n- await checkDocumentTabCount(page, 'Draft', 1);\n- await checkDocumentTabCount(page, 'All', 3);\n+ await checkDocumentCounts(page, { inbox: 0, pending: 2, completed: 0, draft: 1, all: 3 });\n\n await apiSignout({ page });\n }" + }, + { + "item": "F013", + "kind": "pair", + "repository": "hoppscotch", + "commit_url": "https://github.com/hoppscotch/hoppscotch/commit/863b1b29a803f6c456b04dd82d19a53c7afc9102", + "rule": "CG-QUAL-101", + "rule_title": "Loose equality (== / !=)", + "path": "packages/hoppscotch-common/src/lib/sync/settings/index.ts", + "old_path": "packages/hoppscotch-selfhost-web/src/platform/settings/web/index.ts", + "old_line": 49, + "line": 42, + "before": [ + [ + 45, + " const res = await getUserSettings()" + ], + [ + 46, + "" + ], + [ + 47, + " // create user settings if it doesn't exist" + ], + [ + 48, + " E.isLeft(res) &&" + ], + [ + 49, + " res.left.error == \"user_settings/not_found\" &&" + ], + [ + 50, + " (await createUserSettings(JSON.stringify(getDefaultSettings())))" + ], + [ + 51, + "" + ], + [ + 52, + " if (E.isRight(res)) {" + ], + [ + 53, + " runDispatchWithOutSyncing(() => {" + ] + ], + "before_line": 49, + "before_mapped": true, + "after": [ + [ + 38, + "async function loadUserSettings() {" + ], + [ + 39, + " const res = await getUserSettings()" + ], + [ + 40, + "" + ], + [ + 41, + " // create user settings if it doesn't exist" + ], + [ + 42, + " if (E.isLeft(res) && res.left.error == \"user_settings/not_found\") {" + ], + [ + 43, + " const createRes = await createUserSettings(" + ], + [ + 44, + " JSON.stringify(getDefaultSettings())" + ], + [ + 45, + " )" + ], + [ + 46, + "" + ] + ], + "diff": "@@ -45,11 +39,19 @@ async function loadUserSettings() {\n const res = await getUserSettings()\n\n // create user settings if it doesn't exist\n- E.isLeft(res) &&\n- res.left.error == \"user_settings/not_found\" &&\n- (await createUserSettings(JSON.stringify(getDefaultSettings())))\n+ if (E.isLeft(res) && res.left.error == \"user_settings/not_found\") {\n+ const createRes = await createUserSettings(\n+ JSON.stringify(getDefaultSettings())\n+ )\n\n- if (E.isRight(res)) {\n+ if (E.isRight(createRes)) {\n+ runDispatchWithOutSyncing(() => {\n+ bulkApplySettings(\n+ JSON.parse(createRes.right.createUserSettings.properties)\n+ )\n+ })\n+ }\n+ } else if (E.isRight(res)) {\n runDispatchWithOutSyncing(() => {\n bulkApplySettings(JSON.parse(res.right.me.settings.properties))\n })\n" + }, + { + "item": "F014", + "kind": "introduced", + "repository": "erpnext", + "commit_url": "https://github.com/frappe/erpnext/commit/8c85cef29b0b80a23a499516a94d4319cf45503d", + "rule": "CG-QUAL-104", + "rule_title": "var declaration", + "path": "erpnext/stock/doctype/landed_cost_voucher/landed_cost_voucher.js", + "old_path": "erpnext/stock/doctype/landed_cost_voucher/landed_cost_voucher.js", + "old_line": null, + "line": 119, + "before": [ + [ + 112, + "\t\t\t\t\t\tprecision(\"applicable_charges\", item)" + ], + [ + 113, + "\t\t\t\t\t);" + ], + [ + 114, + "\t\t\t\t\ttotal_charges += item.applicable_charges;" + ], + [ + 115, + "\t\t\t\t});" + ], + [ + 116, + "" + ], + [ + 117, + "\t\t\t\tif (total_charges != this.frm.doc.total_taxes_and_charges) {" + ], + [ + 118, + "\t\t\t\t\tvar diff = this.frm.doc.total_taxes_and_charges - flt(total_charges);" + ], + [ + 119, + "\t\t\t\t\titems.slice(-1)[0].applicable_charges += diff;" + ], + [ + 120, + "\t\t\t\t}" + ], + [ + 121, + "\t\t\t\trefresh_field(\"items\");" + ], + [ + 122, + "\t\t\t}" + ], + [ + 123, + "\t\t}" + ], + [ + 124, + "\t}" + ] + ], + "before_line": 118, + "before_mapped": false, + "after": [ + [ + 115, + "\t\t\t\t});" + ], + [ + 116, + "" + ], + [ + 117, + "\t\t\t\tvar charge_precision = precision(\"applicable_charges\", items[0]);" + ], + [ + 118, + "\t\t\t\tvar total = flt(this.frm.doc.total_taxes_and_charges);" + ], + [ + 119, + "\t\t\t\tvar sign = total < 0 ? -1 : 1;" + ], + [ + 120, + "\t\t\t\tvar diff = flt(total - total_charges, charge_precision);" + ], + [ + 121, + "\t\t\t\tfor (const item of items.slice().reverse()) {" + ], + [ + 122, + "\t\t\t\t\tif (!diff) break;" + ], + [ + 123, + "\t\t\t\t\tvar share = flt(item.applicable_charges) * sign;" + ] + ], + "diff": "@@ -114,9 +114,19 @@ erpnext.stock.LandedCostVoucher = class LandedCostVoucher extends erpnext.stock.\n \t\t\t\t\ttotal_charges += item.applicable_charges;\n \t\t\t\t});\n\n-\t\t\t\tif (total_charges != this.frm.doc.total_taxes_and_charges) {\n-\t\t\t\t\tvar diff = this.frm.doc.total_taxes_and_charges - flt(total_charges);\n-\t\t\t\t\titems.slice(-1)[0].applicable_charges += diff;\n+\t\t\t\tvar charge_precision = precision(\"applicable_charges\", items[0]);\n+\t\t\t\tvar total = flt(this.frm.doc.total_taxes_and_charges);\n+\t\t\t\tvar sign = total < 0 ? -1 : 1;\n+\t\t\t\tvar diff = flt(total - total_charges, charge_precision);\n+\t\t\t\tfor (const item of items.slice().reverse()) {\n+\t\t\t\t\tif (!diff) break;\n+\t\t\t\t\tvar share = flt(item.applicable_charges) * sign;\n+\t\t\t\t\tvar adjustment = diff * sign > 0 ? diff : sign * Math.max(diff * sign, -share);\n+\t\t\t\t\titem.applicable_charges = flt(\n+\t\t\t\t\t\tflt(item.applicable_charges) + adjustment,\n+\t\t\t\t\t\tcharge_precision\n+\t\t\t\t\t);\n+\t\t\t\t\tdiff = flt(diff - adjustment, charge_precision);\n \t\t\t\t}\n \t\t\t\trefresh_field(\"items\");\n \t\t\t}\n" + }, + { + "item": "F015", + "kind": "introduced", + "repository": "calcom", + "commit_url": "https://github.com/calcom/cal.com/commit/ab21c7f805a089fa3a11ffd61c4a9aecc349c16c", + "rule": "CG-SEC-105", + "rule_title": "Hardcoded secret", + "path": "packages/features/bookings/lib/handleNewBooking/test/reschedule.test.ts", + "old_path": "packages/features/bookings/lib/handleNewBooking/test/reschedule.test.ts", + "old_line": null, + "line": 806, + "before": [ + [ + 687, + " });" + ], + [ + 688, + " }," + ], + [ + 689, + " timeout" + ], + [ + 690, + " );" + ], + [ + 691, + "" + ], + [ + 692, + " describe(\"Event Type that requires confirmation\", () => {" + ], + [ + 693, + " // NOTE: Tests for BOOKING_REQUESTED webhook have been moved to the new webhook architecture test suite" + ], + [ + 694, + " // See packages/features/webhooks/lib/__tests__/ for tests using the producer/consumer pattern" + ], + [ + 695, + "" + ], + [ + 696, + " test(" + ], + [ + 697, + " `should rechedule a booking, that requires confirmation, without confirmation - When booker is the organizer of the existing booking as well as the event-type" + ], + [ + 698, + " 1. Should cancel the existing booking" + ], + [ + 699, + " 2. Should delete existing calendar invite and Video meeting" + ] + ], + "before_line": 693, + "before_mapped": false, + "after": [ + [ + 802, + " {" + ], + [ + 803, + " type: appStoreMetadata.dailyvideo.type," + ], + [ + 804, + " uid: \"MOCK_ID\"," + ], + [ + 805, + " meetingId: \"MOCK_ID\"," + ], + [ + 806, + " meetingPassword: \"MOCK_PASS\"," + ], + [ + 807, + " meetingUrl: \"http://mock-dailyvideo.example.com\"," + ], + [ + 808, + " }," + ], + [ + 809, + " {" + ], + [ + 810, + " type: appStoreMetadata.googlecalendar.type," + ] + ], + "diff": "@@ -690,8 +660,201 @@ describe(\"handleNewBooking\", () => {\n );\n\n describe(\"Event Type that requires confirmation\", () => {\n- // NOTE: Tests for BOOKING_REQUESTED webhook have been moved to the new webhook architecture test suite\n- // See packages/features/webhooks/lib/__tests__/ for tests using the producer/consumer pattern\n+ test(\n+ `should reschedule a booking that requires confirmation in PENDING state - When a booker(who is not the organizer himself) is doing the reschedule\n+ 1. Should cancel the existing booking\n+ 2. Should delete existing calendar invite and Video meeting\n+ 2. Should create a new booking in the database in PENDING state\n+ 3. Should send BOOKING Requested scenario emails to the booker as well as organizer\n+ 4. Should trigger BOOKING_REQUESTED webhook instead of BOOKING_RESCHEDULED\n+ `,\n+ async ({ emails }) => {\n+ const handleNewBooking = getNewBookingHandler();\n+ const subscriberUrl = \"http://my-webhook.example.com\";\n+ const booker = getBooker({\n+ email: \"booker@example.com\",\n+ name: \"Booker\",\n+ });\n+\n+ const organizer = getOrganizer({\n+ name: \"Organizer\",\n+ email: \"organizer@example.com\",\n+ id: 101,\n+ schedules: [TestData.schedules.IstWorkHours],\n+ credentials: [getGoogleCalendarCredential()],\n+ selectedCalendars: [TestData.selectedCalendars.google],\n+ });\n+ const { dateString: plus1DateString } = getDate({ dateIncrement: 1 });\n+ const uidOfBookingToBeRescheduled = \"n5Wv3eHgconAED2j4gcVhP\";\n+\n+ const scenarioData = getScenarioData({\n+ webhooks: [\n+ {\n+ userId: organizer.id,\n+ eventTriggers: [\"BOOKING_CREATED\"],\n+ subscriberUrl,\n+ active: true,\n... (164 more lines)" + }, + { + "item": "F016", + "kind": "pair", + "repository": "kitsune", + "commit_url": "https://github.com/mozilla/kitsune/commit/59b6937db573505c764bec9c87bd786981848487", + "rule": "CG-QUAL-104", + "rule_title": "var declaration", + "path": "kitsune/sumo/static/sumo/js/markup.js", + "old_path": "kitsune/sumo/static/sumo/js/markup.js", + "old_line": 633, + "line": 633, + "before": [ + [ + 629, + " this.closeTag = \"\";" + ], + [ + 630, + " this.defaultText = \"\";" + ], + [ + 631, + " }," + ], + [ + 632, + " openModal: function (ev) {" + ], + [ + 633, + " var me = this;" + ], + [ + 634, + " var editorEl = me.textarea.closest(\".editor\");" + ], + [ + 635, + " var mediaSearchUrl = editorEl ? editorEl.dataset.mediaSearchUrl : undefined;" + ], + [ + 636, + " var galleryUrl = editorEl ? editorEl.dataset.mediaGalleryUrl : \"\";" + ], + [ + 637, + " var html = parseHTML(" + ] + ], + "before_line": 633, + "before_mapped": true, + "after": [ + [ + 629, + " this.closeTag = \"\";" + ], + [ + 630, + " this.defaultText = \"\";" + ], + [ + 631, + " }," + ], + [ + 632, + " openModal: function (ev) {" + ], + [ + 633, + " var me = this;" + ], + [ + 634, + " var editorEl = me.textarea.closest(\".editor\");" + ], + [ + 635, + " var mediaSearchUrl = editorEl ? editorEl.dataset.mediaSearchUrl : undefined;" + ], + [ + 636, + " var galleryUrl = editorEl ? editorEl.dataset.mediaGalleryUrl : \"\";" + ], + [ + 637, + " var html = parseHTML(" + ] + ], + "diff": "(no diff hunk near this line)" + }, + { + "item": "F017", + "kind": "pair", + "repository": "openlibrary", + "commit_url": "https://github.com/internetarchive/openlibrary/commit/53015559ebcd126850e7b07792ebc7bcfa847919", + "rule": "CG-QUAL-104", + "rule_title": "var declaration", + "path": "openlibrary/plugins/openlibrary/js/jsdef.js", + "old_path": "openlibrary/plugins/openlibrary/js/jsdef.js", + "old_line": 77, + "line": 80, + "before": [ + [ + 73, + " this.index0 = -1;" + ], + [ + 74, + "}" + ], + [ + 75, + "" + ], + [ + 76, + "ForLoop.prototype.next = function() {" + ], + [ + 77, + " var i = this.index0+1;" + ], + [ + 78, + "" + ], + [ + 79, + " this.index0 = i;" + ], + [ + 80, + " this.index = i+1;" + ], + [ + 81, + "" + ] + ], + "before_line": 77, + "before_mapped": true, + "after": [ + [ + 76, + " this.index0 = -1;" + ], + [ + 77, + "}" + ], + [ + 78, + "" + ], + [ + 79, + "ForLoop.prototype.next = function () {" + ], + [ + 80, + " var i = this.index0 + 1;" + ], + [ + 81, + "" + ], + [ + 82, + " this.index0 = i;" + ], + [ + 83, + " this.index = i + 1;" + ], + [ + 84, + "" + ] + ], + "diff": "@@ -73,17 +76,17 @@ export function ForLoop(parent, seq) {\n this.index0 = -1;\n }\n\n-ForLoop.prototype.next = function() {\n- var i = this.index0+1;\n+ForLoop.prototype.next = function () {\n+ var i = this.index0 + 1;\n\n this.index0 = i;\n- this.index = i+1;\n+ this.index = i + 1;\n\n- this.first = (i === 0);\n- this.last = (i === this.length-1);\n+ this.first = i === 0;\n+ this.last = i === this.length - 1;\n\n- this.odd = (this.index % 2 === 1);\n- this.even = (this.index % 2 === 0);\n+ this.odd = this.index % 2 === 1;\n+ this.even = this.index % 2 === 0;\n this.parity = ['even', 'odd'][this.index % 2];\n\n this.revindex0 = this.length - i;" + }, + { + "item": "F018", + "kind": "pair", + "repository": "ckan", + "commit_url": "https://github.com/ckan/ckan/commit/ea3c0fd8814de0fa83e744bcb4d5ab5a93902a34", + "rule": "CG-QUAL-003", + "rule_title": "Function is too complex", + "path": "ckan/logic/action/update.py", + "old_path": "ckan/logic/action/update.py", + "old_line": 474, + "line": 500, + "before": [ + [ + 470, + "" + ], + [ + 471, + " return pkg_custom" + ], + [ + 472, + "" + ], + [ + 473, + "" + ], + [ + 474, + "def package_revise(context: Context, data_dict: DataDict) -> ActionResult.PackageRevise:" + ], + [ + 475, + " '''Revise a dataset (package) selectively with match, filter and" + ], + [ + 476, + " update parameters." + ], + [ + 477, + "" + ], + [ + 478, + " You must be authorized to edit the dataset and the groups that it belongs" + ] + ], + "before_line": 474, + "before_mapped": true, + "after": [ + [ + 496, + "" + ], + [ + 497, + " return pkg_custom" + ], + [ + 498, + "" + ], + [ + 499, + "" + ], + [ + 500, + "def package_revise(context: Context, data_dict: DataDict) -> ActionResult.PackageRevise:" + ], + [ + 501, + " '''Revise a dataset (package) selectively with match, filter and" + ], + [ + 502, + " update parameters." + ], + [ + 503, + "" + ], + [ + 504, + " You must be authorized to edit the dataset and the groups that it belongs" + ] + ], + "diff": "(no diff hunk near this line)" + }, + { + "item": "F019", + "kind": "pair", + "repository": "inventree", + "commit_url": "https://github.com/inventree/InvenTree/commit/5244b8a4f21436f699ea266eb5a7381f899409f0", + "rule": "CG-QUAL-003", + "rule_title": "Function is too complex", + "path": "src/backend/InvenTree/plugin/registry.py", + "old_path": "src/backend/InvenTree/plugin/registry.py", + "old_line": 688, + "line": 689, + "before": [ + [ + 684, + "" + ], + [ + 685, + " # endregion" + ], + [ + 686, + "" + ], + [ + 687, + " # region general internal loading / activating / deactivating / unloading" + ], + [ + 688, + " def _init_plugin(self, plugin, configs: dict):" + ], + [ + 689, + " \"\"\"Initialise a single plugin." + ], + [ + 690, + "" + ], + [ + 691, + " Args:" + ], + [ + 692, + " plugin: Plugin module" + ] + ], + "before_line": 688, + "before_mapped": true, + "after": [ + [ + 685, + "" + ], + [ + 686, + " # endregion" + ], + [ + 687, + "" + ], + [ + 688, + " # region general internal loading / activating / deactivating / unloading" + ], + [ + 689, + " def _init_plugin(self, plugin, configs: dict):" + ], + [ + 690, + " \"\"\"Initialise a single plugin." + ], + [ + 691, + "" + ], + [ + 692, + " Args:" + ], + [ + 693, + " plugin: Plugin module" + ] + ], + "diff": "(no diff hunk near this line)" + }, + { + "item": "F020", + "kind": "introduced", + "repository": "ckan", + "commit_url": "https://github.com/ckan/ckan/commit/7a60264189f543fba04043d2193b09f545a6ef32", + "rule": "CG-QUAL-104", + "rule_title": "var declaration", + "path": "ckan/public-midnight-blue/base/javascript/modules/data-viewer.js", + "old_path": "ckan/public-midnight-blue/base/javascript/modules/data-viewer.js", + "old_line": null, + "line": 27, + "before": [ + [ + 21, + " $('.data-viewer-error', parent).removeClass('js-hide');" + ], + [ + 22, + " this.el.hide();" + ], + [ + 23, + " }," + ], + [ + 24, + "" + ], + [ + 25, + " _onLoad: function() {" + ], + [ + 26, + " var self = this;" + ], + [ + 27, + " var loc = $('body').data('site-root');" + ], + [ + 28, + " // see if page is in part of the same domain" + ], + [ + 29, + " if (this.el.attr('src').substring(0, loc.length) === loc) {" + ], + [ + 30, + " this._recalibrate();" + ], + [ + 31, + " setTimeout(function() {" + ], + [ + 32, + " self._recalibrate();" + ], + [ + 33, + " }, this.options.timeout);" + ] + ], + "before_line": 27, + "before_mapped": false, + "after": [ + [ + 23, + " }," + ], + [ + 24, + "" + ], + [ + 25, + " _onLoad: function() {" + ], + [ + 26, + " var self = this;" + ], + [ + 27, + " var loc = document.head.querySelector(['meta[name=site-root]'])?.content;" + ], + [ + 28, + " // see if page is in part of the same domain" + ], + [ + 29, + " if (this.el.attr('src').substring(0, loc.length) === loc) {" + ], + [ + 30, + " this._recalibrate();" + ], + [ + 31, + " setTimeout(function() {" + ] + ], + "diff": "@@ -24,7 +24,7 @@ this.ckan.module('data-viewer', function (jQuery) {\n\n _onLoad: function() {\n var self = this;\n- var loc = $('body').data('site-root');\n+ var loc = document.head.querySelector(['meta[name=site-root]'])?.content;\n // see if page is in part of the same domain\n if (this.el.attr('src').substring(0, loc.length) === loc) {\n this._recalibrate();\n" + }, + { + "item": "F021", + "kind": "pair", + "repository": "warehouse", + "commit_url": "https://github.com/pypi/warehouse/commit/7320d9052b59a7e3b1c2122034396f5d339cd3e0", + "rule": "CG-SEC-103", + "rule_title": "DOM XSS sink assigned a non-literal value", + "path": "tests/frontend/filter_list_controller_test.js", + "old_path": "tests/frontend/filter_list_controller_test.js", + "old_line": 39, + "line": 101, + "before": [ + [ + 35, + " describe(\"is initialized as expected\", () => {" + ], + [ + 36, + " describe(\"makes expected elements visible\", () => {" + ], + [ + 37, + " let application;" + ], + [ + 38, + " beforeEach(() => {" + ], + [ + 39, + " document.body.innerHTML = `" + ], + [ + 40, + "
" + ], + [ + 41, + " ${testFixtureHTMLVisibilityToggle}" + ], + [ + 42, + "
" + ], + [ + 43, + " `;" + ] + ], + "before_line": 39, + "before_mapped": true, + "after": [ + [ + 97, + " expect(Array.from(filterMyAttr.options).map(option => [option.value, option.selected])).toEqual(values);" + ], + [ + 98, + " };" + ], + [ + 99, + " const appStart = async function () {" + ], + [ + 100, + " const div = document.createElement(\"div\");" + ], + [ + 101, + " div.innerHTML = `" + ], + [ + 102, + "
" + ], + [ + 103, + " ${testFixtureHTMLVisibilityToggle}" + ], + [ + 104, + " ${testFixtureHTMLShowing}" + ], + [ + 105, + " ${testFixtureHTMLFilters}" + ] + ], + "diff": "@@ -5,226 +5,424 @@\n\n import {Application} from \"@hotwired/stimulus\";\n import FilterListController from \"../../warehouse/static/js/warehouse/controllers/filter_list_controller\";\n+import {delay} from \"./utils\";\n\n\n const testFixtureHTMLVisibilityToggle = `\n-

Initially hidden, should end up shown.

\n-

Initially shown, should end up hidden.

\n+

\n+ Initially hidden, should end up shown.\n+

\n+

\n+ Initially shown, should end up hidden.\n+

\n `;\n const testFixtureHTMLShowing = `\n

\n `;\n const testFixtureHTMLFilters = `\n-\n- \n+\n+\n+