From 7de8fc9c06090f02b1e627728d490befc9657b5b Mon Sep 17 00:00:00 2001 From: Jason Turner Date: Wed, 18 Nov 2026 04:23:50 +0100 Subject: [PATCH] Group survivors into patterns With dozens of survivors, the useful conclusions come from groups: twelve `&&` -> `||` survivors in one file mean the tests check only what is accepted. The new strain/patterns.py groups survivors that make the same kind of change in the same file, keeping groups of two or more. - Kinds: a comparison moved by one at its limit, an inverted comparison, == and != swapped, && to ||, || to &&, and arithmetic. Each comes with the test that would kill the whole group. - Each survivor is in one primary pattern at most. - Hotspots, the functions with the most survivors and their not_reached/not_detected split, are listed after the patterns. The console shows a Patterns section before the action items, and summary.md and index.html list them too. report.json gains a `patterns` array with each group's kind, file, advice and mutant IDs; hotspots have `"primary": false`. Fixes #44. Co-Authored-By: Claude Opus 5.5 --- README.md | 11 ++- strain/html_report.py | 15 ++++ strain/patterns.py | 177 +++++++++++++++++++++++++++++++++++++++ strain/report.py | 19 ++++- strain/runner.py | 10 +++ tests/test_end_to_end.py | 46 ++++++++++ tests/test_patterns.py | 87 +++++++++++++++++++ 7 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 strain/patterns.py create mode 100644 tests/test_patterns.py diff --git a/README.md b/README.md index 545cdef..616822e 100644 --- a/README.md +++ b/README.md @@ -78,9 +78,17 @@ Add `--ui live` (with `strain[ui]` installed) to watch the run in a full-screen +With many survivors, start with the **patterns**. They group survivors that make the same kind of change in the same +file, such as every `&&` → `||` or every comparison moved by one at its limit, and each group names the test that would +kill all of them. Each survivor is in one group at most. **Hotspots** are the functions with the most survivors. + ```text parser_tests score 87.8% killed 31 sanitizer 4 timeout 1 survived 5 +Patterns + · 3 survivors turn `&&` into `||` in lexer.cpp: add inputs that must be rejected, since the tests check only what is accepted + · hotspot: parse_header() in parser.cpp has 3 survivors: 1 not_reached, 2 not_detected + Action items 1. parse_header() is pseudo-tested: nothing checks its result 2. lexer.cpp:88 `==` → `!=` survived (not_reached): add a test where tok != END @@ -220,7 +228,8 @@ To suppress mutants in source, add `// mutate-off` to the line. Written to `strain-report/` : -- `report.json`: full results, in the mutation-testing-elements schema +- `report.json`: full results, in the mutation-testing-elements schema, plus a `patterns` array: each group of + survivors with its kind, file, advice and mutant IDs, and the hotspots with `"primary": false` - `events.jsonl`: every event of the run - compiler-style warnings on the console (`file:line:col: warning: survived [ror]`), clickable in editors diff --git a/strain/html_report.py b/strain/html_report.py index 892b445..ce5d374 100644 --- a/strain/html_report.py +++ b/strain/html_report.py @@ -1,9 +1,11 @@ """`index.html`: one self-contained page with the score, a table of files and the source annotated with its mutants.""" +import re from collections.abc import Iterable, Mapping, Sequence from html import escape from .console import HIGH_SCORE, LOW_SCORE +from .patterns import pattern_lines from .report import TAB_WIDTH, FileScore, file_scores, format_score, hint from .results import DETECTED_STATES, CombinedResult, MutantState, mutation_score @@ -126,6 +128,18 @@ def _survivor_list(combined: Sequence[CombinedResult]) -> list[str]: return [f"

Survivors ({len(survivors)})

", ""] +def _inline_code(text: str) -> str: + """`text`, escaped, with its Markdown code spans as .""" + return re.sub(r"`([^`]+)`", r"\1", escape(text)) + + +def _pattern_list(combined: Sequence[CombinedResult]) -> list[str]: + lines = pattern_lines(combined) + if not lines: + return [] + return ["

Patterns

", "
    ", *(f"
  • {_inline_code(line)}
  • " for line in lines), "
"] + + def _file_sections(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> list[str]: sections: list[str] = [] for row in file_scores(combined): @@ -151,6 +165,7 @@ def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) [ f"

{len(combined)} mutants: {_counts(combined)}

", *_files_table(combined), + *_pattern_list(combined), *_survivor_list(combined), *_file_sections(combined, sources), ] diff --git a/strain/patterns.py b/strain/patterns.py new file mode 100644 index 0000000..ac028fa --- /dev/null +++ b/strain/patterns.py @@ -0,0 +1,177 @@ +"""Survivors grouped into the few test-writing actions that would kill most of them. + +One survivor at a time says what to test; a group says what kind of test is missing. Twelve `&&` → `||` survivors in +one file mean the tests check only what is accepted, never the lookalikes that must be rejected. + +Each survivor belongs to one primary pattern at most: the kind of change it makes, in its file. Hotspots, the +functions with the most survivors, cut across those, so they are listed apart and are not primary. +""" + +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Any + +from .results import CombinedResult, MutantState, SurvivorKind + +# A pattern needs at least this many survivors; a single one is just a survivor. +MINIMUM_SURVIVORS = 2 +HOTSPOTS = 5 + + +@dataclass(frozen=True, slots=True) +class Family: + """A kind of change, with what the tests are missing when mutants of that kind survive.""" + + kind: str + change: str + advice: str + + +BOUNDARY = Family("boundary", "move a comparison's limit by one", "add a test exactly at the limit") +NEGATION = Family("negated-comparison", "invert a comparison", "add a test on each side of the comparison") +EQUALITY = Family("equality", "swap `==` and `!=`", "add a test where the values are equal and one where they differ") +AND_TO_OR = Family( + "and-to-or", + "turn `&&` into `||`", + "add inputs that must be rejected, since the tests check only what is accepted", +) +OR_TO_AND = Family("or-to-and", "turn `||` into `&&`", "add an input that satisfies only one of the alternatives") +ARITHMETIC = Family( + "arithmetic", "change an arithmetic operator", "assert the computed values, not just that there are results" +) +FAMILIES = (BOUNDARY, NEGATION, EQUALITY, AND_TO_OR, OR_TO_AND, ARITHMETIC) + +BOUNDARY_CHANGES = frozenset({("<", "<="), ("<=", "<"), (">", ">="), (">=", ">")}) +NEGATION_CHANGES = frozenset({("<", ">="), (">=", "<"), (">", "<="), ("<=", ">")}) + + +def family_of(entry: CombinedResult) -> Family | None: + change = (entry.mutant.original, entry.mutant.replacement) + if change in BOUNDARY_CHANGES: + return BOUNDARY + if change in NEGATION_CHANGES: + return NEGATION + if change in {("==", "!="), ("!=", "==")}: + return EQUALITY + if change == ("&&", "||"): + return AND_TO_OR + if change == ("||", "&&"): + return OR_TO_AND + if entry.mutant.operator in ("aor", "asr"): + return ARITHMETIC + return None + + +def file_name(file: str) -> str: + return PurePosixPath(file).name + + +@dataclass(frozen=True, slots=True) +class Pattern: + family: Family + file: str + mutant_ids: tuple[str, ...] + + def summary(self) -> str: + """`12 survivors turn `&&` into `||` in parser.cpp: add inputs that must be rejected…`""" + return f"{len(self.mutant_ids)} survivors {self.family.change} in {file_name(self.file)}: {self.family.advice}" + + +@dataclass(frozen=True, slots=True) +class Hotspot: + function: str + file: str + mutant_ids: tuple[str, ...] + not_reached: int + not_detected: int + + def summary(self) -> str: + """`clamp() in math.cpp has 3 survivors: 2 not_reached, 1 not_detected`""" + split = [ + f"{count} {kind}" + for kind, count in (("not_reached", self.not_reached), ("not_detected", self.not_detected)) + if count + ] + name = f"{self.function.partition('(')[0]}()" + detail = f": {', '.join(split)}" if split else "" + return f"{name} in {file_name(self.file)} has {len(self.mutant_ids)} survivors{detail}" + + +def survivors(combined: Sequence[CombinedResult]) -> list[CombinedResult]: + return [entry for entry in combined if entry.state is MutantState.SURVIVED] + + +def find_patterns(combined: Sequence[CombinedResult]) -> list[Pattern]: + """The primary patterns, largest first: survivors that make the same kind of change in the same file.""" + groups: dict[tuple[Family, str], list[str]] = {} + for entry in survivors(combined): + family = family_of(entry) + if family is not None: + groups.setdefault((family, entry.mutant.file), []).append(entry.mutant.id) + patterns = [ + Pattern(family, file, tuple(ids)) for (family, file), ids in groups.items() if len(ids) >= MINIMUM_SURVIVORS + ] + return sorted( + patterns, key=lambda pattern: (-len(pattern.mutant_ids), pattern.file, FAMILIES.index(pattern.family)) + ) + + +def find_hotspots(combined: Sequence[CombinedResult], limit: int = HOTSPOTS) -> list[Hotspot]: + """The functions with the most survivors, and how many need a new input or a stronger assertion.""" + groups: dict[tuple[str, str], list[CombinedResult]] = {} + for entry in survivors(combined): + if entry.function is not None: + groups.setdefault((entry.function, entry.mutant.file), []).append(entry) + hotspots = [ + Hotspot( + function, + file, + tuple(entry.mutant.id for entry in entries), + sum(entry.survivor is SurvivorKind.NOT_REACHED for entry in entries), + sum(entry.survivor is SurvivorKind.NOT_DETECTED for entry in entries), + ) + for (function, file), entries in groups.items() + if len(entries) >= MINIMUM_SURVIVORS + ] + return sorted(hotspots, key=lambda hotspot: (-len(hotspot.mutant_ids), hotspot.file, hotspot.function))[:limit] + + +def patterns_json(combined: Sequence[CombinedResult]) -> list[dict[str, Any]]: + """The patterns for report.json; `primary` tells the patterns that share no survivor from the hotspots.""" + return [ + *( + { + "kind": pattern.family.kind, + "primary": True, + "file": pattern.file, + "survivors": len(pattern.mutant_ids), + "summary": pattern.summary(), + "advice": pattern.family.advice, + "mutants": list(pattern.mutant_ids), + } + for pattern in find_patterns(combined) + ), + *( + { + "kind": "hotspot", + "primary": False, + "file": hotspot.file, + "function": hotspot.function, + "survivors": len(hotspot.mutant_ids), + "notReached": hotspot.not_reached, + "notDetected": hotspot.not_detected, + "summary": hotspot.summary(), + "mutants": list(hotspot.mutant_ids), + } + for hotspot in find_hotspots(combined) + ), + ] + + +def pattern_lines(combined: Sequence[CombinedResult]) -> list[str]: + """One line per pattern, then per hotspot, for the console, summary.md and the HTML report.""" + return [ + *(pattern.summary() for pattern in find_patterns(combined)), + *(f"hotspot: {hotspot.summary()}" for hotspot in find_hotspots(combined)), + ] diff --git a/strain/report.py b/strain/report.py index 89c735b..c3ef966 100644 --- a/strain/report.py +++ b/strain/report.py @@ -8,6 +8,7 @@ from typing import Any, Self, TextIO from .mutators import Mutant +from .patterns import pattern_lines, patterns_json from .results import ( DETECTED_STATES, CombinedResult, @@ -183,6 +184,13 @@ def _markdown_file_table(combined: Sequence[CombinedResult]) -> list[str]: ] +def _markdown_patterns(combined: Sequence[CombinedResult]) -> list[str]: + lines = pattern_lines(combined) + if not lines: + return [] + return ["", "### Patterns", "", *(f"- {line}" for line in lines)] + + def _markdown_survivors(combined: Sequence[CombinedResult]) -> list[str]: survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED] if not survivors: @@ -212,6 +220,7 @@ def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None _markdown_counts(combined), "", *_markdown_file_table(combined), + *_markdown_patterns(combined), *_markdown_survivors(combined), ] return "\n".join(lines) + "\n" @@ -234,14 +243,20 @@ def _mutant_element(entry: CombinedResult) -> dict[str, Any]: } -def mutation_testing_elements(combined: Iterable[CombinedResult], sources: Mapping[str, str]) -> dict[str, Any]: +def mutation_testing_elements(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> dict[str, Any]: + """The report in the mutation-testing-elements schema, plus strain's `patterns` for tools and agents.""" files: dict[str, Any] = {} for entry in combined: file_report = files.setdefault( entry.mutant.file, {"language": "cpp", "source": sources[entry.mutant.file], "mutants": []} ) file_report["mutants"].append(_mutant_element(entry)) - return {"schemaVersion": "1", "thresholds": {"high": 80, "low": 60}, "files": files} + return { + "schemaVersion": "1", + "thresholds": {"high": 80, "low": 60}, + "files": files, + "patterns": patterns_json(combined), + } PreviousResults = dict[tuple[str, str], tuple[MutantState, SurvivorKind | None]] diff --git a/strain/runner.py b/strain/runner.py index c4f7afd..1000932 100644 --- a/strain/runner.py +++ b/strain/runner.py @@ -43,6 +43,7 @@ ) from .operator_sites import EngineError, OperatorSite from .html_report import html_report +from .patterns import pattern_lines from .report import ( EventLog, FileScore, @@ -614,6 +615,14 @@ def file_line(self, row: FileScore, width: int) -> str: f" {console.paint(f'{row.survived} survived', survived_style)}" ) + def show_patterns(self, combined: Sequence[CombinedResult]) -> None: + """The groups of survivors that one kind of test would kill, ahead of the survivors one by one.""" + lines = pattern_lines(combined) + if lines: + self.console.heading("Patterns") + for line in lines: + self.console.line(f" · {line}") + def show_action_items(self, combined: Sequence[CombinedResult]) -> None: items = action_items(combined, limit=None if self.options.show_all else DEFAULT_ACTION_ITEMS) if items: @@ -791,6 +800,7 @@ def report(self, combined: Sequence[CombinedResult], events: EventLog, ending: s self.write_reports(combined, sources) events.emit(ending, score=score) self.show_survivors(combined, sources) + self.show_patterns(combined) self.show_action_items(combined) self.show_files(combined) self.show_summary(combined, score) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index e668dd4..0f0da33 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -97,6 +97,21 @@ def mutant_evaluated(self, result: MutantResult, source: str, line: LineCoverage ] +RECOGNISER_HEADER = "[[nodiscard]] bool is_identifier_start(char c);\n" +RECOGNISER_SOURCE = """#include "sample.hpp" + +bool is_identifier_start(char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; +} +""" +# Weak on purpose: only inputs that must be accepted, none at a limit, none that must be rejected. +RECOGNISER_TEST = """#include "sample.hpp" + +int main() { return is_identifier_start('m') && is_identifier_start('M') && is_identifier_start('_') ? 0 : 1; } +""" + + def run_strain(*argv: str) -> tuple[int, str, str]: stdout = io.StringIO() stderr = io.StringIO() @@ -503,6 +518,37 @@ def test_interrupted_run_still_writes_the_reports(self) -> None: self.assertEqual(console.finished_runs, 0) self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source) + def test_survivors_of_weak_tests_group_into_the_patterns_that_name_the_missing_tests(self) -> None: + (self.root / "include" / "sample.hpp").write_text(RECOGNISER_HEADER) + (self.root / "src" / "sample.cpp").write_text(RECOGNISER_SOURCE) + (self.root / "tests" / "sample_tests.cpp").write_text(RECOGNISER_TEST) + (self.root / "tests" / "other_tests.cpp").write_text("int main() { return 0; }\n") + configure_project(self.root, self.build_dir) + + exit_code, stdout, _ = run_strain( + *("--build-dir", str(self.build_dir), "--report-dir", str(self.report_dir)), + *("-R", "^sample_tests$", "--engine", "lexer"), + ) + + self.assertEqual(exit_code, 0, stdout) + boundary = "4 survivors move a comparison's limit by one in sample.cpp: add a test exactly at the limit" + and_to_or = ( + "2 survivors turn `&&` into `||` in sample.cpp: add inputs that must be rejected, " + "since the tests check only what is accepted" + ) + hotspot = "hotspot: is_identifier_start() in sample.cpp has 6 survivors" + self.assertIn(f"▸ Patterns\n · {boundary}\n · {and_to_or}\n · {hotspot}", stdout) + summary = (self.report_dir / "summary.md").read_text() + self.assertIn(f"### Patterns\n\n- {boundary}\n- {and_to_or}\n- {hotspot}", summary) + report = json.loads((self.report_dir / "report.json").read_text()) + primary = [pattern for pattern in report["patterns"] if pattern["primary"]] + self.assertEqual( + [(pattern["kind"], pattern["survivors"]) for pattern in primary], [("boundary", 4), ("and-to-or", 2)] + ) + ids = [mutant for pattern in primary for mutant in pattern["mutants"]] + self.assertEqual(len(ids), len(set(ids))) + self.assertIn("

Patterns

", (self.report_dir / "index.html").read_text()) + def test_dirty_working_tree_is_refused(self) -> None: configure_project(self.root, self.build_dir) (self.root / "src" / "sample.cpp").write_text("// local edit\n") diff --git a/tests/test_patterns.py b/tests/test_patterns.py new file mode 100644 index 0000000..35c2d86 --- /dev/null +++ b/tests/test_patterns.py @@ -0,0 +1,87 @@ +import unittest + +from strain.mutators import Mutant +from strain.patterns import ( + AND_TO_OR, + BOUNDARY, + find_hotspots, + find_patterns, + pattern_lines, + patterns_json, +) +from strain.results import CombinedResult, MutantState, SurvivorKind + + +def entry( + original: str, + replacement: str, + operator: str = "ror", + file: str = "src/parser.cpp", + state: MutantState = MutantState.SURVIVED, + function: str | None = "accept(char)", + survivor: SurvivorKind | None = SurvivorKind.NOT_DETECTED, + offset: int = 0, +) -> CombinedResult: + mutant = Mutant(file, 1, offset + 1, offset, operator, original, replacement) + return CombinedResult(mutant, state, survivor, function, (), ("t",)) + + +class PatternsTest(unittest.TestCase): + def test_survivors_group_by_the_kind_of_change_and_file(self) -> None: + combined = [ + entry("&&", "||", "lcr", offset=1), + entry("&&", "||", "lcr", offset=2), + entry("&&", "||", "lcr", offset=3), + entry("<", "<=", offset=4), + entry(">=", ">", offset=5), + entry("&&", "||", "lcr", file="src/lexer.cpp", offset=6), + entry("&&", "||", "lcr", state=MutantState.KILLED, offset=7), + ] + patterns = find_patterns(combined) + self.assertEqual( + [(pattern.family, pattern.file, len(pattern.mutant_ids)) for pattern in patterns], + [(AND_TO_OR, "src/parser.cpp", 3), (BOUNDARY, "src/parser.cpp", 2)], + ) + self.assertEqual( + patterns[0].summary(), + "3 survivors turn `&&` into `||` in parser.cpp: add inputs that must be rejected, " + "since the tests check only what is accepted", + ) + + def test_each_survivor_is_in_one_primary_pattern_at_most(self) -> None: + combined = [ + entry(original, replacement, offset=index) + for index, (original, replacement) in enumerate( + [("<", "<="), ("<", ">="), ("<=", "<"), ("<=", ">"), ("==", "!="), ("!=", "==")] + ) + ] + ids = [mutant_id for pattern in find_patterns(combined) for mutant_id in pattern.mutant_ids] + self.assertEqual(sorted(ids), sorted(item.mutant.id for item in combined)) + self.assertEqual(len(ids), len(set(ids))) + + def test_hotspots_split_new_inputs_from_stronger_assertions(self) -> None: + combined = [ + entry("+", "-", "aor", function="scale(int)", survivor=SurvivorKind.NOT_REACHED, offset=1), + entry("*", "/", "aor", function="scale(int)", survivor=SurvivorKind.NOT_REACHED, offset=2), + entry("<", "<=", function="scale(int)", offset=3), + entry("<", "<=", function="clamp(int)", offset=4), + ] + [hotspot] = find_hotspots(combined) + self.assertEqual(hotspot.summary(), "scale() in parser.cpp has 3 survivors: 2 not_reached, 1 not_detected") + + def test_report_json_marks_hotspots_as_not_primary(self) -> None: + combined = [entry("&&", "||", "lcr", offset=1), entry("&&", "||", "lcr", offset=2)] + pattern, hotspot = patterns_json(combined) + self.assertEqual( + {key: pattern[key] for key in ("kind", "primary", "file", "survivors")}, + {"kind": "and-to-or", "primary": True, "file": "src/parser.cpp", "survivors": 2}, + ) + self.assertEqual(pattern["mutants"], [item.mutant.id for item in combined]) + self.assertEqual((hotspot["kind"], hotspot["primary"], hotspot["function"]), ("hotspot", False, "accept(char)")) + + def test_a_single_survivor_is_not_a_pattern(self) -> None: + self.assertEqual(pattern_lines([entry("&&", "||", "lcr", function=None)]), []) + + +if __name__ == "__main__": + unittest.main()