diff --git a/README.md b/README.md index 545cdef..616822e 100644 --- a/README.md +++ b/README.md @@ -78,9 +78,17 @@ Add `--ui live` (with `strain[ui]` installed) to watch the run in a full-screen +With many survivors, start with the **patterns**. They group survivors that make the same kind of change in the same +file, such as every `&&` → `||` or every comparison moved by one at its limit, and each group names the test that would +kill all of them. Each survivor is in one group at most. **Hotspots** are the functions with the most survivors. + ```text parser_tests score 87.8% killed 31 sanitizer 4 timeout 1 survived 5 +Patterns + · 3 survivors turn `&&` into `||` in lexer.cpp: add inputs that must be rejected, since the tests check only what is accepted + · hotspot: parse_header() in parser.cpp has 3 survivors: 1 not_reached, 2 not_detected + Action items 1. parse_header() is pseudo-tested: nothing checks its result 2. lexer.cpp:88 `==` → `!=` survived (not_reached): add a test where tok != END @@ -220,7 +228,8 @@ To suppress mutants in source, add `// mutate-off` to the line. Written to `strain-report/` : -- `report.json`: full results, in the mutation-testing-elements schema +- `report.json`: full results, in the mutation-testing-elements schema, plus a `patterns` array: each group of + survivors with its kind, file, advice and mutant IDs, and the hotspots with `"primary": false` - `events.jsonl`: every event of the run - compiler-style warnings on the console (`file:line:col: warning: survived [ror]`), clickable in editors diff --git a/strain/html_report.py b/strain/html_report.py index 892b445..ce5d374 100644 --- a/strain/html_report.py +++ b/strain/html_report.py @@ -1,9 +1,11 @@ """`index.html`: one self-contained page with the score, a table of files and the source annotated with its mutants.""" +import re from collections.abc import Iterable, Mapping, Sequence from html import escape from .console import HIGH_SCORE, LOW_SCORE +from .patterns import pattern_lines from .report import TAB_WIDTH, FileScore, file_scores, format_score, hint from .results import DETECTED_STATES, CombinedResult, MutantState, mutation_score @@ -126,6 +128,18 @@ def _survivor_list(combined: Sequence[CombinedResult]) -> list[str]: return [f"
."""
+ return re.sub(r"`([^`]+)`", r"\1", escape(text))
+
+
+def _pattern_list(combined: Sequence[CombinedResult]) -> list[str]:
+ lines = pattern_lines(combined)
+ if not lines:
+ return []
+ return ["Patterns
", "", *(f"- {_inline_code(line)}
" for line in lines), "
"]
+
+
def _file_sections(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> list[str]:
sections: list[str] = []
for row in file_scores(combined):
@@ -151,6 +165,7 @@ def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str])
[
f"{len(combined)} mutants: {_counts(combined)}
",
*_files_table(combined),
+ *_pattern_list(combined),
*_survivor_list(combined),
*_file_sections(combined, sources),
]
diff --git a/strain/patterns.py b/strain/patterns.py
new file mode 100644
index 0000000..ac028fa
--- /dev/null
+++ b/strain/patterns.py
@@ -0,0 +1,177 @@
+"""Survivors grouped into the few test-writing actions that would kill most of them.
+
+One survivor at a time says what to test; a group says what kind of test is missing. Twelve `&&` → `||` survivors in
+one file mean the tests check only what is accepted, never the lookalikes that must be rejected.
+
+Each survivor belongs to one primary pattern at most: the kind of change it makes, in its file. Hotspots, the
+functions with the most survivors, cut across those, so they are listed apart and are not primary.
+"""
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import PurePosixPath
+from typing import Any
+
+from .results import CombinedResult, MutantState, SurvivorKind
+
+# A pattern needs at least this many survivors; a single one is just a survivor.
+MINIMUM_SURVIVORS = 2
+HOTSPOTS = 5
+
+
+@dataclass(frozen=True, slots=True)
+class Family:
+ """A kind of change, with what the tests are missing when mutants of that kind survive."""
+
+ kind: str
+ change: str
+ advice: str
+
+
+BOUNDARY = Family("boundary", "move a comparison's limit by one", "add a test exactly at the limit")
+NEGATION = Family("negated-comparison", "invert a comparison", "add a test on each side of the comparison")
+EQUALITY = Family("equality", "swap `==` and `!=`", "add a test where the values are equal and one where they differ")
+AND_TO_OR = Family(
+ "and-to-or",
+ "turn `&&` into `||`",
+ "add inputs that must be rejected, since the tests check only what is accepted",
+)
+OR_TO_AND = Family("or-to-and", "turn `||` into `&&`", "add an input that satisfies only one of the alternatives")
+ARITHMETIC = Family(
+ "arithmetic", "change an arithmetic operator", "assert the computed values, not just that there are results"
+)
+FAMILIES = (BOUNDARY, NEGATION, EQUALITY, AND_TO_OR, OR_TO_AND, ARITHMETIC)
+
+BOUNDARY_CHANGES = frozenset({("<", "<="), ("<=", "<"), (">", ">="), (">=", ">")})
+NEGATION_CHANGES = frozenset({("<", ">="), (">=", "<"), (">", "<="), ("<=", ">")})
+
+
+def family_of(entry: CombinedResult) -> Family | None:
+ change = (entry.mutant.original, entry.mutant.replacement)
+ if change in BOUNDARY_CHANGES:
+ return BOUNDARY
+ if change in NEGATION_CHANGES:
+ return NEGATION
+ if change in {("==", "!="), ("!=", "==")}:
+ return EQUALITY
+ if change == ("&&", "||"):
+ return AND_TO_OR
+ if change == ("||", "&&"):
+ return OR_TO_AND
+ if entry.mutant.operator in ("aor", "asr"):
+ return ARITHMETIC
+ return None
+
+
+def file_name(file: str) -> str:
+ return PurePosixPath(file).name
+
+
+@dataclass(frozen=True, slots=True)
+class Pattern:
+ family: Family
+ file: str
+ mutant_ids: tuple[str, ...]
+
+ def summary(self) -> str:
+ """`12 survivors turn `&&` into `||` in parser.cpp: add inputs that must be rejected…`"""
+ return f"{len(self.mutant_ids)} survivors {self.family.change} in {file_name(self.file)}: {self.family.advice}"
+
+
+@dataclass(frozen=True, slots=True)
+class Hotspot:
+ function: str
+ file: str
+ mutant_ids: tuple[str, ...]
+ not_reached: int
+ not_detected: int
+
+ def summary(self) -> str:
+ """`clamp() in math.cpp has 3 survivors: 2 not_reached, 1 not_detected`"""
+ split = [
+ f"{count} {kind}"
+ for kind, count in (("not_reached", self.not_reached), ("not_detected", self.not_detected))
+ if count
+ ]
+ name = f"{self.function.partition('(')[0]}()"
+ detail = f": {', '.join(split)}" if split else ""
+ return f"{name} in {file_name(self.file)} has {len(self.mutant_ids)} survivors{detail}"
+
+
+def survivors(combined: Sequence[CombinedResult]) -> list[CombinedResult]:
+ return [entry for entry in combined if entry.state is MutantState.SURVIVED]
+
+
+def find_patterns(combined: Sequence[CombinedResult]) -> list[Pattern]:
+ """The primary patterns, largest first: survivors that make the same kind of change in the same file."""
+ groups: dict[tuple[Family, str], list[str]] = {}
+ for entry in survivors(combined):
+ family = family_of(entry)
+ if family is not None:
+ groups.setdefault((family, entry.mutant.file), []).append(entry.mutant.id)
+ patterns = [
+ Pattern(family, file, tuple(ids)) for (family, file), ids in groups.items() if len(ids) >= MINIMUM_SURVIVORS
+ ]
+ return sorted(
+ patterns, key=lambda pattern: (-len(pattern.mutant_ids), pattern.file, FAMILIES.index(pattern.family))
+ )
+
+
+def find_hotspots(combined: Sequence[CombinedResult], limit: int = HOTSPOTS) -> list[Hotspot]:
+ """The functions with the most survivors, and how many need a new input or a stronger assertion."""
+ groups: dict[tuple[str, str], list[CombinedResult]] = {}
+ for entry in survivors(combined):
+ if entry.function is not None:
+ groups.setdefault((entry.function, entry.mutant.file), []).append(entry)
+ hotspots = [
+ Hotspot(
+ function,
+ file,
+ tuple(entry.mutant.id for entry in entries),
+ sum(entry.survivor is SurvivorKind.NOT_REACHED for entry in entries),
+ sum(entry.survivor is SurvivorKind.NOT_DETECTED for entry in entries),
+ )
+ for (function, file), entries in groups.items()
+ if len(entries) >= MINIMUM_SURVIVORS
+ ]
+ return sorted(hotspots, key=lambda hotspot: (-len(hotspot.mutant_ids), hotspot.file, hotspot.function))[:limit]
+
+
+def patterns_json(combined: Sequence[CombinedResult]) -> list[dict[str, Any]]:
+ """The patterns for report.json; `primary` tells the patterns that share no survivor from the hotspots."""
+ return [
+ *(
+ {
+ "kind": pattern.family.kind,
+ "primary": True,
+ "file": pattern.file,
+ "survivors": len(pattern.mutant_ids),
+ "summary": pattern.summary(),
+ "advice": pattern.family.advice,
+ "mutants": list(pattern.mutant_ids),
+ }
+ for pattern in find_patterns(combined)
+ ),
+ *(
+ {
+ "kind": "hotspot",
+ "primary": False,
+ "file": hotspot.file,
+ "function": hotspot.function,
+ "survivors": len(hotspot.mutant_ids),
+ "notReached": hotspot.not_reached,
+ "notDetected": hotspot.not_detected,
+ "summary": hotspot.summary(),
+ "mutants": list(hotspot.mutant_ids),
+ }
+ for hotspot in find_hotspots(combined)
+ ),
+ ]
+
+
+def pattern_lines(combined: Sequence[CombinedResult]) -> list[str]:
+ """One line per pattern, then per hotspot, for the console, summary.md and the HTML report."""
+ return [
+ *(pattern.summary() for pattern in find_patterns(combined)),
+ *(f"hotspot: {hotspot.summary()}" for hotspot in find_hotspots(combined)),
+ ]
diff --git a/strain/report.py b/strain/report.py
index 89c735b..c3ef966 100644
--- a/strain/report.py
+++ b/strain/report.py
@@ -8,6 +8,7 @@
from typing import Any, Self, TextIO
from .mutators import Mutant
+from .patterns import pattern_lines, patterns_json
from .results import (
DETECTED_STATES,
CombinedResult,
@@ -183,6 +184,13 @@ def _markdown_file_table(combined: Sequence[CombinedResult]) -> list[str]:
]
+def _markdown_patterns(combined: Sequence[CombinedResult]) -> list[str]:
+ lines = pattern_lines(combined)
+ if not lines:
+ return []
+ return ["", "### Patterns", "", *(f"- {line}" for line in lines)]
+
+
def _markdown_survivors(combined: Sequence[CombinedResult]) -> list[str]:
survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED]
if not survivors:
@@ -212,6 +220,7 @@ def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None
_markdown_counts(combined),
"",
*_markdown_file_table(combined),
+ *_markdown_patterns(combined),
*_markdown_survivors(combined),
]
return "\n".join(lines) + "\n"
@@ -234,14 +243,20 @@ def _mutant_element(entry: CombinedResult) -> dict[str, Any]:
}
-def mutation_testing_elements(combined: Iterable[CombinedResult], sources: Mapping[str, str]) -> dict[str, Any]:
+def mutation_testing_elements(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> dict[str, Any]:
+ """The report in the mutation-testing-elements schema, plus strain's `patterns` for tools and agents."""
files: dict[str, Any] = {}
for entry in combined:
file_report = files.setdefault(
entry.mutant.file, {"language": "cpp", "source": sources[entry.mutant.file], "mutants": []}
)
file_report["mutants"].append(_mutant_element(entry))
- return {"schemaVersion": "1", "thresholds": {"high": 80, "low": 60}, "files": files}
+ return {
+ "schemaVersion": "1",
+ "thresholds": {"high": 80, "low": 60},
+ "files": files,
+ "patterns": patterns_json(combined),
+ }
PreviousResults = dict[tuple[str, str], tuple[MutantState, SurvivorKind | None]]
diff --git a/strain/runner.py b/strain/runner.py
index c4f7afd..1000932 100644
--- a/strain/runner.py
+++ b/strain/runner.py
@@ -43,6 +43,7 @@
)
from .operator_sites import EngineError, OperatorSite
from .html_report import html_report
+from .patterns import pattern_lines
from .report import (
EventLog,
FileScore,
@@ -614,6 +615,14 @@ def file_line(self, row: FileScore, width: int) -> str:
f" {console.paint(f'{row.survived} survived', survived_style)}"
)
+ def show_patterns(self, combined: Sequence[CombinedResult]) -> None:
+ """The groups of survivors that one kind of test would kill, ahead of the survivors one by one."""
+ lines = pattern_lines(combined)
+ if lines:
+ self.console.heading("Patterns")
+ for line in lines:
+ self.console.line(f" · {line}")
+
def show_action_items(self, combined: Sequence[CombinedResult]) -> None:
items = action_items(combined, limit=None if self.options.show_all else DEFAULT_ACTION_ITEMS)
if items:
@@ -791,6 +800,7 @@ def report(self, combined: Sequence[CombinedResult], events: EventLog, ending: s
self.write_reports(combined, sources)
events.emit(ending, score=score)
self.show_survivors(combined, sources)
+ self.show_patterns(combined)
self.show_action_items(combined)
self.show_files(combined)
self.show_summary(combined, score)
diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py
index e668dd4..0f0da33 100644
--- a/tests/test_end_to_end.py
+++ b/tests/test_end_to_end.py
@@ -97,6 +97,21 @@ def mutant_evaluated(self, result: MutantResult, source: str, line: LineCoverage
]
+RECOGNISER_HEADER = "[[nodiscard]] bool is_identifier_start(char c);\n"
+RECOGNISER_SOURCE = """#include "sample.hpp"
+
+bool is_identifier_start(char c)
+{
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_';
+}
+"""
+# Weak on purpose: only inputs that must be accepted, none at a limit, none that must be rejected.
+RECOGNISER_TEST = """#include "sample.hpp"
+
+int main() { return is_identifier_start('m') && is_identifier_start('M') && is_identifier_start('_') ? 0 : 1; }
+"""
+
+
def run_strain(*argv: str) -> tuple[int, str, str]:
stdout = io.StringIO()
stderr = io.StringIO()
@@ -503,6 +518,37 @@ def test_interrupted_run_still_writes_the_reports(self) -> None:
self.assertEqual(console.finished_runs, 0)
self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source)
+ def test_survivors_of_weak_tests_group_into_the_patterns_that_name_the_missing_tests(self) -> None:
+ (self.root / "include" / "sample.hpp").write_text(RECOGNISER_HEADER)
+ (self.root / "src" / "sample.cpp").write_text(RECOGNISER_SOURCE)
+ (self.root / "tests" / "sample_tests.cpp").write_text(RECOGNISER_TEST)
+ (self.root / "tests" / "other_tests.cpp").write_text("int main() { return 0; }\n")
+ configure_project(self.root, self.build_dir)
+
+ exit_code, stdout, _ = run_strain(
+ *("--build-dir", str(self.build_dir), "--report-dir", str(self.report_dir)),
+ *("-R", "^sample_tests$", "--engine", "lexer"),
+ )
+
+ self.assertEqual(exit_code, 0, stdout)
+ boundary = "4 survivors move a comparison's limit by one in sample.cpp: add a test exactly at the limit"
+ and_to_or = (
+ "2 survivors turn `&&` into `||` in sample.cpp: add inputs that must be rejected, "
+ "since the tests check only what is accepted"
+ )
+ hotspot = "hotspot: is_identifier_start() in sample.cpp has 6 survivors"
+ self.assertIn(f"▸ Patterns\n · {boundary}\n · {and_to_or}\n · {hotspot}", stdout)
+ summary = (self.report_dir / "summary.md").read_text()
+ self.assertIn(f"### Patterns\n\n- {boundary}\n- {and_to_or}\n- {hotspot}", summary)
+ report = json.loads((self.report_dir / "report.json").read_text())
+ primary = [pattern for pattern in report["patterns"] if pattern["primary"]]
+ self.assertEqual(
+ [(pattern["kind"], pattern["survivors"]) for pattern in primary], [("boundary", 4), ("and-to-or", 2)]
+ )
+ ids = [mutant for pattern in primary for mutant in pattern["mutants"]]
+ self.assertEqual(len(ids), len(set(ids)))
+ self.assertIn("Patterns
", (self.report_dir / "index.html").read_text())
+
def test_dirty_working_tree_is_refused(self) -> None:
configure_project(self.root, self.build_dir)
(self.root / "src" / "sample.cpp").write_text("// local edit\n")
diff --git a/tests/test_patterns.py b/tests/test_patterns.py
new file mode 100644
index 0000000..35c2d86
--- /dev/null
+++ b/tests/test_patterns.py
@@ -0,0 +1,87 @@
+import unittest
+
+from strain.mutators import Mutant
+from strain.patterns import (
+ AND_TO_OR,
+ BOUNDARY,
+ find_hotspots,
+ find_patterns,
+ pattern_lines,
+ patterns_json,
+)
+from strain.results import CombinedResult, MutantState, SurvivorKind
+
+
+def entry(
+ original: str,
+ replacement: str,
+ operator: str = "ror",
+ file: str = "src/parser.cpp",
+ state: MutantState = MutantState.SURVIVED,
+ function: str | None = "accept(char)",
+ survivor: SurvivorKind | None = SurvivorKind.NOT_DETECTED,
+ offset: int = 0,
+) -> CombinedResult:
+ mutant = Mutant(file, 1, offset + 1, offset, operator, original, replacement)
+ return CombinedResult(mutant, state, survivor, function, (), ("t",))
+
+
+class PatternsTest(unittest.TestCase):
+ def test_survivors_group_by_the_kind_of_change_and_file(self) -> None:
+ combined = [
+ entry("&&", "||", "lcr", offset=1),
+ entry("&&", "||", "lcr", offset=2),
+ entry("&&", "||", "lcr", offset=3),
+ entry("<", "<=", offset=4),
+ entry(">=", ">", offset=5),
+ entry("&&", "||", "lcr", file="src/lexer.cpp", offset=6),
+ entry("&&", "||", "lcr", state=MutantState.KILLED, offset=7),
+ ]
+ patterns = find_patterns(combined)
+ self.assertEqual(
+ [(pattern.family, pattern.file, len(pattern.mutant_ids)) for pattern in patterns],
+ [(AND_TO_OR, "src/parser.cpp", 3), (BOUNDARY, "src/parser.cpp", 2)],
+ )
+ self.assertEqual(
+ patterns[0].summary(),
+ "3 survivors turn `&&` into `||` in parser.cpp: add inputs that must be rejected, "
+ "since the tests check only what is accepted",
+ )
+
+ def test_each_survivor_is_in_one_primary_pattern_at_most(self) -> None:
+ combined = [
+ entry(original, replacement, offset=index)
+ for index, (original, replacement) in enumerate(
+ [("<", "<="), ("<", ">="), ("<=", "<"), ("<=", ">"), ("==", "!="), ("!=", "==")]
+ )
+ ]
+ ids = [mutant_id for pattern in find_patterns(combined) for mutant_id in pattern.mutant_ids]
+ self.assertEqual(sorted(ids), sorted(item.mutant.id for item in combined))
+ self.assertEqual(len(ids), len(set(ids)))
+
+ def test_hotspots_split_new_inputs_from_stronger_assertions(self) -> None:
+ combined = [
+ entry("+", "-", "aor", function="scale(int)", survivor=SurvivorKind.NOT_REACHED, offset=1),
+ entry("*", "/", "aor", function="scale(int)", survivor=SurvivorKind.NOT_REACHED, offset=2),
+ entry("<", "<=", function="scale(int)", offset=3),
+ entry("<", "<=", function="clamp(int)", offset=4),
+ ]
+ [hotspot] = find_hotspots(combined)
+ self.assertEqual(hotspot.summary(), "scale() in parser.cpp has 3 survivors: 2 not_reached, 1 not_detected")
+
+ def test_report_json_marks_hotspots_as_not_primary(self) -> None:
+ combined = [entry("&&", "||", "lcr", offset=1), entry("&&", "||", "lcr", offset=2)]
+ pattern, hotspot = patterns_json(combined)
+ self.assertEqual(
+ {key: pattern[key] for key in ("kind", "primary", "file", "survivors")},
+ {"kind": "and-to-or", "primary": True, "file": "src/parser.cpp", "survivors": 2},
+ )
+ self.assertEqual(pattern["mutants"], [item.mutant.id for item in combined])
+ self.assertEqual((hotspot["kind"], hotspot["primary"], hotspot["function"]), ("hotspot", False, "accept(char)"))
+
+ def test_a_single_survivor_is_not_a_pattern(self) -> None:
+ self.assertEqual(pattern_lines([entry("&&", "||", "lcr", function=None)]), [])
+
+
+if __name__ == "__main__":
+ unittest.main()