Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,17 @@ Add `--ui live` (with `strain[ui]` installed) to watch the run in a full-screen

<!-- TODO(claude): paste a real console summary once one exists -->

With many survivors, start with the **patterns**. They group survivors that make the same kind of change in the same
file, such as every `&&` → `||` or every comparison moved by one at its limit, and each group names the test that would
kill all of them. Each survivor is in one group at most. **Hotspots** are the functions with the most survivors.

```text
parser_tests score 87.8% killed 31 sanitizer 4 timeout 1 survived 5

Patterns
· 3 survivors turn `&&` into `||` in lexer.cpp: add inputs that must be rejected, since the tests check only what is accepted
· hotspot: parse_header() in parser.cpp has 3 survivors: 1 not_reached, 2 not_detected

Action items
1. parse_header() is pseudo-tested: nothing checks its result
2. lexer.cpp:88 `==` → `!=` survived (not_reached): add a test where tok != END
Expand Down Expand Up @@ -220,7 +228,8 @@ To suppress mutants in source, add `// mutate-off` to the line.

Written to `strain-report/` <!-- TODO(claude): confirm path -->:

- `report.json`: full results, in the mutation-testing-elements schema
- `report.json`: full results, in the mutation-testing-elements schema, plus a `patterns` array: each group of
survivors with its kind, file, advice and mutant IDs, and the hotspots with `"primary": false`
- `events.jsonl`: every event of the run
- compiler-style warnings on the console (`file:line:col: warning: survived [ror]`), clickable in editors

Expand Down
15 changes: 15 additions & 0 deletions strain/html_report.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
"""`index.html`: one self-contained page with the score, a table of files and the source annotated with its mutants."""

import re
from collections.abc import Iterable, Mapping, Sequence
from html import escape

from .console import HIGH_SCORE, LOW_SCORE
from .patterns import pattern_lines
from .report import TAB_WIDTH, FileScore, file_scores, format_score, hint
from .results import DETECTED_STATES, CombinedResult, MutantState, mutation_score

Expand Down Expand Up @@ -126,6 +128,18 @@ def _survivor_list(combined: Sequence[CombinedResult]) -> list[str]:
return [f"<h2>Survivors ({len(survivors)})</h2>", "<ul>", *links, "</ul>"]


def _inline_code(text: str) -> str:
"""`text`, escaped, with its Markdown code spans as <code>."""
return re.sub(r"`([^`]+)`", r"<code>\1</code>", escape(text))


def _pattern_list(combined: Sequence[CombinedResult]) -> list[str]:
lines = pattern_lines(combined)
if not lines:
return []
return ["<h2>Patterns</h2>", "<ul>", *(f"<li>{_inline_code(line)}</li>" for line in lines), "</ul>"]


def _file_sections(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> list[str]:
sections: list[str] = []
for row in file_scores(combined):
Expand All @@ -151,6 +165,7 @@ def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str])
[
f"<p>{len(combined)} mutants: {_counts(combined)}</p>",
*_files_table(combined),
*_pattern_list(combined),
*_survivor_list(combined),
*_file_sections(combined, sources),
]
Expand Down
177 changes: 177 additions & 0 deletions strain/patterns.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
"""Survivors grouped into the few test-writing actions that would kill most of them.

One survivor at a time says what to test; a group says what kind of test is missing. Twelve `&&` → `||` survivors in
one file mean the tests check only what is accepted, never the lookalikes that must be rejected.

Each survivor belongs to one primary pattern at most: the kind of change it makes, in its file. Hotspots, the
functions with the most survivors, cut across those, so they are listed apart and are not primary.
"""

from collections.abc import Sequence
from dataclasses import dataclass
from pathlib import PurePosixPath
from typing import Any

from .results import CombinedResult, MutantState, SurvivorKind

# A pattern needs at least this many survivors; a single one is just a survivor.
MINIMUM_SURVIVORS = 2
HOTSPOTS = 5


@dataclass(frozen=True, slots=True)
class Family:
"""A kind of change, with what the tests are missing when mutants of that kind survive."""

kind: str
change: str
advice: str


BOUNDARY = Family("boundary", "move a comparison's limit by one", "add a test exactly at the limit")
NEGATION = Family("negated-comparison", "invert a comparison", "add a test on each side of the comparison")
EQUALITY = Family("equality", "swap `==` and `!=`", "add a test where the values are equal and one where they differ")
AND_TO_OR = Family(
"and-to-or",
"turn `&&` into `||`",
"add inputs that must be rejected, since the tests check only what is accepted",
)
OR_TO_AND = Family("or-to-and", "turn `||` into `&&`", "add an input that satisfies only one of the alternatives")
ARITHMETIC = Family(
"arithmetic", "change an arithmetic operator", "assert the computed values, not just that there are results"
)
FAMILIES = (BOUNDARY, NEGATION, EQUALITY, AND_TO_OR, OR_TO_AND, ARITHMETIC)

BOUNDARY_CHANGES = frozenset({("<", "<="), ("<=", "<"), (">", ">="), (">=", ">")})
NEGATION_CHANGES = frozenset({("<", ">="), (">=", "<"), (">", "<="), ("<=", ">")})


def family_of(entry: CombinedResult) -> Family | None:
change = (entry.mutant.original, entry.mutant.replacement)
if change in BOUNDARY_CHANGES:
return BOUNDARY
if change in NEGATION_CHANGES:
return NEGATION
if change in {("==", "!="), ("!=", "==")}:
return EQUALITY
if change == ("&&", "||"):
return AND_TO_OR
if change == ("||", "&&"):
return OR_TO_AND
if entry.mutant.operator in ("aor", "asr"):
return ARITHMETIC
return None


def file_name(file: str) -> str:
return PurePosixPath(file).name


@dataclass(frozen=True, slots=True)
class Pattern:
family: Family
file: str
mutant_ids: tuple[str, ...]

def summary(self) -> str:
"""`12 survivors turn `&&` into `||` in parser.cpp: add inputs that must be rejected…`"""
return f"{len(self.mutant_ids)} survivors {self.family.change} in {file_name(self.file)}: {self.family.advice}"


@dataclass(frozen=True, slots=True)
class Hotspot:
function: str
file: str
mutant_ids: tuple[str, ...]
not_reached: int
not_detected: int

def summary(self) -> str:
"""`clamp() in math.cpp has 3 survivors: 2 not_reached, 1 not_detected`"""
split = [
f"{count} {kind}"
for kind, count in (("not_reached", self.not_reached), ("not_detected", self.not_detected))
if count
]
name = f"{self.function.partition('(')[0]}()"
detail = f": {', '.join(split)}" if split else ""
return f"{name} in {file_name(self.file)} has {len(self.mutant_ids)} survivors{detail}"


def survivors(combined: Sequence[CombinedResult]) -> list[CombinedResult]:
return [entry for entry in combined if entry.state is MutantState.SURVIVED]


def find_patterns(combined: Sequence[CombinedResult]) -> list[Pattern]:
"""The primary patterns, largest first: survivors that make the same kind of change in the same file."""
groups: dict[tuple[Family, str], list[str]] = {}
for entry in survivors(combined):
family = family_of(entry)
if family is not None:
groups.setdefault((family, entry.mutant.file), []).append(entry.mutant.id)
patterns = [
Pattern(family, file, tuple(ids)) for (family, file), ids in groups.items() if len(ids) >= MINIMUM_SURVIVORS
]
return sorted(
patterns, key=lambda pattern: (-len(pattern.mutant_ids), pattern.file, FAMILIES.index(pattern.family))
)


def find_hotspots(combined: Sequence[CombinedResult], limit: int = HOTSPOTS) -> list[Hotspot]:
"""The functions with the most survivors, and how many need a new input or a stronger assertion."""
groups: dict[tuple[str, str], list[CombinedResult]] = {}
for entry in survivors(combined):
if entry.function is not None:
groups.setdefault((entry.function, entry.mutant.file), []).append(entry)
hotspots = [
Hotspot(
function,
file,
tuple(entry.mutant.id for entry in entries),
sum(entry.survivor is SurvivorKind.NOT_REACHED for entry in entries),
sum(entry.survivor is SurvivorKind.NOT_DETECTED for entry in entries),
)
for (function, file), entries in groups.items()
if len(entries) >= MINIMUM_SURVIVORS
]
return sorted(hotspots, key=lambda hotspot: (-len(hotspot.mutant_ids), hotspot.file, hotspot.function))[:limit]


def patterns_json(combined: Sequence[CombinedResult]) -> list[dict[str, Any]]:
"""The patterns for report.json; `primary` tells the patterns that share no survivor from the hotspots."""
return [
*(
{
"kind": pattern.family.kind,
"primary": True,
"file": pattern.file,
"survivors": len(pattern.mutant_ids),
"summary": pattern.summary(),
"advice": pattern.family.advice,
"mutants": list(pattern.mutant_ids),
}
for pattern in find_patterns(combined)
),
*(
{
"kind": "hotspot",
"primary": False,
"file": hotspot.file,
"function": hotspot.function,
"survivors": len(hotspot.mutant_ids),
"notReached": hotspot.not_reached,
"notDetected": hotspot.not_detected,
"summary": hotspot.summary(),
"mutants": list(hotspot.mutant_ids),
}
for hotspot in find_hotspots(combined)
),
]


def pattern_lines(combined: Sequence[CombinedResult]) -> list[str]:
"""One line per pattern, then per hotspot, for the console, summary.md and the HTML report."""
return [
*(pattern.summary() for pattern in find_patterns(combined)),
*(f"hotspot: {hotspot.summary()}" for hotspot in find_hotspots(combined)),
]
19 changes: 17 additions & 2 deletions strain/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from typing import Any, Self, TextIO

from .mutators import Mutant
from .patterns import pattern_lines, patterns_json
from .results import (
DETECTED_STATES,
CombinedResult,
Expand Down Expand Up @@ -183,6 +184,13 @@ def _markdown_file_table(combined: Sequence[CombinedResult]) -> list[str]:
]


def _markdown_patterns(combined: Sequence[CombinedResult]) -> list[str]:
lines = pattern_lines(combined)
if not lines:
return []
return ["", "### Patterns", "", *(f"- {line}" for line in lines)]


def _markdown_survivors(combined: Sequence[CombinedResult]) -> list[str]:
survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED]
if not survivors:
Expand Down Expand Up @@ -212,6 +220,7 @@ def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None
_markdown_counts(combined),
"",
*_markdown_file_table(combined),
*_markdown_patterns(combined),
*_markdown_survivors(combined),
]
return "\n".join(lines) + "\n"
Expand All @@ -234,14 +243,20 @@ def _mutant_element(entry: CombinedResult) -> dict[str, Any]:
}


def mutation_testing_elements(combined: Iterable[CombinedResult], sources: Mapping[str, str]) -> dict[str, Any]:
def mutation_testing_elements(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> dict[str, Any]:
"""The report in the mutation-testing-elements schema, plus strain's `patterns` for tools and agents."""
files: dict[str, Any] = {}
for entry in combined:
file_report = files.setdefault(
entry.mutant.file, {"language": "cpp", "source": sources[entry.mutant.file], "mutants": []}
)
file_report["mutants"].append(_mutant_element(entry))
return {"schemaVersion": "1", "thresholds": {"high": 80, "low": 60}, "files": files}
return {
"schemaVersion": "1",
"thresholds": {"high": 80, "low": 60},
"files": files,
"patterns": patterns_json(combined),
}


PreviousResults = dict[tuple[str, str], tuple[MutantState, SurvivorKind | None]]
Expand Down
10 changes: 10 additions & 0 deletions strain/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
)
from .operator_sites import EngineError, OperatorSite
from .html_report import html_report
from .patterns import pattern_lines
from .report import (
EventLog,
FileScore,
Expand Down Expand Up @@ -614,6 +615,14 @@ def file_line(self, row: FileScore, width: int) -> str:
f" {console.paint(f'{row.survived} survived', survived_style)}"
)

def show_patterns(self, combined: Sequence[CombinedResult]) -> None:
"""The groups of survivors that one kind of test would kill, ahead of the survivors one by one."""
lines = pattern_lines(combined)
if lines:
self.console.heading("Patterns")
for line in lines:
self.console.line(f" · {line}")

def show_action_items(self, combined: Sequence[CombinedResult]) -> None:
items = action_items(combined, limit=None if self.options.show_all else DEFAULT_ACTION_ITEMS)
if items:
Expand Down Expand Up @@ -791,6 +800,7 @@ def report(self, combined: Sequence[CombinedResult], events: EventLog, ending: s
self.write_reports(combined, sources)
events.emit(ending, score=score)
self.show_survivors(combined, sources)
self.show_patterns(combined)
self.show_action_items(combined)
self.show_files(combined)
self.show_summary(combined, score)
Expand Down
46 changes: 46 additions & 0 deletions tests/test_end_to_end.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,21 @@ def mutant_evaluated(self, result: MutantResult, source: str, line: LineCoverage
]


RECOGNISER_HEADER = "[[nodiscard]] bool is_identifier_start(char c);\n"
RECOGNISER_SOURCE = """#include "sample.hpp"

bool is_identifier_start(char c)
{
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_';
}
"""
# Weak on purpose: only inputs that must be accepted, none at a limit, none that must be rejected.
RECOGNISER_TEST = """#include "sample.hpp"

int main() { return is_identifier_start('m') && is_identifier_start('M') && is_identifier_start('_') ? 0 : 1; }
"""


def run_strain(*argv: str) -> tuple[int, str, str]:
stdout = io.StringIO()
stderr = io.StringIO()
Expand Down Expand Up @@ -503,6 +518,37 @@ def test_interrupted_run_still_writes_the_reports(self) -> None:
self.assertEqual(console.finished_runs, 0)
self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source)

def test_survivors_of_weak_tests_group_into_the_patterns_that_name_the_missing_tests(self) -> None:
(self.root / "include" / "sample.hpp").write_text(RECOGNISER_HEADER)
(self.root / "src" / "sample.cpp").write_text(RECOGNISER_SOURCE)
(self.root / "tests" / "sample_tests.cpp").write_text(RECOGNISER_TEST)
(self.root / "tests" / "other_tests.cpp").write_text("int main() { return 0; }\n")
configure_project(self.root, self.build_dir)

exit_code, stdout, _ = run_strain(
*("--build-dir", str(self.build_dir), "--report-dir", str(self.report_dir)),
*("-R", "^sample_tests$", "--engine", "lexer"),
)

self.assertEqual(exit_code, 0, stdout)
boundary = "4 survivors move a comparison's limit by one in sample.cpp: add a test exactly at the limit"
and_to_or = (
"2 survivors turn `&&` into `||` in sample.cpp: add inputs that must be rejected, "
"since the tests check only what is accepted"
)
hotspot = "hotspot: is_identifier_start() in sample.cpp has 6 survivors"
self.assertIn(f"▸ Patterns\n · {boundary}\n · {and_to_or}\n · {hotspot}", stdout)
summary = (self.report_dir / "summary.md").read_text()
self.assertIn(f"### Patterns\n\n- {boundary}\n- {and_to_or}\n- {hotspot}", summary)
report = json.loads((self.report_dir / "report.json").read_text())
primary = [pattern for pattern in report["patterns"] if pattern["primary"]]
self.assertEqual(
[(pattern["kind"], pattern["survivors"]) for pattern in primary], [("boundary", 4), ("and-to-or", 2)]
)
ids = [mutant for pattern in primary for mutant in pattern["mutants"]]
self.assertEqual(len(ids), len(set(ids)))
self.assertIn("<h2>Patterns</h2>", (self.report_dir / "index.html").read_text())

def test_dirty_working_tree_is_refused(self) -> None:
configure_project(self.root, self.build_dir)
(self.root / "src" / "sample.cpp").write_text("// local edit\n")
Expand Down
Loading
Loading