From cd6ad458478aca2a4c0bfe65f3dd778c0173747a Mon Sep 17 00:00:00 2001 From: Jason Turner Date: Wed, 18 Nov 2026 03:38:25 +0100 Subject: [PATCH 1/2] Run each mutant's covering tests fastest first The per-mutant strategy stops at the first test that detects a mutant, but it ran the covering tests in the order they were selected, so a slow test could run before a fast one that kills the mutant on its own. It now sorts them by baseline duration, then by name. BaselinedTest and CoveringTest carry the baseline duration instead of the timeout derived from it; mutant_timeout() derives the timeout. Resuming looks up every test's recorded outcome rather than stopping at the first test without one. Each run measures its own baselines, so two tests with close durations can swap places between the interrupted run and the resumed one, and the old lookup would then run tests again. Fixes #41. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- strain/runner.py | 54 ++++++++++++++++++++++++++-------------- tests/test_end_to_end.py | 28 +++++++++++++++++++-- tests/test_runner.py | 45 ++++++++++++++++++++++++++++++++- 4 files changed, 107 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 545cdef..fb15b78 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ strain run --safety-nets=warn | `--build-dir` | `.` | CMake build folder | | `-R` / `-E` | | Include / exclude tests by regex, as in ctest | | `--since ` | | Mutate only lines changed since a git ref | -| `--strategy` | `per-test` | `per-test` or `per-mutant` | +| `--strategy` | `per-test` | `per-test`, or `per-mutant`: build each mutant once and run its covering tests, fastest first, until one detects it | | `--skip-killed` | off | Skip mutants an earlier test already killed | | `--no-build-cache` | off | Don't reuse or save the [build cache](#build-cache) | | `--engine` | `auto` | `cppcheck`, `tree-sitter`, `lexer` | diff --git a/strain/runner.py b/strain/runner.py index c4f7afd..4bec8b7 100644 --- a/strain/runner.py +++ b/strain/runner.py @@ -150,19 +150,28 @@ class PlannedMutant: mutant: Mutant +def mutant_timeout(baseline_seconds: float) -> float: + """How long a test may run with a mutant in place, from how long it took without one.""" + return max(MINIMUM_TIMEOUT_SECONDS, baseline_seconds * TIMEOUT_FACTOR) + + @dataclass(frozen=True, slots=True) class BaselinedTest: name: str - timeout: float + duration: float sources: list[SourceFile] @dataclass(frozen=True, slots=True) class CoveringTest: name: str - timeout: float + duration: float source: SourceFile + @property + def timeout(self) -> float: + return mutant_timeout(self.duration) + @dataclass(slots=True) class CoveredMutant: @@ -459,15 +468,16 @@ def evaluate(self, test: str, planned: PlannedMutant, timeout: float, label: str return self.run_mutated(test, timeout, planned.source.coverage, planned.mutant.line, label) def run_baseline(self, test: str, events: EventLog) -> float | None: + """How long `test` takes without a mutant, or None when it is excluded.""" self.coverage.reset() self.console.status(f" running {test} without mutation…") baseline = ctest.run_test(self.build_dir, test, BASELINE_TIMEOUT_SECONDS) problem = baseline_problem(baseline) events.emit("baseline", test=test, duration=baseline.duration, problem=problem) if problem is None: - timeout = max(MINIMUM_TIMEOUT_SECONDS, baseline.duration * TIMEOUT_FACTOR) + timeout = mutant_timeout(baseline.duration) self.console.step(f"baseline passed in {baseline.duration:.2f}s; mutants time out after {timeout:.0f}s") - return timeout + return baseline.duration if self.options.strict: raise StrainError(f"test {test} {problem}; excluded tests are fatal with --strict") self.console.warn(f"test {test} {problem}; excluding it") @@ -499,9 +509,10 @@ def ignored_outcome(self, planned: PlannedMutant) -> Outcome | None: return None def run_test(self, test: str, events: EventLog) -> list[MutantResult]: - timeout = self.run_baseline(test, events) - if timeout is None: + duration = self.run_baseline(test, events) + if duration is None: return [] + timeout = mutant_timeout(duration) coverage = self.coverage.collect() self.verify_coverage_data(test, coverage) planned = self.plan_mutants(self.source_files(coverage, self.test_code(test))) @@ -662,23 +673,26 @@ def baseline_tests(self, tests: Sequence[str], events: EventLog) -> list[Baselin for number, test in enumerate(tests, start=1): self.console.heading(f"Baseline {number}/{len(tests)}: {test}") self.console.test_started(test, number, len(tests)) - timeout = self.run_baseline(test, events) - if timeout is None: + duration = self.run_baseline(test, events) + if duration is None: continue coverage = self.coverage.collect() self.verify_coverage_data(test, coverage) - baselined.append(BaselinedTest(test, timeout, self.source_files(coverage, self.test_code(test)))) + baselined.append(BaselinedTest(test, duration, self.source_files(coverage, self.test_code(test)))) return baselined def cover_mutants(self, tests: Sequence[BaselinedTest]) -> list[CoveredMutant]: - """Every mutant on a line some test executes, with the tests that execute it in the order they were selected.""" + """Every mutant on a line some test executes, with the tests that execute it, fastest first. + + Running stops at the first test that detects the mutant, so a fast test that kills it spares the slow ones. + """ covered: dict[str, CoveredMutant] = {} - for test in tests: + for test in sorted(tests, key=lambda test: (test.duration, test.name)): for source in test.sources: suppressed = frozenset(suppressed_lines(source.text, self.config.arid)) for mutant in self.mutants_for(source): entry = covered.setdefault(mutant.id, CoveredMutant(PlannedMutant(source, suppressed, mutant))) - entry.tests.append(CoveringTest(test.name, test.timeout, source)) + entry.tests.append(CoveringTest(test.name, test.duration, source)) return sorted(covered.values(), key=lambda entry: (entry.planned.mutant.file, entry.planned.mutant.offset)) def plan_covered_mutants(self, covered: Sequence[CoveredMutant]) -> None: @@ -694,13 +708,15 @@ def plan_covered_mutants(self, covered: Sequence[CoveredMutant]) -> None: self.console.plan([entry.planned.mutant for entry in covered]) def previous_outcomes(self, entry: CoveredMutant, events: EventLog) -> list[tuple[CoveringTest, Outcome]]: - """The outcomes an interrupted run recorded for `entry`, up to the first test it did not get to.""" + """The outcomes an interrupted run recorded for `entry`. + + Every test is looked up: the order is by baseline duration, and close durations can swap between runs. + """ outcomes: list[tuple[CoveringTest, Outcome]] = [] for test in entry.tests: previous = self.previous_outcome(test.name, entry.planned.mutant, events) - if previous is None: - break - outcomes.append((test, previous)) + if previous is not None: + outcomes.append((test, previous)) return outcomes def run_covering_tests( @@ -730,9 +746,11 @@ def covered_outcomes( if ignored is not None: return [(test, ignored) for test in entry.tests] outcomes = self.previous_outcomes(entry, events) - if any(outcome.state in DETECTED_STATES for _, outcome in outcomes) or len(outcomes) == len(entry.tests): + done = {test.name for test, _ in outcomes} + remaining = [test for test in entry.tests if test.name not in done] + if any(outcome.state in DETECTED_STATES for _, outcome in outcomes) or not remaining: return outcomes - return outcomes + self.run_covering_tests(entry, entry.tests[len(outcomes) :], label) + return outcomes + self.run_covering_tests(entry, remaining, label) def evaluate_covered(self, entry: CoveredMutant, events: EventLog, progress: str) -> list[MutantResult]: mutant = entry.planned.mutant diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index e668dd4..c641b2f 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -97,6 +97,9 @@ def mutant_evaluated(self, result: MutantResult, source: str, line: LineCoverage ] +OTHER_TESTS_CHECKING_ADD = '#include "sample.hpp"\n\nint main() { return add(1, 1) == 2 ? 0 : 1; }\n' + + def run_strain(*argv: str) -> tuple[int, str, str]: stdout = io.StringIO() stderr = io.StringIO() @@ -295,6 +298,7 @@ def run_per_mutant(self) -> tuple[int, str, str]: ) def test_per_mutant_strategy_builds_each_mutant_once_and_stops_at_the_first_kill(self) -> None: + (self.root / "tests" / "other_tests.cpp").write_text(OTHER_TESTS_CHECKING_ADD) configure_project(self.root, self.build_dir) exit_code, stdout, _ = self.run_per_mutant() @@ -305,10 +309,30 @@ def test_per_mutant_strategy_builds_each_mutant_once_and_stops_at_the_first_kill self.assertIn("[1/8] killed", stdout) self.assertIn("sample_tests score 40.0% killed 1 sanitizer 1 timeout 0 survived 3", stdout) self.assertIn("✓ sources restored and rebuilt", stdout) + # Both tests kill the mutant of add; whichever runs first is the only one to run. add = [event["test"] for event in self.mutant_events() if event["line"] == 3] - self.assertEqual(add, ["sample_tests"]) + self.assertEqual(len(add), 1, add) self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source) + def test_per_mutant_strategy_runs_the_fastest_covering_test_first(self) -> None: + (self.root / "tests" / "other_tests.cpp").write_text(OTHER_TESTS_CHECKING_ADD) + sample_tests = self.root / "tests" / "sample_tests.cpp" + sample_tests.write_text( + "#include \n#include \n" + + sample_tests.read_text().replace( + "int main()\n{\n", "int main()\n{\n std::this_thread::sleep_for(std::chrono::seconds(1));\n", 1 + ) + ) + configure_project(self.root, self.build_dir) + + exit_code, stdout, _ = self.run_per_mutant() + + self.assertEqual(exit_code, 0, stdout) + # sample_tests is selected first, but other_tests is faster and kills the mutant of add on its own. + add = [event["test"] for event in self.mutant_events() if event["line"] == 3] + self.assertEqual(add, ["other_tests"]) + self.assertEqual(self.mutant_states(), SINGLE_TEST_STATES) + def test_per_mutant_strategy_runs_every_covering_test_of_a_survivor(self) -> None: (self.root / "tests" / "sample_tests.cpp").write_text( '#include "sample.hpp"\n\nint main() { static_cast(add(1, 1)); return 0; }\n' @@ -319,7 +343,7 @@ def test_per_mutant_strategy_runs_every_covering_test_of_a_survivor(self) -> Non self.assertEqual(exit_code, 0, stdout) self.assertEqual(self.mutant_states(), [(3, "aor", "-", "survived")]) - self.assertEqual([event["test"] for event in self.mutant_events()], ["sample_tests", "other_tests"]) + self.assertEqual(sorted(event["test"] for event in self.mutant_events()), ["other_tests", "sample_tests"]) def test_a_second_run_at_the_same_commit_does_not_rebuild_mutants_known_to_break_the_build(self) -> None: configure_project(self.root, self.build_dir) diff --git a/tests/test_runner.py b/tests/test_runner.py index 252fb1f..0bf3fc9 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,8 +1,13 @@ import os import unittest from pathlib import Path +from unittest import mock -from strain.runner import short_path +from strain.mutators import Mutant +from strain.results import MutantState, SurvivorKind +from strain.runner import CoveredMutant, CoveringTest, Outcome, PlannedMutant, Runner, short_path + +MUTANT = Mutant("src/a.cpp", 3, 11, 40, "ror", "<", "<=") class ShortPathTest(unittest.TestCase): @@ -13,5 +18,43 @@ def test_a_path_far_from_the_working_directory_stays_absolute(self) -> None: self.assertEqual(short_path(Path(Path.cwd().anchor)), str(Path(Path.cwd().anchor))) +class ResumeTest(unittest.TestCase): + """Resuming a per-mutant run reuses what the interrupted run recorded and runs only the rest.""" + + def setUp(self) -> None: + self.recorded: dict[str, Outcome] = {} + self.runner = object.__new__(Runner) + self.runner.ignored_outcome = lambda planned: None # type: ignore[method-assign] + self.runner.previous_outcome = lambda test, mutant, events: self.recorded.get(test) # type: ignore[method-assign] + self.runner.run_covering_tests = mock.Mock(side_effect=self.run_tests) # type: ignore[method-assign] + source = mock.sentinel.source + self.entry = CoveredMutant( + PlannedMutant(source, frozenset(), MUTANT), # type: ignore[arg-type] + [CoveringTest(name, 0.1, source) for name in ("fast", "middle", "slow")], # type: ignore[arg-type] + ) + + @staticmethod + def run_tests(entry: CoveredMutant, tests: list[CoveringTest], label: str) -> list[tuple[CoveringTest, Outcome]]: + return [(test, Outcome(MutantState.SURVIVED, SurvivorKind.NOT_DETECTED)) for test in tests] + + def outcomes(self) -> list[tuple[str, MutantState]]: + return [(test.name, outcome.state) for test, outcome in self.runner.covered_outcomes(self.entry, None, "")] # type: ignore[arg-type] + + def test_tests_recorded_out_of_order_are_not_run_again(self) -> None: + # The baselines of the interrupted run ordered the tests differently. + self.recorded["slow"] = Outcome(MutantState.SURVIVED, SurvivorKind.NOT_DETECTED) + self.assertEqual( + self.outcomes(), + [("slow", MutantState.SURVIVED), ("fast", MutantState.SURVIVED), ("middle", MutantState.SURVIVED)], + ) + [call] = self.runner.run_covering_tests.call_args_list # type: ignore[attr-defined] + self.assertEqual([test.name for test in call.args[1]], ["fast", "middle"]) + + def test_a_recorded_kill_runs_nothing(self) -> None: + self.recorded["middle"] = Outcome(MutantState.KILLED) + self.assertEqual(self.outcomes(), [("middle", MutantState.KILLED)]) + self.runner.run_covering_tests.assert_not_called() # type: ignore[attr-defined] + + if __name__ == "__main__": unittest.main() From e70bbdb394b556eb42d045bcf18c77d94d9215b6 Mon Sep 17 00:00:00 2001 From: Jason Turner Date: Wed, 18 Nov 2026 04:08:04 +0100 Subject: [PATCH 2/2] Don't assume which of two killing tests runs first Both tests in the stop-at-the-first-kill test kill the mutant of add, and with the fastest-first order either can run first, so the per-test summary lines depend on timing. Check the combined file score instead. Co-Authored-By: Claude Opus 5.5 --- tests/test_end_to_end.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index c641b2f..a488988 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -307,7 +307,8 @@ def test_per_mutant_strategy_builds_each_mutant_once_and_stops_at_the_first_kill self.assertEqual(self.mutant_states(), SINGLE_TEST_STATES) self.assertIn("· src/sample.cpp: 8 mutants on 6 lines", stdout) self.assertIn("[1/8] killed", stdout) - self.assertIn("sample_tests score 40.0% killed 1 sanitizer 1 timeout 0 survived 3", stdout) + # Either test may be the one that kills add, so the per-test lines vary; the combined score doesn't. + self.assertIn("src/sample.cpp 40.0% 8 mutants 2 detected 3 survived", stdout) self.assertIn("✓ sources restored and rebuilt", stdout) # Both tests kill the mutant of add; whichever runs first is the only one to run. add = [event["test"] for event in self.mutant_events() if event["line"] == 3]