diff --git a/README.md b/README.md index 545cdef..fb15b78 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ strain run --safety-nets=warn | `--build-dir` | `.` | CMake build folder | | `-R` / `-E` | | Include / exclude tests by regex, as in ctest | | `--since ` | | Mutate only lines changed since a git ref | -| `--strategy` | `per-test` | `per-test` or `per-mutant` | +| `--strategy` | `per-test` | `per-test`, or `per-mutant`: build each mutant once and run its covering tests, fastest first, until one detects it | | `--skip-killed` | off | Skip mutants an earlier test already killed | | `--no-build-cache` | off | Don't reuse or save the [build cache](#build-cache) | | `--engine` | `auto` | `cppcheck`, `tree-sitter`, `lexer` | diff --git a/strain/runner.py b/strain/runner.py index c4f7afd..4bec8b7 100644 --- a/strain/runner.py +++ b/strain/runner.py @@ -150,19 +150,28 @@ class PlannedMutant: mutant: Mutant +def mutant_timeout(baseline_seconds: float) -> float: + """How long a test may run with a mutant in place, from how long it took without one.""" + return max(MINIMUM_TIMEOUT_SECONDS, baseline_seconds * TIMEOUT_FACTOR) + + @dataclass(frozen=True, slots=True) class BaselinedTest: name: str - timeout: float + duration: float sources: list[SourceFile] @dataclass(frozen=True, slots=True) class CoveringTest: name: str - timeout: float + duration: float source: SourceFile + @property + def timeout(self) -> float: + return mutant_timeout(self.duration) + @dataclass(slots=True) class CoveredMutant: @@ -459,15 +468,16 @@ def evaluate(self, test: str, planned: PlannedMutant, timeout: float, label: str return self.run_mutated(test, timeout, planned.source.coverage, planned.mutant.line, label) def run_baseline(self, test: str, events: EventLog) -> float | None: + """How long `test` takes without a mutant, or None when it is excluded.""" self.coverage.reset() self.console.status(f" running {test} without mutation…") baseline = ctest.run_test(self.build_dir, test, BASELINE_TIMEOUT_SECONDS) problem = baseline_problem(baseline) events.emit("baseline", test=test, duration=baseline.duration, problem=problem) if problem is None: - timeout = max(MINIMUM_TIMEOUT_SECONDS, baseline.duration * TIMEOUT_FACTOR) + timeout = mutant_timeout(baseline.duration) self.console.step(f"baseline passed in {baseline.duration:.2f}s; mutants time out after {timeout:.0f}s") - return timeout + return baseline.duration if self.options.strict: raise StrainError(f"test {test} {problem}; excluded tests are fatal with --strict") self.console.warn(f"test {test} {problem}; excluding it") @@ -499,9 +509,10 @@ def ignored_outcome(self, planned: PlannedMutant) -> Outcome | None: return None def run_test(self, test: str, events: EventLog) -> list[MutantResult]: - timeout = self.run_baseline(test, events) - if timeout is None: + duration = self.run_baseline(test, events) + if duration is None: return [] + timeout = mutant_timeout(duration) coverage = self.coverage.collect() self.verify_coverage_data(test, coverage) planned = self.plan_mutants(self.source_files(coverage, self.test_code(test))) @@ -662,23 +673,26 @@ def baseline_tests(self, tests: Sequence[str], events: EventLog) -> list[Baselin for number, test in enumerate(tests, start=1): self.console.heading(f"Baseline {number}/{len(tests)}: {test}") self.console.test_started(test, number, len(tests)) - timeout = self.run_baseline(test, events) - if timeout is None: + duration = self.run_baseline(test, events) + if duration is None: continue coverage = self.coverage.collect() self.verify_coverage_data(test, coverage) - baselined.append(BaselinedTest(test, timeout, self.source_files(coverage, self.test_code(test)))) + baselined.append(BaselinedTest(test, duration, self.source_files(coverage, self.test_code(test)))) return baselined def cover_mutants(self, tests: Sequence[BaselinedTest]) -> list[CoveredMutant]: - """Every mutant on a line some test executes, with the tests that execute it in the order they were selected.""" + """Every mutant on a line some test executes, with the tests that execute it, fastest first. + + Running stops at the first test that detects the mutant, so a fast test that kills it spares the slow ones. + """ covered: dict[str, CoveredMutant] = {} - for test in tests: + for test in sorted(tests, key=lambda test: (test.duration, test.name)): for source in test.sources: suppressed = frozenset(suppressed_lines(source.text, self.config.arid)) for mutant in self.mutants_for(source): entry = covered.setdefault(mutant.id, CoveredMutant(PlannedMutant(source, suppressed, mutant))) - entry.tests.append(CoveringTest(test.name, test.timeout, source)) + entry.tests.append(CoveringTest(test.name, test.duration, source)) return sorted(covered.values(), key=lambda entry: (entry.planned.mutant.file, entry.planned.mutant.offset)) def plan_covered_mutants(self, covered: Sequence[CoveredMutant]) -> None: @@ -694,13 +708,15 @@ def plan_covered_mutants(self, covered: Sequence[CoveredMutant]) -> None: self.console.plan([entry.planned.mutant for entry in covered]) def previous_outcomes(self, entry: CoveredMutant, events: EventLog) -> list[tuple[CoveringTest, Outcome]]: - """The outcomes an interrupted run recorded for `entry`, up to the first test it did not get to.""" + """The outcomes an interrupted run recorded for `entry`. + + Every test is looked up: the order is by baseline duration, and close durations can swap between runs. + """ outcomes: list[tuple[CoveringTest, Outcome]] = [] for test in entry.tests: previous = self.previous_outcome(test.name, entry.planned.mutant, events) - if previous is None: - break - outcomes.append((test, previous)) + if previous is not None: + outcomes.append((test, previous)) return outcomes def run_covering_tests( @@ -730,9 +746,11 @@ def covered_outcomes( if ignored is not None: return [(test, ignored) for test in entry.tests] outcomes = self.previous_outcomes(entry, events) - if any(outcome.state in DETECTED_STATES for _, outcome in outcomes) or len(outcomes) == len(entry.tests): + done = {test.name for test, _ in outcomes} + remaining = [test for test in entry.tests if test.name not in done] + if any(outcome.state in DETECTED_STATES for _, outcome in outcomes) or not remaining: return outcomes - return outcomes + self.run_covering_tests(entry, entry.tests[len(outcomes) :], label) + return outcomes + self.run_covering_tests(entry, remaining, label) def evaluate_covered(self, entry: CoveredMutant, events: EventLog, progress: str) -> list[MutantResult]: mutant = entry.planned.mutant diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index e668dd4..a488988 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -97,6 +97,9 @@ def mutant_evaluated(self, result: MutantResult, source: str, line: LineCoverage ] +OTHER_TESTS_CHECKING_ADD = '#include "sample.hpp"\n\nint main() { return add(1, 1) == 2 ? 0 : 1; }\n' + + def run_strain(*argv: str) -> tuple[int, str, str]: stdout = io.StringIO() stderr = io.StringIO() @@ -295,6 +298,7 @@ def run_per_mutant(self) -> tuple[int, str, str]: ) def test_per_mutant_strategy_builds_each_mutant_once_and_stops_at_the_first_kill(self) -> None: + (self.root / "tests" / "other_tests.cpp").write_text(OTHER_TESTS_CHECKING_ADD) configure_project(self.root, self.build_dir) exit_code, stdout, _ = self.run_per_mutant() @@ -303,12 +307,33 @@ def test_per_mutant_strategy_builds_each_mutant_once_and_stops_at_the_first_kill self.assertEqual(self.mutant_states(), SINGLE_TEST_STATES) self.assertIn("· src/sample.cpp: 8 mutants on 6 lines", stdout) self.assertIn("[1/8] killed", stdout) - self.assertIn("sample_tests score 40.0% killed 1 sanitizer 1 timeout 0 survived 3", stdout) + # Either test may be the one that kills add, so the per-test lines vary; the combined score doesn't. + self.assertIn("src/sample.cpp 40.0% 8 mutants 2 detected 3 survived", stdout) self.assertIn("✓ sources restored and rebuilt", stdout) + # Both tests kill the mutant of add; whichever runs first is the only one to run. add = [event["test"] for event in self.mutant_events() if event["line"] == 3] - self.assertEqual(add, ["sample_tests"]) + self.assertEqual(len(add), 1, add) self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source) + def test_per_mutant_strategy_runs_the_fastest_covering_test_first(self) -> None: + (self.root / "tests" / "other_tests.cpp").write_text(OTHER_TESTS_CHECKING_ADD) + sample_tests = self.root / "tests" / "sample_tests.cpp" + sample_tests.write_text( + "#include \n#include \n" + + sample_tests.read_text().replace( + "int main()\n{\n", "int main()\n{\n std::this_thread::sleep_for(std::chrono::seconds(1));\n", 1 + ) + ) + configure_project(self.root, self.build_dir) + + exit_code, stdout, _ = self.run_per_mutant() + + self.assertEqual(exit_code, 0, stdout) + # sample_tests is selected first, but other_tests is faster and kills the mutant of add on its own. + add = [event["test"] for event in self.mutant_events() if event["line"] == 3] + self.assertEqual(add, ["other_tests"]) + self.assertEqual(self.mutant_states(), SINGLE_TEST_STATES) + def test_per_mutant_strategy_runs_every_covering_test_of_a_survivor(self) -> None: (self.root / "tests" / "sample_tests.cpp").write_text( '#include "sample.hpp"\n\nint main() { static_cast(add(1, 1)); return 0; }\n' @@ -319,7 +344,7 @@ def test_per_mutant_strategy_runs_every_covering_test_of_a_survivor(self) -> Non self.assertEqual(exit_code, 0, stdout) self.assertEqual(self.mutant_states(), [(3, "aor", "-", "survived")]) - self.assertEqual([event["test"] for event in self.mutant_events()], ["sample_tests", "other_tests"]) + self.assertEqual(sorted(event["test"] for event in self.mutant_events()), ["other_tests", "sample_tests"]) def test_a_second_run_at_the_same_commit_does_not_rebuild_mutants_known_to_break_the_build(self) -> None: configure_project(self.root, self.build_dir) diff --git a/tests/test_runner.py b/tests/test_runner.py index 252fb1f..0bf3fc9 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,8 +1,13 @@ import os import unittest from pathlib import Path +from unittest import mock -from strain.runner import short_path +from strain.mutators import Mutant +from strain.results import MutantState, SurvivorKind +from strain.runner import CoveredMutant, CoveringTest, Outcome, PlannedMutant, Runner, short_path + +MUTANT = Mutant("src/a.cpp", 3, 11, 40, "ror", "<", "<=") class ShortPathTest(unittest.TestCase): @@ -13,5 +18,43 @@ def test_a_path_far_from_the_working_directory_stays_absolute(self) -> None: self.assertEqual(short_path(Path(Path.cwd().anchor)), str(Path(Path.cwd().anchor))) +class ResumeTest(unittest.TestCase): + """Resuming a per-mutant run reuses what the interrupted run recorded and runs only the rest.""" + + def setUp(self) -> None: + self.recorded: dict[str, Outcome] = {} + self.runner = object.__new__(Runner) + self.runner.ignored_outcome = lambda planned: None # type: ignore[method-assign] + self.runner.previous_outcome = lambda test, mutant, events: self.recorded.get(test) # type: ignore[method-assign] + self.runner.run_covering_tests = mock.Mock(side_effect=self.run_tests) # type: ignore[method-assign] + source = mock.sentinel.source + self.entry = CoveredMutant( + PlannedMutant(source, frozenset(), MUTANT), # type: ignore[arg-type] + [CoveringTest(name, 0.1, source) for name in ("fast", "middle", "slow")], # type: ignore[arg-type] + ) + + @staticmethod + def run_tests(entry: CoveredMutant, tests: list[CoveringTest], label: str) -> list[tuple[CoveringTest, Outcome]]: + return [(test, Outcome(MutantState.SURVIVED, SurvivorKind.NOT_DETECTED)) for test in tests] + + def outcomes(self) -> list[tuple[str, MutantState]]: + return [(test.name, outcome.state) for test, outcome in self.runner.covered_outcomes(self.entry, None, "")] # type: ignore[arg-type] + + def test_tests_recorded_out_of_order_are_not_run_again(self) -> None: + # The baselines of the interrupted run ordered the tests differently. + self.recorded["slow"] = Outcome(MutantState.SURVIVED, SurvivorKind.NOT_DETECTED) + self.assertEqual( + self.outcomes(), + [("slow", MutantState.SURVIVED), ("fast", MutantState.SURVIVED), ("middle", MutantState.SURVIVED)], + ) + [call] = self.runner.run_covering_tests.call_args_list # type: ignore[attr-defined] + self.assertEqual([test.name for test in call.args[1]], ["fast", "middle"]) + + def test_a_recorded_kill_runs_nothing(self) -> None: + self.recorded["middle"] = Outcome(MutantState.KILLED) + self.assertEqual(self.outcomes(), [("middle", MutantState.KILLED)]) + self.runner.run_covering_tests.assert_not_called() # type: ignore[attr-defined] + + if __name__ == "__main__": unittest.main()