Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ strain run --safety-nets=warn
| `--build-dir` | `.` | CMake build folder |
| `-R` / `-E` | | Include / exclude tests by regex, as in ctest |
| `--since <ref>` | | Mutate only lines changed since a git ref |
| `--strategy` | `per-test` | `per-test` or `per-mutant` |
| `--strategy` | `per-test` | `per-test`, or `per-mutant`: build each mutant once and run its covering tests, fastest first, until one detects it |
| `--skip-killed` | off | Skip mutants an earlier test already killed |
| `--no-build-cache` | off | Don't reuse or save the [build cache](#build-cache) |
| `--engine` | `auto` | `cppcheck`, `tree-sitter`, `lexer` |
Expand Down
54 changes: 36 additions & 18 deletions strain/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,19 +150,28 @@ class PlannedMutant:
mutant: Mutant


def mutant_timeout(baseline_seconds: float) -> float:
"""How long a test may run with a mutant in place, from how long it took without one."""
return max(MINIMUM_TIMEOUT_SECONDS, baseline_seconds * TIMEOUT_FACTOR)


@dataclass(frozen=True, slots=True)
class BaselinedTest:
name: str
timeout: float
duration: float
sources: list[SourceFile]


@dataclass(frozen=True, slots=True)
class CoveringTest:
name: str
timeout: float
duration: float
source: SourceFile

@property
def timeout(self) -> float:
return mutant_timeout(self.duration)


@dataclass(slots=True)
class CoveredMutant:
Expand Down Expand Up @@ -459,15 +468,16 @@ def evaluate(self, test: str, planned: PlannedMutant, timeout: float, label: str
return self.run_mutated(test, timeout, planned.source.coverage, planned.mutant.line, label)

def run_baseline(self, test: str, events: EventLog) -> float | None:
"""How long `test` takes without a mutant, or None when it is excluded."""
self.coverage.reset()
self.console.status(f" running {test} without mutation…")
baseline = ctest.run_test(self.build_dir, test, BASELINE_TIMEOUT_SECONDS)
problem = baseline_problem(baseline)
events.emit("baseline", test=test, duration=baseline.duration, problem=problem)
if problem is None:
timeout = max(MINIMUM_TIMEOUT_SECONDS, baseline.duration * TIMEOUT_FACTOR)
timeout = mutant_timeout(baseline.duration)
self.console.step(f"baseline passed in {baseline.duration:.2f}s; mutants time out after {timeout:.0f}s")
return timeout
return baseline.duration
if self.options.strict:
raise StrainError(f"test {test} {problem}; excluded tests are fatal with --strict")
self.console.warn(f"test {test} {problem}; excluding it")
Expand Down Expand Up @@ -499,9 +509,10 @@ def ignored_outcome(self, planned: PlannedMutant) -> Outcome | None:
return None

def run_test(self, test: str, events: EventLog) -> list[MutantResult]:
timeout = self.run_baseline(test, events)
if timeout is None:
duration = self.run_baseline(test, events)
if duration is None:
return []
timeout = mutant_timeout(duration)
coverage = self.coverage.collect()
self.verify_coverage_data(test, coverage)
planned = self.plan_mutants(self.source_files(coverage, self.test_code(test)))
Expand Down Expand Up @@ -662,23 +673,26 @@ def baseline_tests(self, tests: Sequence[str], events: EventLog) -> list[Baselin
for number, test in enumerate(tests, start=1):
self.console.heading(f"Baseline {number}/{len(tests)}: {test}")
self.console.test_started(test, number, len(tests))
timeout = self.run_baseline(test, events)
if timeout is None:
duration = self.run_baseline(test, events)
if duration is None:
continue
coverage = self.coverage.collect()
self.verify_coverage_data(test, coverage)
baselined.append(BaselinedTest(test, timeout, self.source_files(coverage, self.test_code(test))))
baselined.append(BaselinedTest(test, duration, self.source_files(coverage, self.test_code(test))))
return baselined

def cover_mutants(self, tests: Sequence[BaselinedTest]) -> list[CoveredMutant]:
"""Every mutant on a line some test executes, with the tests that execute it in the order they were selected."""
"""Every mutant on a line some test executes, with the tests that execute it, fastest first.

Running stops at the first test that detects the mutant, so a fast test that kills it spares the slow ones.
"""
covered: dict[str, CoveredMutant] = {}
for test in tests:
for test in sorted(tests, key=lambda test: (test.duration, test.name)):
for source in test.sources:
suppressed = frozenset(suppressed_lines(source.text, self.config.arid))
for mutant in self.mutants_for(source):
entry = covered.setdefault(mutant.id, CoveredMutant(PlannedMutant(source, suppressed, mutant)))
entry.tests.append(CoveringTest(test.name, test.timeout, source))
entry.tests.append(CoveringTest(test.name, test.duration, source))
return sorted(covered.values(), key=lambda entry: (entry.planned.mutant.file, entry.planned.mutant.offset))

def plan_covered_mutants(self, covered: Sequence[CoveredMutant]) -> None:
Expand All @@ -694,13 +708,15 @@ def plan_covered_mutants(self, covered: Sequence[CoveredMutant]) -> None:
self.console.plan([entry.planned.mutant for entry in covered])

def previous_outcomes(self, entry: CoveredMutant, events: EventLog) -> list[tuple[CoveringTest, Outcome]]:
"""The outcomes an interrupted run recorded for `entry`, up to the first test it did not get to."""
"""The outcomes an interrupted run recorded for `entry`.

Every test is looked up: the order is by baseline duration, and close durations can swap between runs.
"""
outcomes: list[tuple[CoveringTest, Outcome]] = []
for test in entry.tests:
previous = self.previous_outcome(test.name, entry.planned.mutant, events)
if previous is None:
break
outcomes.append((test, previous))
if previous is not None:
outcomes.append((test, previous))
return outcomes

def run_covering_tests(
Expand Down Expand Up @@ -730,9 +746,11 @@ def covered_outcomes(
if ignored is not None:
return [(test, ignored) for test in entry.tests]
outcomes = self.previous_outcomes(entry, events)
if any(outcome.state in DETECTED_STATES for _, outcome in outcomes) or len(outcomes) == len(entry.tests):
done = {test.name for test, _ in outcomes}
remaining = [test for test in entry.tests if test.name not in done]
if any(outcome.state in DETECTED_STATES for _, outcome in outcomes) or not remaining:
return outcomes
return outcomes + self.run_covering_tests(entry, entry.tests[len(outcomes) :], label)
return outcomes + self.run_covering_tests(entry, remaining, label)

def evaluate_covered(self, entry: CoveredMutant, events: EventLog, progress: str) -> list[MutantResult]:
mutant = entry.planned.mutant
Expand Down
31 changes: 28 additions & 3 deletions tests/test_end_to_end.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ def mutant_evaluated(self, result: MutantResult, source: str, line: LineCoverage
]


OTHER_TESTS_CHECKING_ADD = '#include "sample.hpp"\n\nint main() { return add(1, 1) == 2 ? 0 : 1; }\n'


def run_strain(*argv: str) -> tuple[int, str, str]:
stdout = io.StringIO()
stderr = io.StringIO()
Expand Down Expand Up @@ -295,6 +298,7 @@ def run_per_mutant(self) -> tuple[int, str, str]:
)

def test_per_mutant_strategy_builds_each_mutant_once_and_stops_at_the_first_kill(self) -> None:
(self.root / "tests" / "other_tests.cpp").write_text(OTHER_TESTS_CHECKING_ADD)
configure_project(self.root, self.build_dir)

exit_code, stdout, _ = self.run_per_mutant()
Expand All @@ -303,12 +307,33 @@ def test_per_mutant_strategy_builds_each_mutant_once_and_stops_at_the_first_kill
self.assertEqual(self.mutant_states(), SINGLE_TEST_STATES)
self.assertIn("· src/sample.cpp: 8 mutants on 6 lines", stdout)
self.assertIn("[1/8] killed", stdout)
self.assertIn("sample_tests score 40.0% killed 1 sanitizer 1 timeout 0 survived 3", stdout)
# Either test may be the one that kills add, so the per-test lines vary; the combined score doesn't.
self.assertIn("src/sample.cpp 40.0% 8 mutants 2 detected 3 survived", stdout)
self.assertIn("✓ sources restored and rebuilt", stdout)
# Both tests kill the mutant of add; whichever runs first is the only one to run.
add = [event["test"] for event in self.mutant_events() if event["line"] == 3]
self.assertEqual(add, ["sample_tests"])
self.assertEqual(len(add), 1, add)
self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source)

def test_per_mutant_strategy_runs_the_fastest_covering_test_first(self) -> None:
(self.root / "tests" / "other_tests.cpp").write_text(OTHER_TESTS_CHECKING_ADD)
sample_tests = self.root / "tests" / "sample_tests.cpp"
sample_tests.write_text(
"#include <chrono>\n#include <thread>\n"
+ sample_tests.read_text().replace(
"int main()\n{\n", "int main()\n{\n std::this_thread::sleep_for(std::chrono::seconds(1));\n", 1
)
)
configure_project(self.root, self.build_dir)

exit_code, stdout, _ = self.run_per_mutant()

self.assertEqual(exit_code, 0, stdout)
# sample_tests is selected first, but other_tests is faster and kills the mutant of add on its own.
add = [event["test"] for event in self.mutant_events() if event["line"] == 3]
self.assertEqual(add, ["other_tests"])
self.assertEqual(self.mutant_states(), SINGLE_TEST_STATES)

def test_per_mutant_strategy_runs_every_covering_test_of_a_survivor(self) -> None:
(self.root / "tests" / "sample_tests.cpp").write_text(
'#include "sample.hpp"\n\nint main() { static_cast<void>(add(1, 1)); return 0; }\n'
Expand All @@ -319,7 +344,7 @@ def test_per_mutant_strategy_runs_every_covering_test_of_a_survivor(self) -> Non

self.assertEqual(exit_code, 0, stdout)
self.assertEqual(self.mutant_states(), [(3, "aor", "-", "survived")])
self.assertEqual([event["test"] for event in self.mutant_events()], ["sample_tests", "other_tests"])
self.assertEqual(sorted(event["test"] for event in self.mutant_events()), ["other_tests", "sample_tests"])

def test_a_second_run_at_the_same_commit_does_not_rebuild_mutants_known_to_break_the_build(self) -> None:
configure_project(self.root, self.build_dir)
Expand Down
45 changes: 44 additions & 1 deletion tests/test_runner.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
import os
import unittest
from pathlib import Path
from unittest import mock

from strain.runner import short_path
from strain.mutators import Mutant
from strain.results import MutantState, SurvivorKind
from strain.runner import CoveredMutant, CoveringTest, Outcome, PlannedMutant, Runner, short_path

MUTANT = Mutant("src/a.cpp", 3, 11, 40, "ror", "<", "<=")


class ShortPathTest(unittest.TestCase):
Expand All @@ -13,5 +18,43 @@ def test_a_path_far_from_the_working_directory_stays_absolute(self) -> None:
self.assertEqual(short_path(Path(Path.cwd().anchor)), str(Path(Path.cwd().anchor)))


class ResumeTest(unittest.TestCase):
"""Resuming a per-mutant run reuses what the interrupted run recorded and runs only the rest."""

def setUp(self) -> None:
self.recorded: dict[str, Outcome] = {}
self.runner = object.__new__(Runner)
self.runner.ignored_outcome = lambda planned: None # type: ignore[method-assign]
self.runner.previous_outcome = lambda test, mutant, events: self.recorded.get(test) # type: ignore[method-assign]
self.runner.run_covering_tests = mock.Mock(side_effect=self.run_tests) # type: ignore[method-assign]
source = mock.sentinel.source
self.entry = CoveredMutant(
PlannedMutant(source, frozenset(), MUTANT), # type: ignore[arg-type]
[CoveringTest(name, 0.1, source) for name in ("fast", "middle", "slow")], # type: ignore[arg-type]
)

@staticmethod
def run_tests(entry: CoveredMutant, tests: list[CoveringTest], label: str) -> list[tuple[CoveringTest, Outcome]]:
return [(test, Outcome(MutantState.SURVIVED, SurvivorKind.NOT_DETECTED)) for test in tests]

def outcomes(self) -> list[tuple[str, MutantState]]:
return [(test.name, outcome.state) for test, outcome in self.runner.covered_outcomes(self.entry, None, "")] # type: ignore[arg-type]

def test_tests_recorded_out_of_order_are_not_run_again(self) -> None:
# The baselines of the interrupted run ordered the tests differently.
self.recorded["slow"] = Outcome(MutantState.SURVIVED, SurvivorKind.NOT_DETECTED)
self.assertEqual(
self.outcomes(),
[("slow", MutantState.SURVIVED), ("fast", MutantState.SURVIVED), ("middle", MutantState.SURVIVED)],
)
[call] = self.runner.run_covering_tests.call_args_list # type: ignore[attr-defined]
self.assertEqual([test.name for test in call.args[1]], ["fast", "middle"])

def test_a_recorded_kill_runs_nothing(self) -> None:
self.recorded["middle"] = Outcome(MutantState.KILLED)
self.assertEqual(self.outcomes(), [("middle", MutantState.KILLED)])
self.runner.run_covering_tests.assert_not_called() # type: ignore[attr-defined]


if __name__ == "__main__":
unittest.main()
Loading