diff --git a/README.md b/README.md index 4d69921..b648b49 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,7 @@ A mutant that doesn't compile is a `compile_error` for every test, so strain bui | `killed` | A test failed | Yes | | `killed_by_sanitizer` | ASan or UBSan caught it, the assertions didn't | Yes, flagged | | `killed_by_resource_limit` | The test reached its [memory limit](#memory-limit) | Yes | +| `killed_at_compile_time` | A `static_assert` or a constant evaluation failed with it in place | Yes | | `timeout` | Ran too long | Yes | | `survived` | All tests passed | Yes | | `skipped_unevaluated` | Operand never evaluated by this test | No | @@ -168,7 +169,25 @@ A mutant that doesn't compile is a `compile_error` for every test, so strain bui Survivors are labeled `not_reached` (needs a new test input) or `not_detected` (needs a stronger assertion) when branch coverage is available. -Mutation score = (killed + killed_by_sanitizer + killed_by_resource_limit + timeout) ÷ (those + survived) +Mutation score = (killed + killed_by_sanitizer + killed_by_resource_limit + killed_at_compile_time + timeout) ÷ +(those + survived) + +### Compile-time code + +gcov counts what runs, so a `constexpr` or `consteval` function checked only by `static_assert` has no executed lines. +strain would never mutate it, and a header used only that way doesn't appear in the coverage data at all. strain finds +these functions in the sources instead: + +- **By default:** the lines of `constexpr` and `consteval` functions that hold mutants but that no test runs are listed + under **Not mutated: evaluated at compile time only**. That covers the console, `summary.md`, `index.html` and + `compileTimeOnly` in `report.json`. You can tell "well tested" from "never looked at". +- **With `--compile-time`:** after the tests, strain mutates those lines and builds everything with each mutant. + - A failed `static_assert` or constant evaluation kills it as `killed_at_compile_time`. Its detail names the failed + check. + - A mutant that builds survives, with a hint to add a `static_assert`. + - Invalid code stays `compile_error`. +- **Any mutant:** a build or `-fsyntax-only` check that fails on a compile-time check counts as `killed_at_compile_time`, + even on lines the tests run. The [build cache](#build-cache) remembers it like any other build failure. ### Memory limit @@ -222,6 +241,7 @@ strain run --safety-nets=warn | `--threshold` | | Exit non-zero below this score | | `--strict` | off | Abort if any test is excluded | | `--max-per-line` | | Cap mutants per source line | +| `--compile-time` | off | Also mutate `constexpr` code no test runs; see [Compile-time code](#compile-time-code) | | `--memory-limit` | 4× the baseline, at least 512 MB | Memory each test may use with a mutant in place, in MB; `0` for no limit. See [Memory limit](#memory-limit) | | `--resume` | | Continue an interrupted run | | `--all` | off | Show every survivor, not just the top actions | diff --git a/strain/build.py b/strain/build.py index 4ba4792..2c8937b 100644 --- a/strain/build.py +++ b/strain/build.py @@ -45,9 +45,16 @@ def has_gcov_instrumentation(self) -> bool: def has_source_based_coverage(self) -> bool: return any(argument in SOURCE_BASED_COVERAGE_FLAGS for argument in self.arguments) - def passes_syntax_check(self) -> bool: - completed = subprocess.run(self.syntax_only_arguments(), cwd=self.directory, capture_output=True) - return completed.returncode == 0 + def syntax_check(self) -> str | None: + """None when the file compiles with -fsyntax-only, or else the compiler's diagnostics. + + -fsyntax-only still evaluates `static_assert` and constant expressions, so its diagnostics tell a mutant that + fails a compile-time check from one that is invalid code. + """ + completed = subprocess.run( + self.syntax_only_arguments(), cwd=self.directory, capture_output=True, text=True, errors="replace" + ) + return None if completed.returncode == 0 else completed.stdout + completed.stderr def _command_from_entry(entry: Mapping[str, Any]) -> CompileCommand: diff --git a/strain/cli.py b/strain/cli.py index 326b79f..9850fcc 100644 --- a/strain/cli.py +++ b/strain/cli.py @@ -82,6 +82,11 @@ def _add_policy_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--threshold", type=float, metavar="PERCENT", help="exit non-zero below this score") parser.add_argument("--strict", action="store_true", help="abort if any test is excluded") parser.add_argument("--max-per-line", type=_positive_int, metavar="N", help="cap mutants per source line") + parser.add_argument( + "--compile-time", + action="store_true", + help="also mutate constexpr code no test runs; a failed static_assert or constant evaluation kills a mutant", + ) parser.add_argument( "--memory-limit", type=_megabytes, @@ -129,6 +134,7 @@ def options_from_arguments(arguments: argparse.Namespace) -> RunOptions: strict=arguments.strict, max_per_line=arguments.max_per_line, memory_limit=arguments.memory_limit, + compile_time=arguments.compile_time, resume=arguments.resume, show_all=arguments.show_all, ui=arguments.ui, diff --git a/strain/compile_time.py b/strain/compile_time.py new file mode 100644 index 0000000..9583da0 --- /dev/null +++ b/strain/compile_time.py @@ -0,0 +1,142 @@ +"""Code that only the compiler runs: constexpr and consteval functions evaluated in constant expressions. + +gcov counts what runs, so a constexpr function checked only by `static_assert` has no executed lines and is never +mutated, and a header used only that way doesn't appear in the coverage data at all. strain finds these functions in +the sources instead, says which of their lines no test runs, and with --compile-time mutates them: a mutant that +makes a `static_assert` or a constant evaluation fail is killed at compile time. +""" + +import re +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any + +from .lexer import Token, TokenKind, tokenize + +CONSTANT_KEYWORDS = frozenset({"constexpr", "consteval"}) +# Diagnostics of a compile-time check that failed, from GCC and Clang. Anything else that stops the build means the +# mutant is simply invalid code: a compile error. +COMPILE_TIME_CHECK_FAILED = re.compile( + r"static assertion failed|static_assert failed|non-constant condition for static assertion" + r"|is not a constant expression|not an integral constant expression|must be initialized by a constant expression" +) +KILLED_AT_COMPILE_TIME = "killed at compile time" +# The name results get for the mutants --compile-time builds: the compile-time checks are what tests them. +COMPILE_TIME_TEST = "compile-time checks" +NOT_MUTATED_HEADING = "Not mutated: evaluated at compile time only" +NOT_MUTATED_HINT = ( + "no test runs these constexpr lines; --compile-time mutates them, and a failed static_assert kills them" +) + + +@dataclass(frozen=True, slots=True) +class ConstantFunction: + name: str + keyword: str + # The lines with code between the function's braces. + body_lines: frozenset[int] + + +def compile_time_check_failure(diagnostics: str) -> str | None: + """The diagnostic line of the compile-time check that failed, or None when the build failed for another reason.""" + for line in diagnostics.splitlines(): + if COMPILE_TIME_CHECK_FAILED.search(line): + return line.strip() + return None + + +def _matching(tokens: list[Token], start: int, opening: str, closing: str) -> int | None: + depth = 0 + for index in range(start, len(tokens)): + if tokens[index].text == opening: + depth += 1 + elif tokens[index].text == closing: + depth -= 1 + if depth == 0: + return index + return None + + +def _body_start(tokens: list[Token], start: int) -> tuple[str, int] | None: + """After a constexpr or consteval keyword: the function's name and opening brace, or None for anything else. + + A variable reaches `=` or `;` first, or opens a brace before any parameter list; `if constexpr` is not a + declaration at all. + """ + name: str | None = None + index = start + while index < len(tokens): + token = tokens[index] + if token.text == ";" or (token.text == "=" and tokens[index - 1].text != "operator"): + return None + if token.text == "(": + if name is None: + previous = tokens[index - 1] + name = previous.text if previous.kind is TokenKind.IDENTIFIER else "operator" + closing = _matching(tokens, index, "(", ")") + if closing is None: + return None + index = closing + elif token.text == "{": + return None if name is None else (name, index) + index += 1 + return None + + +def constant_functions(source: str) -> list[ConstantFunction]: + tokens = [token for token in tokenize(source) if token.kind is not TokenKind.COMMENT] + functions: list[ConstantFunction] = [] + for index, token in enumerate(tokens): + if token.text not in CONSTANT_KEYWORDS or (index > 0 and tokens[index - 1].text == "if"): + continue + found = _body_start(tokens, index + 1) + if found is None: + continue + name, opening = found + closing = _matching(tokens, opening, "{", "}") + if closing is None: + continue + lines = frozenset(inner.line for inner in tokens[opening + 1 : closing]) + functions.append(ConstantFunction(name, token.text, lines)) + return functions + + +def constant_lines(source: str) -> dict[int, str]: + """Each line inside a constexpr or consteval function, with the innermost function's name.""" + lines: dict[int, str] = {} + # Outer functions first, so a lambda or local function inside one names its own lines. + for function in sorted(constant_functions(source), key=lambda function: -len(function.body_lines)): + lines.update(dict.fromkeys(function.body_lines, function.name)) + return lines + + +@dataclass(frozen=True, slots=True) +class NotMutated: + """Lines of a constexpr or consteval function that hold mutants but that no test runs, so none was tested.""" + + file: str + function: str + lines: tuple[int, ...] + mutants: int + + @property + def location(self) -> str: + first, last = self.lines[0], self.lines[-1] + return f"{self.file}:{first}" if first == last else f"{self.file}:{first}-{last}" + + def summary(self) -> str: + """`src/limits.hpp:6-7 clamp_index(): 3 mutants on 2 lines`""" + mutants = f"{self.mutants} mutant{'' if self.mutants == 1 else 's'}" + lines = f"{len(self.lines)} line{'' if len(self.lines) == 1 else 's'}" + return f"{self.location} {self.function}(): {mutants} on {lines}" + + +def not_mutated(found: "Sequence[tuple[Any, str]]") -> list[NotMutated]: + """Groups (planned mutant, function) pairs by file and function.""" + groups: dict[tuple[str, str], list[int]] = {} + for planned, function in found: + groups.setdefault((planned.mutant.file, function), []).append(planned.mutant.line) + return [ + NotMutated(file, function, tuple(sorted(set(lines))), len(lines)) + for (file, function), lines in sorted(groups.items(), key=lambda item: (item[0][0], min(item[1]))) + ] diff --git a/strain/console.py b/strain/console.py index b6d2246..131ec40 100644 --- a/strain/console.py +++ b/strain/console.py @@ -25,6 +25,7 @@ MutantState.KILLED: GREEN, MutantState.KILLED_BY_SANITIZER: GREEN, MutantState.KILLED_BY_RESOURCE_LIMIT: GREEN, + MutantState.KILLED_AT_COMPILE_TIME: GREEN, MutantState.TIMEOUT: GREEN, MutantState.SURVIVED: BOLD + RED, MutantState.SKIPPED_UNEVALUATED: DIM, diff --git a/strain/html_report.py b/strain/html_report.py index 892b445..ec80d21 100644 --- a/strain/html_report.py +++ b/strain/html_report.py @@ -4,6 +4,7 @@ from html import escape from .console import HIGH_SCORE, LOW_SCORE +from .compile_time import NOT_MUTATED_HEADING, NOT_MUTATED_HINT, NotMutated from .report import TAB_WIDTH, FileScore, file_scores, format_score, hint from .results import DETECTED_STATES, CombinedResult, MutantState, mutation_score @@ -145,7 +146,22 @@ def _counts(combined: Sequence[CombinedResult]) -> str: return ", ".join(f"{escape(state)} {states.count(state)}" for state in MutantState if state in states) -def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> str: +def _not_mutated_list(not_mutated: Sequence[NotMutated]) -> list[str]: + if not not_mutated: + return [] + items = (f"
  • {escape(entry.summary())}
  • " for entry in not_mutated) + return [ + f"

    {NOT_MUTATED_HEADING}

    ", + f"

    {escape(NOT_MUTATED_HINT.capitalize())}.

    ", + "", + ] + + +def html_report( + combined: Sequence[CombinedResult], sources: Mapping[str, str], not_mutated: Sequence[NotMutated] = () +) -> str: score = mutation_score(entry.state for entry in combined) body = ( [ @@ -156,7 +172,7 @@ def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) ] if combined else ["

    No mutants were generated.

    "] - ) + ) + _not_mutated_list(not_mutated) page = [ "", '', diff --git a/strain/report.py b/strain/report.py index ccf17c6..ef431a0 100644 --- a/strain/report.py +++ b/strain/report.py @@ -7,6 +7,7 @@ from types import TracebackType from typing import Any, Self, TextIO +from .compile_time import COMPILE_TIME_TEST, NOT_MUTATED_HEADING, NOT_MUTATED_HINT, NotMutated from .mutators import Mutant from .results import ( DETECTED_STATES, @@ -22,6 +23,7 @@ MutantState.KILLED: "Killed", MutantState.KILLED_BY_SANITIZER: "Killed", MutantState.KILLED_BY_RESOURCE_LIMIT: "Killed", + MutantState.KILLED_AT_COMPILE_TIME: "Killed", MutantState.TIMEOUT: "Timeout", MutantState.SURVIVED: "Survived", MutantState.SKIPPED_UNEVALUATED: "Ignored", @@ -39,13 +41,15 @@ def format_score(score: float | None) -> str: def summary_line(test: str, results: Sequence[MutantResult], width: int) -> str: states = [result.state for result in results] - # Rare enough to show only when it happens. + # Rare enough to show only when they happen. memory = states.count(MutantState.KILLED_BY_RESOURCE_LIMIT) + compile_time = states.count(MutantState.KILLED_AT_COMPILE_TIME) return ( f"{test:<{width}} score {format_score(mutation_score(states))}" f" killed {states.count(MutantState.KILLED)}" f" sanitizer {states.count(MutantState.KILLED_BY_SANITIZER)}" f"{f' memory {memory}' if memory else ''}" + f"{f' compile-time {compile_time}' if compile_time else ''}" f" timeout {states.count(MutantState.TIMEOUT)}" f" survived {states.count(MutantState.SURVIVED)}" ) @@ -113,6 +117,8 @@ def file_scores(combined: Iterable[CombinedResult]) -> list[FileScore]: def hint(entry: CombinedResult) -> str: if entry.survivor is SurvivorKind.NOT_REACHED: return NOT_REACHED_HINT + if entry.covered_by == (COMPILE_TIME_TEST,): + return f"add a static_assert where `{entry.mutant.original}` and `{entry.mutant.replacement}` give different results" if entry.mutant.operator == "aor": return "assert the computed value" if entry.mutant.operator == "asr": @@ -203,11 +209,26 @@ def _markdown_survivors(combined: Sequence[CombinedResult]) -> list[str]: ] -def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None) -> str: +def _markdown_not_mutated(not_mutated: Sequence[NotMutated]) -> list[str]: + if not not_mutated: + return [] + return [ + "", + f"### {NOT_MUTATED_HEADING}", + "", + f"{NOT_MUTATED_HINT.capitalize()}.", + "", + *(f"- `{entry.location}` {entry.summary().partition(' ')[2]}" for entry in not_mutated), + ] + + +def markdown_summary( + combined: Sequence[CombinedResult], threshold: float | None, not_mutated: Sequence[NotMutated] = () +) -> str: """A summary for CI pages such as `$GITHUB_STEP_SUMMARY` or a pull request comment.""" heading = ["## strain mutation testing", ""] if not combined: - return "\n".join([*heading, "No mutants were generated."]) + "\n" + return "\n".join([*heading, "No mutants were generated.", *_markdown_not_mutated(not_mutated)]) + "\n" score = mutation_score(entry.state for entry in combined) lines = [ *heading, @@ -217,6 +238,7 @@ def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None "", *_markdown_file_table(combined), *_markdown_survivors(combined), + *_markdown_not_mutated(not_mutated), ] return "\n".join(lines) + "\n" @@ -238,14 +260,27 @@ def _mutant_element(entry: CombinedResult) -> dict[str, Any]: } -def mutation_testing_elements(combined: Iterable[CombinedResult], sources: Mapping[str, str]) -> dict[str, Any]: +def not_mutated_json(not_mutated: Sequence[NotMutated]) -> list[dict[str, Any]]: + return [ + {"file": entry.file, "function": entry.function, "lines": list(entry.lines), "mutants": entry.mutants} + for entry in not_mutated + ] + + +def mutation_testing_elements( + combined: Iterable[CombinedResult], sources: Mapping[str, str], not_mutated: Sequence[NotMutated] = () +) -> dict[str, Any]: + """The report in the mutation-testing-elements schema, plus `compileTimeOnly`: the constexpr code not mutated.""" files: dict[str, Any] = {} for entry in combined: file_report = files.setdefault( entry.mutant.file, {"language": "cpp", "source": sources[entry.mutant.file], "mutants": []} ) file_report["mutants"].append(_mutant_element(entry)) - return {"schemaVersion": "1", "thresholds": {"high": 80, "low": 60}, "files": files} + report: dict[str, Any] = {"schemaVersion": "1", "thresholds": {"high": 80, "low": 60}, "files": files} + if not_mutated: + report["compileTimeOnly"] = not_mutated_json(not_mutated) + return report PreviousResults = dict[tuple[str, str], tuple[MutantState, SurvivorKind | None]] diff --git a/strain/results.py b/strain/results.py index ec38d43..f34caf0 100644 --- a/strain/results.py +++ b/strain/results.py @@ -19,6 +19,7 @@ class MutantState(enum.StrEnum): KILLED = "killed" KILLED_BY_SANITIZER = "killed_by_sanitizer" KILLED_BY_RESOURCE_LIMIT = "killed_by_resource_limit" + KILLED_AT_COMPILE_TIME = "killed_at_compile_time" TIMEOUT = "timeout" SURVIVED = "survived" SKIPPED_UNEVALUATED = "skipped_unevaluated" @@ -32,7 +33,13 @@ class SurvivorKind(enum.StrEnum): DETECTED_STATES = frozenset( - {MutantState.KILLED, MutantState.KILLED_BY_SANITIZER, MutantState.KILLED_BY_RESOURCE_LIMIT, MutantState.TIMEOUT} + { + MutantState.KILLED, + MutantState.KILLED_BY_SANITIZER, + MutantState.KILLED_BY_RESOURCE_LIMIT, + MutantState.KILLED_AT_COMPILE_TIME, + MutantState.TIMEOUT, + } ) SCORED_STATES = DETECTED_STATES | {MutantState.SURVIVED} diff --git a/strain/runner.py b/strain/runner.py index 2824355..9a69e8b 100644 --- a/strain/runner.py +++ b/strain/runner.py @@ -4,6 +4,7 @@ import json import math import os +import re import shlex import time from collections.abc import Iterator, Mapping, Sequence @@ -29,6 +30,16 @@ from .build_cache import BuildFailureCache, cache_path from .config import CONFIG_FILE, Config, load_config from .console import BOLD, DIM, RED, STATE_STYLES, UiMode, create_console, plural, score_style +from .compile_time import ( + COMPILE_TIME_TEST, + KILLED_AT_COMPILE_TIME, + NOT_MUTATED_HEADING, + NOT_MUTATED_HINT, + NotMutated, + compile_time_check_failure, + constant_lines, + not_mutated, +) from .coverage import FileCoverage, GcovCoverage from .estimate import Estimate, MutantCost, format_eta from .journal import Journal @@ -87,6 +98,9 @@ MEMORY_FACTOR = 4 MINIMUM_MEMORY_MEGABYTES = 512 DEFAULT_ACTION_ITEMS = 10 +CXX_SUFFIXES = frozenset( + {".h", ".hh", ".hpp", ".hxx", ".h++", ".ipp", ".inl", ".tpp", ".c", ".cc", ".cpp", ".cxx", ".c++"} +) SOURCE_ENCODING = "utf-8" SOURCE_ERRORS = "surrogateescape" @@ -127,6 +141,7 @@ class RunOptions: max_per_line: int | None = None # Megabytes per test run with a mutant; None derives it from the baseline, and 0 turns the limit off. memory_limit: int | None = None + compile_time: bool = False resume: bool = False show_all: bool = False ui: UiMode = UiMode.AUTO @@ -210,6 +225,39 @@ class Outcome: detail: str = "" +# An absolute path in a compiler diagnostic: it ends at whitespace, a colon, a quote or a bracket. +ABSOLUTE_PATH = re.compile(r"(? str: + """`diagnostics` with paths in the project relative to it, so details and the build cache stay short. + + A path counts if it resolves into the project, so a symlinked spelling such as macOS's /var for /private/var + is shortened too. + """ + + def shorten(match: re.Match[str]) -> str: + try: + return Path(match.group(0)).resolve().relative_to(source_dir).as_posix() + except (OSError, ValueError): + return match.group(0) + + return ABSOLUTE_PATH.sub(shorten, diagnostics) + + +def failure_description(diagnostics: str, otherwise: str) -> str: + """Why a mutant did not build: the compile-time check it failed, which kills it, or else `otherwise`.""" + check = compile_time_check_failure(diagnostics) + return otherwise if check is None else f"{KILLED_AT_COMPILE_TIME}: {check}" + + +def build_failure_state(failure: str) -> MutantState: + """A failed compile-time check detects the mutant; any other build failure only means it is invalid code.""" + return ( + MutantState.KILLED_AT_COMPILE_TIME if failure.startswith(KILLED_AT_COMPILE_TIME) else MutantState.COMPILE_ERROR + ) + + def survivor_kind(coverage: FileCoverage, line: int) -> SurvivorKind | None: line_coverage = coverage.lines.get(line) if line_coverage is None or line_coverage.branches == 0: @@ -240,6 +288,10 @@ def __init__(self, options: RunOptions, out: TextIO, err: TextIO) -> None: self.test_targets: dict[str, Target] = {} self.test_executables: dict[str, Path] = {} self.memory_limits: dict[str, ctest.MemoryLimit] = {} + self.build_log: list[str] = [] + # Every line some test ran without a mutant, by file, to find the constexpr code that only the compiler runs. + self.runtime_lines: dict[Path, set[int]] = {} + self.compile_time_only: list[NotMutated] = [] self.times = MutantTimes() self.eta = "" self.build_is_mutated = False @@ -254,9 +306,16 @@ def show_banner(self) -> None: console.field("reports", str(self.options.report_dir), 7) def build(self, target: str | None = None) -> bool: + """Builds `target`, keeping what it printed in `build_log` to tell a failed compile-time check apart.""" self.console.build_started(shlex.join(build_command(Path(short_path(self.build_dir)), target, self.build_jobs))) + self.build_log = [] + + def show(line: str) -> None: + self.build_log.append(line) + self.console.build_output(line) + try: - return build(self.build_dir, target, self.console.build_output, self.build_jobs) + return build(self.build_dir, target, show, self.build_jobs) finally: self.build_clock.build_finished() @@ -468,11 +527,13 @@ def build_failure(self, planned: PlannedMutant, targets: Sequence[str | None], l """Compiles the mutated source; returns why it does not build, or None when it does.""" command = self.database.command_for(planned.source.path) self.console.status(f"{label} checking syntax…") - if command is not None and not command.passes_syntax_check(): - return "rejected by -fsyntax-only" + diagnostics = None if command is None else command.syntax_check() + if diagnostics is not None: + return failure_description(relative_paths(diagnostics, self.source_dir), "rejected by -fsyntax-only") self.console.status(f"{label} building…") - if not all(self.build(target) for target in targets): - return "build failed" + for target in targets: + if not self.build(target): + return failure_description(relative_paths("\n".join(self.build_log), self.source_dir), "build failed") return None def build_mutant(self, planned: PlannedMutant, targets: Sequence[str | None], label: str) -> Outcome | None: @@ -482,14 +543,14 @@ def build_mutant(self, planned: PlannedMutant, targets: Sequence[str | None], la if failure is None: return None self.build_failures.add(planned.mutant, failure) - return Outcome(MutantState.COMPILE_ERROR, detail=failure) + return Outcome(build_failure_state(failure), detail=failure) def known_build_failure(self, planned: PlannedMutant) -> Outcome | None: - """The compile error a mutant known to break the build gets without mutating or building anything.""" + """The outcome a mutant known not to build gets without mutating or building anything.""" failure = self.build_failures.failure(planned.mutant) - return ( - None if failure is None else Outcome(MutantState.COMPILE_ERROR, detail=f"{failure}, known from the cache") - ) + if failure is None: + return None + return Outcome(build_failure_state(failure), detail=f"{failure}, known from the cache") def run_mutated(self, test: str, timeout: float, coverage: FileCoverage, line: int, label: str) -> Outcome: self.console.status(f"{label} running {test}…") @@ -575,6 +636,7 @@ def run_test(self, test: str, events: EventLog) -> list[MutantResult]: timeout = mutant_timeout(duration) coverage = self.coverage.collect() self.verify_coverage_data(test, coverage) + self.note_runtime_lines(coverage) planned = self.plan_mutants(self.source_files(coverage, self.test_code(test))) self.console.plan([entry.mutant for entry in planned]) estimate = Estimate([MutantCost(entry.mutant.file, duration) for entry in planned]) @@ -613,6 +675,89 @@ def test_planned( self.show_mutant(progress, entry.mutant, outcome.state, outcome.detail) return outcome.state + def note_runtime_lines(self, coverage: dict[Path, FileCoverage]) -> None: + for path, file_coverage in coverage.items(): + self.runtime_lines.setdefault(path.resolve(), set()).update(file_coverage.executed_lines()) + + def project_files(self, tests: Sequence[str]) -> Iterator[SourceFile]: + """The project's C++ sources and headers that might hold constexpr code: not test code, not excluded.""" + test_code = frozenset().union(*(self.test_code(test) for test in tests)) + for directory, subdirectories, files in os.walk(self.source_dir): + folder = Path(directory) + subdirectories[:] = sorted( + name + for name in subdirectories + if not name.startswith(".") and not (folder / name).resolve().is_relative_to(self.build_dir) + ) + for name in sorted(files): + path = (folder / name).resolve() + if path.suffix not in CXX_SUFFIXES or path in test_code: + continue + relative = path.relative_to(self.source_dir).as_posix() + data = path.read_bytes() + if self.config.is_excluded(relative) or (b"constexpr" not in data and b"consteval" not in data): + continue + yield SourceFile(path, relative, data.decode(SOURCE_ENCODING, SOURCE_ERRORS), FileCoverage()) + + def compile_time_mutants(self, tests: Sequence[str]) -> list[tuple[PlannedMutant, str]]: + """The mutants on lines of constexpr and consteval functions that no test ran, with each function's name.""" + found: list[tuple[PlannedMutant, str]] = [] + for source in self.project_files(tests): + executed = self.runtime_lines.get(source.path, set()) + functions = {line: name for line, name in constant_lines(source.text).items() if line not in executed} + lines = set(functions) + if self.since is not None: + lines &= self.since.get(source.relative, set()) + suppressed = frozenset(suppressed_lines(source.text, self.config.arid)) + for mutant in mutants_at(source.relative, self.sites_for(source), lines, self.options.max_per_line): + found.append((PlannedMutant(source, suppressed, mutant), functions[mutant.line])) + return found + + def check_compile_time(self, tests: Sequence[str], events: EventLog) -> None: + """Reports the constexpr code no test runs, or with --compile-time mutates it and builds each mutant.""" + found = self.compile_time_mutants(tests) + if not self.options.compile_time: + self.compile_time_only = not_mutated(found) + return + if found: + self.run_compile_time( + [planned for planned, _ in found], {planned.mutant.id: name for planned, name in found}, events + ) + + def run_compile_time( + self, planned: Sequence[PlannedMutant], functions: Mapping[str, str], events: EventLog + ) -> None: + self.console.heading("Compile time") + self.console.test_started(COMPILE_TIME_TEST, 1, 1) + self.console.note(f"{plural(len(planned), 'mutant')} on constexpr lines no test runs") + self.console.plan([entry.mutant for entry in planned]) + estimate = Estimate([MutantCost(entry.mutant.file, 0.0) for entry in planned]) + self.eta = "" + width = len(str(len(planned))) + for number, entry in enumerate(planned, start=1): + progress = f"[{number:>{width}}/{len(planned)}]" + self.console.mutant_started(number - 1, entry.source.text, entry.mutant) + with self.estimating(estimate, number - 1, events): + outcome = self.evaluate_at_compile_time(entry, f" {progress} {describe_mutant(entry.mutant)}") + result = MutantResult(entry.mutant, COMPILE_TIME_TEST, outcome.state, None, functions[entry.mutant.id]) + self.record(result, events) + self.console.mutant_evaluated(result, entry.source.text, None) + self.show_mutant(progress, entry.mutant, outcome.state, outcome.detail) + self.console.mutant_finished(number - 1, outcome.state) + self.rebuild_if_mutated() + + def evaluate_at_compile_time(self, planned: PlannedMutant, label: str) -> Outcome: + """Builds everything with the mutant in place: only a failed compile-time check can detect it.""" + ignored = self.ignored_outcome(planned) + if ignored is not None: + return ignored + known = self.known_build_failure(planned) + if known is not None: + return known + with self.mutated(planned): + problem = self.build_mutant(planned, [None], label) + return problem or Outcome(MutantState.SURVIVED, detail="builds, and no test runs this line") + def verify_coverage_data(self, test: str, coverage: dict[Path, FileCoverage]) -> None: if self.coverage_verified: return @@ -682,11 +827,20 @@ def read_sources(self, combined: Sequence[CombinedResult]) -> dict[str, str]: def write_reports(self, combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> None: report_dir = self.options.report_dir - (report_dir / "report.json").write_text( - json.dumps(mutation_testing_elements(combined, sources), indent=2, ensure_ascii=False), encoding="utf-8" - ) - (report_dir / "index.html").write_text(html_report(combined, sources), encoding="utf-8") - (report_dir / "summary.md").write_text(markdown_summary(combined, self.options.threshold), encoding="utf-8") + not_mutated = self.compile_time_only + report = mutation_testing_elements(combined, sources, not_mutated) + (report_dir / "report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + (report_dir / "index.html").write_text(html_report(combined, sources, not_mutated), encoding="utf-8") + summary = markdown_summary(combined, self.options.threshold, not_mutated) + (report_dir / "summary.md").write_text(summary, encoding="utf-8") + + def show_not_mutated(self) -> None: + if not self.compile_time_only: + return + self.console.heading(NOT_MUTATED_HEADING) + for entry in self.compile_time_only: + self.console.line(f" · {entry.summary()}") + self.console.line(f" {self.console.paint(NOT_MUTATED_HINT, DIM)}") def show_survivors(self, combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> None: survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED] @@ -770,6 +924,7 @@ def baseline_tests(self, tests: Sequence[str], events: EventLog) -> list[Baselin continue coverage = self.coverage.collect() self.verify_coverage_data(test, coverage) + self.note_runtime_lines(coverage) baselined.append(BaselinedTest(test, duration, self.source_files(coverage, self.test_code(test)))) return baselined @@ -900,9 +1055,12 @@ def require_mutants(self, combined: Sequence[CombinedResult]) -> None: """Without --since, a run that mutated nothing tested nothing, so it must not pass.""" if combined or self.options.since is not None: return + compile_time = sum(entry.mutants for entry in self.compile_time_only) + hint = f"; {plural(compile_time, 'mutant')} wait on constexpr lines no test runs: try --compile-time" raise StrainError( "no mutant was generated for any test, so nothing was mutation tested; check that the tests pass, that " f"the build writes coverage data and that {CONFIG_FILE} does not exclude every source" + f"{hint if compile_time else ''}" ) def report(self, combined: Sequence[CombinedResult], events: EventLog, ending: str, start: float) -> float | None: @@ -914,6 +1072,7 @@ def report(self, combined: Sequence[CombinedResult], events: EventLog, ending: s self.show_survivors(combined, sources) self.show_action_items(combined) self.show_files(combined) + self.show_not_mutated() self.show_summary(combined, score) self.console.line(f"\n{self.console.paint(f'{ending} after {time.monotonic() - start:.1f}s', DIM)}") return score @@ -935,6 +1094,7 @@ def run(self) -> int: self.console.step(f"resuming: {plural(len(events.previous_results()), 'earlier result')} reused") try: self.run_strategy(tests, events) + self.check_compile_time(tests, events) except KeyboardInterrupt: self.report(combine(self.results), events, "interrupted", start) raise diff --git a/strain/tui.py b/strain/tui.py index 9532a6e..01bbec5 100644 --- a/strain/tui.py +++ b/strain/tui.py @@ -66,6 +66,7 @@ MutantState.KILLED: "✔", MutantState.KILLED_BY_SANITIZER: "✔", MutantState.KILLED_BY_RESOURCE_LIMIT: "✔", + MutantState.KILLED_AT_COMPILE_TIME: "✔", MutantState.TIMEOUT: "✔", MutantState.SURVIVED: "✗", MutantState.SKIPPED_UNEVALUATED: "·", @@ -78,6 +79,7 @@ MutantState.KILLED: "green", MutantState.KILLED_BY_SANITIZER: "green", MutantState.KILLED_BY_RESOURCE_LIMIT: "green", + MutantState.KILLED_AT_COMPILE_TIME: "green", MutantState.TIMEOUT: "green", MutantState.SURVIVED: "bold red", MutantState.SKIPPED_UNEVALUATED: "dim", @@ -90,6 +92,7 @@ SHORT_STATES = { MutantState.KILLED_BY_SANITIZER: "sanitizer", MutantState.KILLED_BY_RESOURCE_LIMIT: "memory", + MutantState.KILLED_AT_COMPILE_TIME: "constexpr", MutantState.COMPILE_ERROR: "no build", MutantState.SKIPPED_UNEVALUATED: "skipped", } @@ -365,7 +368,8 @@ def run_scores(state: LiveState) -> Text: for kind, count in state.counts().items(): scores.append(" ") scores.append_text(state_text(kind, glyph_only=True)) - scores.append(f" {kind} {count}", style=STATE_STYLES[kind]) + # The short labels, so the totals fit beside the timing even with every kind of kill. + scores.append(f" {SHORT_STATES.get(kind, kind)} {count}", style=STATE_STYLES[kind]) return scores @@ -746,7 +750,7 @@ def refresh_filters(self) -> None: for state, count in counts.items(): tab_id = f"filter-{state}" existing = filters.query(f"#{tab_id}") - label = f"{state} {count}" + label = f"{SHORT_STATES.get(state, state)} {count}" if existing: existing.first(Tab).label = label else: diff --git a/tests/test_cli.py b/tests/test_cli.py index 92eef46..5d35493 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,6 +41,7 @@ def parse(*argv: str): "3", "--memory-limit", "2048", + "--compile-time", "--resume", "--all", "--report-dir", @@ -64,6 +65,7 @@ def test_defaults_match_the_readme(self) -> None: self.assertFalse(options.strict) self.assertIsNone(options.max_per_line) self.assertIsNone(options.memory_limit) + self.assertFalse(options.compile_time) self.assertFalse(options.resume) self.assertFalse(options.show_all) self.assertIs(options.engine, Engine.AUTO) @@ -82,6 +84,7 @@ def test_every_option(self) -> None: self.assertTrue(options.strict) self.assertEqual(options.max_per_line, 3) self.assertEqual(options.memory_limit, 2048) + self.assertTrue(options.compile_time) self.assertTrue(options.resume) self.assertTrue(options.show_all) self.assertEqual(options.report_dir, Path("out")) diff --git a/tests/test_compile_time.py b/tests/test_compile_time.py new file mode 100644 index 0000000..2af605d --- /dev/null +++ b/tests/test_compile_time.py @@ -0,0 +1,135 @@ +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +from strain.compile_time import ( + NotMutated, + compile_time_check_failure, + constant_functions, + constant_lines, + not_mutated, +) +from strain.mutators import Mutant +from strain.results import MutantState +from strain.runner import build_failure_state, failure_description, relative_paths + +SOURCE = """\ +template constexpr T twice(T x) noexcept(true) +{ + if constexpr (sizeof(T) > 1) { return x + x; } + return x; +} +consteval int square(int value) { return value * value; } +constexpr int answer = 42; +static constexpr int table[] = { 1, 2 }; +constexpr Point origin{ 0, 0 }; +constexpr int declared_only(int value); +constexpr auto lambda = [](int value) { return value; }; +struct S +{ + constexpr S &operator=(const S &) { return *this; } +}; +constexpr int outer(int v) +{ + constexpr auto inner = [](int w) constexpr { return w; }; + // constexpr int commented(int v) { return v; } + return inner(v); +} +""" + + +class ConstantFunctionsTest(unittest.TestCase): + def test_finds_function_bodies_and_skips_variables_declarations_and_if_constexpr(self) -> None: + self.assertEqual( + [(function.name, function.keyword, sorted(function.body_lines)) for function in constant_functions(SOURCE)], + [ + ("twice", "constexpr", [3, 4]), + ("square", "consteval", [6]), + ("operator", "constexpr", [14]), + ("outer", "constexpr", [18, 20]), + ], + ) + + def test_each_line_names_its_function(self) -> None: + self.assertEqual(constant_lines(SOURCE)[3], "twice") + self.assertNotIn(7, constant_lines(SOURCE)) + + +class CompileTimeCheckFailureTest(unittest.TestCase): + def test_failed_checks_from_gcc_and_clang(self) -> None: + for diagnostic in ( + "a.cpp:2:33: error: static assertion failed", + "a.cpp:4:22: error: non-constant condition for static assertion", + "a.cpp:3:38: error: '(10 / 0)' is not a constant expression", + "a.cpp:2:15: error: static assertion failed due to requirement 'clamp(9, 4) == 3'", + "a.cpp:2:15: error: static_assert expression is not an integral constant expression", + "a.cpp:5:17: error: constexpr variable 'x' must be initialized by a constant expression", + ): + with self.subTest(diagnostic=diagnostic): + self.assertEqual( + compile_time_check_failure(f"[1/2] Building\n{diagnostic}\nninja: build stopped"), diagnostic + ) + + def test_invalid_code_is_not_a_failed_check(self) -> None: + self.assertIsNone(compile_time_check_failure("b.cpp:1:27: error: expected primary-expression before '/' token")) + + def test_a_failed_check_kills_the_mutant_and_any_other_failure_is_a_compile_error(self) -> None: + killed = failure_description("a.cpp:2:33: error: static assertion failed", "build failed") + self.assertEqual(killed, "killed at compile time: a.cpp:2:33: error: static assertion failed") + self.assertIs(build_failure_state(killed), MutantState.KILLED_AT_COMPILE_TIME) + # The build cache keeps the description, so a cached failure keeps its state. + self.assertIs(build_failure_state(f"{killed}, known from the cache"), MutantState.KILLED_AT_COMPILE_TIME) + self.assertIs( + build_failure_state(failure_description("error: expected ';'", "build failed")), MutantState.COMPILE_ERROR + ) + + +class RelativePathsTest(unittest.TestCase): + def test_paths_into_the_project_are_shortened_through_symlinks(self) -> None: + with tempfile.TemporaryDirectory() as folder: + real = Path(folder, "real", "project") + (real / "tests").mkdir(parents=True) + (real / "tests" / "a.cpp").write_text("") + link = Path(folder, "link") + link.symlink_to(real.parent) + source_dir = real.resolve() + diagnostics = ( + f"{link}/project/tests/a.cpp:3:15: error: static assertion failed\n" + f"{source_dir}/tests/a.cpp:4:1: note: here\n" + "/usr/include/c++/v1/vector:10:2: note: elsewhere" + ) + self.assertEqual( + relative_paths(diagnostics, source_dir), + "tests/a.cpp:3:15: error: static assertion failed\n" + "tests/a.cpp:4:1: note: here\n" + "/usr/include/c++/v1/vector:10:2: note: elsewhere", + ) + + +class NotMutatedTest(unittest.TestCase): + def test_groups_by_file_and_function_with_a_line_range(self) -> None: + def planned(line: int, column: int) -> SimpleNamespace: + return SimpleNamespace( + mutant=Mutant("include/limits.hpp", line, column, line * 100 + column, "ror", "<", "<=") + ) + + found = [ + (planned(7, 13), "clamp"), + (planned(7, 20), "clamp"), + (planned(8, 13), "clamp"), + (planned(12, 5), "square"), + ] + self.assertEqual( + not_mutated(found), # type: ignore[arg-type] + [ + NotMutated("include/limits.hpp", "clamp", (7, 8), 3), + NotMutated("include/limits.hpp", "square", (12,), 1), + ], + ) + self.assertEqual(not_mutated(found)[0].summary(), "include/limits.hpp:7-8 clamp(): 3 mutants on 2 lines") # type: ignore[arg-type] + self.assertEqual(not_mutated(found)[1].summary(), "include/limits.hpp:12 square(): 1 mutant on 1 line") # type: ignore[arg-type] + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 77f81b5..c0389ac 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -115,6 +115,40 @@ def mutant_evaluated(self, result: MutantResult, source: str, line: LineCoverage COUNTDOWN_TEST = '#include "sample.hpp"\n\nint main() { return countdown(3).size() == 3 ? 0 : 1; }\n' +CONSTEXPR_HEADER = """#ifndef SAMPLE_HPP +#define SAMPLE_HPP + +using Value = int; + +constexpr int clamp_index(int index, int size) +{ + if (index < 0) { return 0; } + if (index >= size) { return size - 1; } + return index; +} + +constexpr Value first_of(const Value *values) +{ + const Value *start = values; + return start[0]; +} + +[[nodiscard]] int add(int lhs, int rhs); + +#endif +""" +CONSTEXPR_TEST = """#include "sample.hpp" + +static_assert(clamp_index(-1, 4) == 0); +static_assert(clamp_index(9, 4) == 3); +static_assert(clamp_index(2, 4) == 2); +constexpr Value values[] = { 7 }; +static_assert(first_of(values) == 7); + +int main() { return add(2, 3) == 5 ? 0 : 1; } +""" + + def run_strain(*argv: str) -> tuple[int, str, str]: stdout = io.StringIO() stderr = io.StringIO() @@ -590,6 +624,65 @@ def test_a_mutant_that_allocates_forever_stops_at_the_memory_limit_under_asan(se def test_a_mutant_that_allocates_forever_stops_at_the_memory_limit_without_asan(self) -> None: self.run_countdown("--coverage", "--coverage", "--safety-nets", "ignore") + def run_constexpr(self, *argv: str) -> str: + (self.root / "include" / "sample.hpp").write_text(CONSTEXPR_HEADER) + (self.root / "src" / "sample.cpp").write_text( + '#include "sample.hpp"\n\nint add(int lhs, int rhs) { return lhs + rhs; }\n' + ) + (self.root / "tests" / "sample_tests.cpp").write_text(CONSTEXPR_TEST) + (self.root / "tests" / "other_tests.cpp").write_text("int main() { return 0; }\n") + configure_project(self.root, self.build_dir, cxx_flags=f"{CXX_FLAGS} -std=c++20") + exit_code, stdout, _ = run_strain( + *("--build-dir", str(self.build_dir), "--report-dir", str(self.report_dir)), + *("-R", "^sample_tests$", "--engine", "lexer", *argv), + ) + self.assertEqual(exit_code, 0, stdout) + return stdout + + def header_states(self) -> list[tuple[int, str, str, str]]: + report = json.loads((self.report_dir / "report.json").read_text()) + mutants = report["files"].get("include/sample.hpp", {"mutants": []})["mutants"] + return sorted( + (mutant["location"]["start"]["line"], mutant["mutatorName"], mutant["replacement"], mutant["statusReason"]) + for mutant in mutants + ) + + def test_constexpr_code_checked_only_by_static_assert_is_reported_as_not_mutated(self) -> None: + stdout = self.run_constexpr() + + self.assertIn("▸ Not mutated: evaluated at compile time only", stdout) + self.assertIn("· include/sample.hpp:8-9 clamp_index(): 5 mutants on 2 lines", stdout) + self.assertIn("· include/sample.hpp:15 first_of(): 1 mutant on 1 line", stdout) + self.assertEqual(self.header_states(), []) + report = json.loads((self.report_dir / "report.json").read_text()) + self.assertEqual( + report["compileTimeOnly"][0], + {"file": "include/sample.hpp", "function": "clamp_index", "lines": [8, 9], "mutants": 5}, + ) + summary = (self.report_dir / "summary.md").read_text() + self.assertIn("### Not mutated: evaluated at compile time only", summary) + self.assertIn("- `include/sample.hpp:8-9` clamp_index(): 5 mutants on 2 lines", summary) + + def test_compile_time_mutants_fail_static_asserts_or_survive_while_invalid_code_is_a_compile_error(self) -> None: + stdout = self.run_constexpr("--compile-time") + + self.assertEqual( + self.header_states(), + [ + (8, "ror", "<=", "survived"), + (8, "ror", ">=", "killed_at_compile_time"), + (9, "aor", "+", "killed_at_compile_time"), + (9, "ror", "<", "killed_at_compile_time"), + (9, "ror", ">", "survived"), + (15, "aor", "/", "compile_error"), + ], + ) + self.assertNotIn("Not mutated", stdout) + self.assertRegex( + stdout, r"killed at compile time: tests/sample_tests\.cpp:3:\d+: error: static assertion failed" + ) + self.assertIn("add a static_assert where `<` and `<=` give different results", stdout) + def test_dirty_working_tree_is_refused(self) -> None: configure_project(self.root, self.build_dir) (self.root / "src" / "sample.cpp").write_text("// local edit\n")