Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ A mutant that doesn't compile is a `compile_error` for every test, so strain bui
| `killed` | A test failed | Yes |
| `killed_by_sanitizer` | ASan or UBSan caught it, the assertions didn't | Yes, flagged |
| `killed_by_resource_limit` | The test reached its [memory limit](#memory-limit) | Yes |
| `killed_at_compile_time` | A `static_assert` or a constant evaluation failed with it in place | Yes |
| `timeout` | Ran too long | Yes |
| `survived` | All tests passed | Yes |
| `skipped_unevaluated` | Operand never evaluated by this test | No |
Expand All @@ -168,7 +169,25 @@ A mutant that doesn't compile is a `compile_error` for every test, so strain bui

Survivors are labeled `not_reached` (needs a new test input) or `not_detected` (needs a stronger assertion) when branch coverage is available.

Mutation score = (killed + killed_by_sanitizer + killed_by_resource_limit + timeout) ÷ (those + survived)
Mutation score = (killed + killed_by_sanitizer + killed_by_resource_limit + killed_at_compile_time + timeout) ÷
(those + survived)

### Compile-time code

gcov counts what runs, so a `constexpr` or `consteval` function checked only by `static_assert` has no executed lines.
strain would never mutate it, and a header used only that way doesn't appear in the coverage data at all. strain finds
these functions in the sources instead:

- **By default:** the lines of `constexpr` and `consteval` functions that hold mutants but that no test runs are listed
under **Not mutated: evaluated at compile time only**. That covers the console, `summary.md`, `index.html` and
`compileTimeOnly` in `report.json`. You can tell "well tested" from "never looked at".
- **With `--compile-time`:** after the tests, strain mutates those lines and builds everything with each mutant.
- A failed `static_assert` or constant evaluation kills it as `killed_at_compile_time`. Its detail names the failed
check.
- A mutant that builds survives, with a hint to add a `static_assert`.
- Invalid code stays `compile_error`.
- **Any mutant:** a build or `-fsyntax-only` check that fails on a compile-time check counts as `killed_at_compile_time`,
even on lines the tests run. The [build cache](#build-cache) remembers it like any other build failure.

### Memory limit

Expand Down Expand Up @@ -222,6 +241,7 @@ strain run --safety-nets=warn
| `--threshold` | | Exit non-zero below this score |
| `--strict` | off | Abort if any test is excluded |
| `--max-per-line` | | Cap mutants per source line |
| `--compile-time` | off | Also mutate `constexpr` code no test runs; see [Compile-time code](#compile-time-code) |
| `--memory-limit` | 4× the baseline, at least 512 MB | Memory each test may use with a mutant in place, in MB; `0` for no limit. See [Memory limit](#memory-limit) |
| `--resume` | | Continue an interrupted run |
| `--all` | off | Show every survivor, not just the top actions |
Expand Down
13 changes: 10 additions & 3 deletions strain/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,16 @@ def has_gcov_instrumentation(self) -> bool:
def has_source_based_coverage(self) -> bool:
return any(argument in SOURCE_BASED_COVERAGE_FLAGS for argument in self.arguments)

def passes_syntax_check(self) -> bool:
completed = subprocess.run(self.syntax_only_arguments(), cwd=self.directory, capture_output=True)
return completed.returncode == 0
def syntax_check(self) -> str | None:
"""None when the file compiles with -fsyntax-only, or else the compiler's diagnostics.

-fsyntax-only still evaluates `static_assert` and constant expressions, so its diagnostics tell a mutant that
fails a compile-time check from one that is invalid code.
"""
completed = subprocess.run(
self.syntax_only_arguments(), cwd=self.directory, capture_output=True, text=True, errors="replace"
)
return None if completed.returncode == 0 else completed.stdout + completed.stderr


def _command_from_entry(entry: Mapping[str, Any]) -> CompileCommand:
Expand Down
6 changes: 6 additions & 0 deletions strain/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,11 @@ def _add_policy_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--threshold", type=float, metavar="PERCENT", help="exit non-zero below this score")
parser.add_argument("--strict", action="store_true", help="abort if any test is excluded")
parser.add_argument("--max-per-line", type=_positive_int, metavar="N", help="cap mutants per source line")
parser.add_argument(
"--compile-time",
action="store_true",
help="also mutate constexpr code no test runs; a failed static_assert or constant evaluation kills a mutant",
)
parser.add_argument(
"--memory-limit",
type=_megabytes,
Expand Down Expand Up @@ -129,6 +134,7 @@ def options_from_arguments(arguments: argparse.Namespace) -> RunOptions:
strict=arguments.strict,
max_per_line=arguments.max_per_line,
memory_limit=arguments.memory_limit,
compile_time=arguments.compile_time,
resume=arguments.resume,
show_all=arguments.show_all,
ui=arguments.ui,
Expand Down
142 changes: 142 additions & 0 deletions strain/compile_time.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
"""Code that only the compiler runs: constexpr and consteval functions evaluated in constant expressions.

gcov counts what runs, so a constexpr function checked only by `static_assert` has no executed lines and is never
mutated, and a header used only that way doesn't appear in the coverage data at all. strain finds these functions in
the sources instead, says which of their lines no test runs, and with --compile-time mutates them: a mutant that
makes a `static_assert` or a constant evaluation fail is killed at compile time.
"""

import re
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any

from .lexer import Token, TokenKind, tokenize

CONSTANT_KEYWORDS = frozenset({"constexpr", "consteval"})
# Diagnostics of a compile-time check that failed, from GCC and Clang. Anything else that stops the build means the
# mutant is simply invalid code: a compile error.
COMPILE_TIME_CHECK_FAILED = re.compile(
r"static assertion failed|static_assert failed|non-constant condition for static assertion"
r"|is not a constant expression|not an integral constant expression|must be initialized by a constant expression"
)
KILLED_AT_COMPILE_TIME = "killed at compile time"
# The name results get for the mutants --compile-time builds: the compile-time checks are what tests them.
COMPILE_TIME_TEST = "compile-time checks"
NOT_MUTATED_HEADING = "Not mutated: evaluated at compile time only"
NOT_MUTATED_HINT = (
"no test runs these constexpr lines; --compile-time mutates them, and a failed static_assert kills them"
)


@dataclass(frozen=True, slots=True)
class ConstantFunction:
name: str
keyword: str
# The lines with code between the function's braces.
body_lines: frozenset[int]


def compile_time_check_failure(diagnostics: str) -> str | None:
"""The diagnostic line of the compile-time check that failed, or None when the build failed for another reason."""
for line in diagnostics.splitlines():
if COMPILE_TIME_CHECK_FAILED.search(line):
return line.strip()
return None


def _matching(tokens: list[Token], start: int, opening: str, closing: str) -> int | None:
depth = 0
for index in range(start, len(tokens)):
if tokens[index].text == opening:
depth += 1
elif tokens[index].text == closing:
depth -= 1
if depth == 0:
return index
return None


def _body_start(tokens: list[Token], start: int) -> tuple[str, int] | None:
"""After a constexpr or consteval keyword: the function's name and opening brace, or None for anything else.

A variable reaches `=` or `;` first, or opens a brace before any parameter list; `if constexpr` is not a
declaration at all.
"""
name: str | None = None
index = start
while index < len(tokens):
token = tokens[index]
if token.text == ";" or (token.text == "=" and tokens[index - 1].text != "operator"):
return None
if token.text == "(":
if name is None:
previous = tokens[index - 1]
name = previous.text if previous.kind is TokenKind.IDENTIFIER else "operator"
closing = _matching(tokens, index, "(", ")")
if closing is None:
return None
index = closing
elif token.text == "{":
return None if name is None else (name, index)
index += 1
return None


def constant_functions(source: str) -> list[ConstantFunction]:
tokens = [token for token in tokenize(source) if token.kind is not TokenKind.COMMENT]
functions: list[ConstantFunction] = []
for index, token in enumerate(tokens):
if token.text not in CONSTANT_KEYWORDS or (index > 0 and tokens[index - 1].text == "if"):
continue
found = _body_start(tokens, index + 1)
if found is None:
continue
name, opening = found
closing = _matching(tokens, opening, "{", "}")
if closing is None:
continue
lines = frozenset(inner.line for inner in tokens[opening + 1 : closing])
functions.append(ConstantFunction(name, token.text, lines))
return functions


def constant_lines(source: str) -> dict[int, str]:
"""Each line inside a constexpr or consteval function, with the innermost function's name."""
lines: dict[int, str] = {}
# Outer functions first, so a lambda or local function inside one names its own lines.
for function in sorted(constant_functions(source), key=lambda function: -len(function.body_lines)):
lines.update(dict.fromkeys(function.body_lines, function.name))
return lines


@dataclass(frozen=True, slots=True)
class NotMutated:
"""Lines of a constexpr or consteval function that hold mutants but that no test runs, so none was tested."""

file: str
function: str
lines: tuple[int, ...]
mutants: int

@property
def location(self) -> str:
first, last = self.lines[0], self.lines[-1]
return f"{self.file}:{first}" if first == last else f"{self.file}:{first}-{last}"

def summary(self) -> str:
"""`src/limits.hpp:6-7 clamp_index(): 3 mutants on 2 lines`"""
mutants = f"{self.mutants} mutant{'' if self.mutants == 1 else 's'}"
lines = f"{len(self.lines)} line{'' if len(self.lines) == 1 else 's'}"
return f"{self.location} {self.function}(): {mutants} on {lines}"


def not_mutated(found: "Sequence[tuple[Any, str]]") -> list[NotMutated]:
"""Groups (planned mutant, function) pairs by file and function."""
groups: dict[tuple[str, str], list[int]] = {}
for planned, function in found:
groups.setdefault((planned.mutant.file, function), []).append(planned.mutant.line)
return [
NotMutated(file, function, tuple(sorted(set(lines))), len(lines))
for (file, function), lines in sorted(groups.items(), key=lambda item: (item[0][0], min(item[1])))
]
1 change: 1 addition & 0 deletions strain/console.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
MutantState.KILLED: GREEN,
MutantState.KILLED_BY_SANITIZER: GREEN,
MutantState.KILLED_BY_RESOURCE_LIMIT: GREEN,
MutantState.KILLED_AT_COMPILE_TIME: GREEN,
MutantState.TIMEOUT: GREEN,
MutantState.SURVIVED: BOLD + RED,
MutantState.SKIPPED_UNEVALUATED: DIM,
Expand Down
20 changes: 18 additions & 2 deletions strain/html_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from html import escape

from .console import HIGH_SCORE, LOW_SCORE
from .compile_time import NOT_MUTATED_HEADING, NOT_MUTATED_HINT, NotMutated
from .report import TAB_WIDTH, FileScore, file_scores, format_score, hint
from .results import DETECTED_STATES, CombinedResult, MutantState, mutation_score

Expand Down Expand Up @@ -145,7 +146,22 @@ def _counts(combined: Sequence[CombinedResult]) -> str:
return ", ".join(f"{escape(state)} {states.count(state)}" for state in MutantState if state in states)


def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> str:
def _not_mutated_list(not_mutated: Sequence[NotMutated]) -> list[str]:
if not not_mutated:
return []
items = (f"<li>{escape(entry.summary())}</li>" for entry in not_mutated)
return [
f"<h2>{NOT_MUTATED_HEADING}</h2>",
f"<p>{escape(NOT_MUTATED_HINT.capitalize())}.</p>",
"<ul>",
*items,
"</ul>",
]


def html_report(
combined: Sequence[CombinedResult], sources: Mapping[str, str], not_mutated: Sequence[NotMutated] = ()
) -> str:
score = mutation_score(entry.state for entry in combined)
body = (
[
Expand All @@ -156,7 +172,7 @@ def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str])
]
if combined
else ["<p>No mutants were generated.</p>"]
)
) + _not_mutated_list(not_mutated)
page = [
"<!DOCTYPE html>",
'<html lang="en"><head><meta charset="utf-8">',
Expand Down
45 changes: 40 additions & 5 deletions strain/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from types import TracebackType
from typing import Any, Self, TextIO

from .compile_time import COMPILE_TIME_TEST, NOT_MUTATED_HEADING, NOT_MUTATED_HINT, NotMutated
from .mutators import Mutant
from .results import (
DETECTED_STATES,
Expand All @@ -22,6 +23,7 @@
MutantState.KILLED: "Killed",
MutantState.KILLED_BY_SANITIZER: "Killed",
MutantState.KILLED_BY_RESOURCE_LIMIT: "Killed",
MutantState.KILLED_AT_COMPILE_TIME: "Killed",
MutantState.TIMEOUT: "Timeout",
MutantState.SURVIVED: "Survived",
MutantState.SKIPPED_UNEVALUATED: "Ignored",
Expand All @@ -39,13 +41,15 @@ def format_score(score: float | None) -> str:

def summary_line(test: str, results: Sequence[MutantResult], width: int) -> str:
states = [result.state for result in results]
# Rare enough to show only when it happens.
# Rare enough to show only when they happen.
memory = states.count(MutantState.KILLED_BY_RESOURCE_LIMIT)
compile_time = states.count(MutantState.KILLED_AT_COMPILE_TIME)
return (
f"{test:<{width}} score {format_score(mutation_score(states))}"
f" killed {states.count(MutantState.KILLED)}"
f" sanitizer {states.count(MutantState.KILLED_BY_SANITIZER)}"
f"{f' memory {memory}' if memory else ''}"
f"{f' compile-time {compile_time}' if compile_time else ''}"
f" timeout {states.count(MutantState.TIMEOUT)}"
f" survived {states.count(MutantState.SURVIVED)}"
)
Expand Down Expand Up @@ -113,6 +117,8 @@ def file_scores(combined: Iterable[CombinedResult]) -> list[FileScore]:
def hint(entry: CombinedResult) -> str:
if entry.survivor is SurvivorKind.NOT_REACHED:
return NOT_REACHED_HINT
if entry.covered_by == (COMPILE_TIME_TEST,):
return f"add a static_assert where `{entry.mutant.original}` and `{entry.mutant.replacement}` give different results"
if entry.mutant.operator == "aor":
return "assert the computed value"
if entry.mutant.operator == "asr":
Expand Down Expand Up @@ -203,11 +209,26 @@ def _markdown_survivors(combined: Sequence[CombinedResult]) -> list[str]:
]


def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None) -> str:
def _markdown_not_mutated(not_mutated: Sequence[NotMutated]) -> list[str]:
if not not_mutated:
return []
return [
"",
f"### {NOT_MUTATED_HEADING}",
"",
f"{NOT_MUTATED_HINT.capitalize()}.",
"",
*(f"- `{entry.location}` {entry.summary().partition(' ')[2]}" for entry in not_mutated),
]


def markdown_summary(
combined: Sequence[CombinedResult], threshold: float | None, not_mutated: Sequence[NotMutated] = ()
) -> str:
"""A summary for CI pages such as `$GITHUB_STEP_SUMMARY` or a pull request comment."""
heading = ["## strain mutation testing", ""]
if not combined:
return "\n".join([*heading, "No mutants were generated."]) + "\n"
return "\n".join([*heading, "No mutants were generated.", *_markdown_not_mutated(not_mutated)]) + "\n"
score = mutation_score(entry.state for entry in combined)
lines = [
*heading,
Expand All @@ -217,6 +238,7 @@ def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None
"",
*_markdown_file_table(combined),
*_markdown_survivors(combined),
*_markdown_not_mutated(not_mutated),
]
return "\n".join(lines) + "\n"

Expand All @@ -238,14 +260,27 @@ def _mutant_element(entry: CombinedResult) -> dict[str, Any]:
}


def mutation_testing_elements(combined: Iterable[CombinedResult], sources: Mapping[str, str]) -> dict[str, Any]:
def not_mutated_json(not_mutated: Sequence[NotMutated]) -> list[dict[str, Any]]:
return [
{"file": entry.file, "function": entry.function, "lines": list(entry.lines), "mutants": entry.mutants}
for entry in not_mutated
]


def mutation_testing_elements(
combined: Iterable[CombinedResult], sources: Mapping[str, str], not_mutated: Sequence[NotMutated] = ()
) -> dict[str, Any]:
"""The report in the mutation-testing-elements schema, plus `compileTimeOnly`: the constexpr code not mutated."""
files: dict[str, Any] = {}
for entry in combined:
file_report = files.setdefault(
entry.mutant.file, {"language": "cpp", "source": sources[entry.mutant.file], "mutants": []}
)
file_report["mutants"].append(_mutant_element(entry))
return {"schemaVersion": "1", "thresholds": {"high": 80, "low": 60}, "files": files}
report: dict[str, Any] = {"schemaVersion": "1", "thresholds": {"high": 80, "low": 60}, "files": files}
if not_mutated:
report["compileTimeOnly"] = not_mutated_json(not_mutated)
return report


PreviousResults = dict[tuple[str, str], tuple[MutantState, SurvivorKind | None]]
Expand Down
Loading
Loading