diff --git a/cli/aidd_docs/memory/testing.md b/cli/aidd_docs/memory/testing.md index 181c56f5..879e659f 100644 --- a/cli/aidd_docs/memory/testing.md +++ b/cli/aidd_docs/memory/testing.md @@ -36,6 +36,7 @@ How this package is tested: the layers, the tools, the conventions. - `pnpm smoke:real` reaches real host registries: [`smoke-real.md`](internal/smoke-real.md). - `pnpm smoke:collision` runs the real `codex` and `copilot` binaries in a throwaway HOME against a person's own install under the keys aidd uses, and checks `setup` and `clean` leave it as seeded. `smoke:real`'s unique names cannot meet that case. - `pnpm test:mutation:`; `mutation-scopes.json` declares each scope's globs (a leading `!` excludes) and the floor its score must hold. `tools` is split one scope per tool profile (`tools-claude`, `tools-codex`, …): a profile is a static declaration whose every mutant reruns each test that loads it, and the profiles together outlasted every other scope on a two-core runner. A weekly scheduled run replays every mutant with `--force`, so drift through a dependency an incremental run never replays is bounded to a week. `scripts/run-mutation.mjs` fails under the floor and keeps one incremental file per scope under `reports/mutation//`; `--force` reruns every mutant. Before a run the runner prunes the incremental file to kills alone: stryker reuses a result unless the mutant's file or a test that covered it changed, so a test written after the fact never reaches a mutant recorded as survived, uncovered or static, and a scope measured 74 read 67 in CI until it did. Raise a floor to the measured score after a run; never lower one without the reason in that file. +- `node scripts/run-mutation.mjs --changed []` mutates only the lines changed since `` (default `origin/next`) and names each survivor with its file and line. It holds no floor and writes no scope's incremental file, so it checks a branch before a push without moving any gate. - A unit or integration test reads the repository through `tests/helpers/repository-root.ts`, never by climbing `../` or `process.cwd()`: a mutation run copies `cli/` into a sandbox, where a relative climb lands nowhere (`tests-reach-the-repository-through-one-helper.arch.test.ts`). - Read counts live: a suite failing before producing a test contributes zero. diff --git a/cli/scripts/run-mutation.d.mts b/cli/scripts/run-mutation.d.mts index 36b9d527..3aea3128 100644 --- a/cli/scripts/run-mutation.d.mts +++ b/cli/scripts/run-mutation.d.mts @@ -7,7 +7,14 @@ export interface MutationReport { readonly files?: Readonly< Record< string, - { readonly mutants: readonly { readonly status: string; readonly static?: boolean }[] } + { + readonly mutants: readonly { + readonly status: string; + readonly static?: boolean; + readonly mutatorName?: string; + readonly location?: { readonly start: { readonly line: number } }; + }[]; + } > >; } @@ -21,3 +28,6 @@ export function strykerArgs( ): string[]; export function scoreOf(report: MutationReport): number; export function breakVerdict(score: number, declared: MutationScope): string | null; +export function changedRanges(diff: string): string[]; +export function changedArgs(ranges: readonly string[]): string[]; +export function survivorsOf(report: MutationReport): string[]; diff --git a/cli/scripts/run-mutation.mjs b/cli/scripts/run-mutation.mjs index 0c6f7e98..daa55461 100644 --- a/cli/scripts/run-mutation.mjs +++ b/cli/scripts/run-mutation.mjs @@ -62,6 +62,47 @@ export function pruneIncremental(report) { return { ...report, files }; } +const HUNK = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/; + +/** The lines a `git diff -U0` adds or changes under `src/`, as stryker `file:start-end` ranges. + * A pure deletion adds no line to mutate, and a file outside `src/` belongs to no scope. */ +export function changedRanges(diff) { + const ranges = []; + let file = null; + for (const line of diff.split("\n")) { + if (line.startsWith("+++ ")) { + const target = line.slice(4); + file = target.startsWith("b/") ? target.slice(2) : null; + continue; + } + const hunk = HUNK.exec(line); + if (hunk === null || file === null || !file.startsWith("src/") || !file.endsWith(".ts")) + continue; + const start = Number(hunk[1]); + const count = hunk[2] === undefined ? 1 : Number(hunk[2]); + if (count > 0) ranges.push(`${file}:${start}-${start + count - 1}`); + } + return ranges; +} + +/** No scope, no incremental file and no floor: a check on a branch must never move a gate. */ +export function changedArgs(ranges) { + return ["run", "--mutate", ranges.join(",")]; +} + +export function survivorsOf(report) { + const survivors = []; + for (const [name, file] of Object.entries(report.files ?? {})) { + for (const mutant of file.mutants) { + if (mutant.status !== "Survived" && mutant.status !== "NoCoverage") continue; + survivors.push( + `${name}:${mutant.location.start.line} ${mutant.mutatorName} (${mutant.status})` + ); + } + } + return survivors; +} + /** Below the declared floor is a failure the run itself raises; stryker's own `thresholds` * would need a config file per scope to say the same thing. */ export function breakVerdict(score, declared) { @@ -84,14 +125,56 @@ function pruneIncrementalFile(path) { } function usage(problem, scopes) { - console.error(`${problem}\n\nUsage: node scripts/run-mutation.mjs [--force]`); + console.error( + `${problem}\n\nUsage: node scripts/run-mutation.mjs [--force]\n node scripts/run-mutation.mjs --changed []` + ); console.error(`Scopes: ${Object.keys(scopes).join(", ")}`); process.exit(1); } +function git(args) { + const result = spawnSync("git", args, { cwd: CLI_ROOT, encoding: "utf8" }); + if (result.status !== 0) throw new Error(`git ${args.join(" ")} failed: ${result.stderr}`); + return result.stdout.trim(); +} + +function fileReports(dir) { + for (const name of WRITTEN_REPORTS) { + const written = join(REPORT_ROOT, name); + if (existsSync(written)) renameSync(written, join(dir, name)); + } +} + +function mainChanged(base = "origin/next") { + const mergeBase = git(["merge-base", base, "HEAD"]); + const ranges = changedRanges(git(["diff", "-U0", "--relative", mergeBase, "--", "src"])); + if (ranges.length === 0) { + console.log(`No line under src/ changed since ${base}: nothing to mutate.`); + return; + } + const dir = join(REPORT_ROOT, "changed"); + mkdirSync(dir, { recursive: true }); + const result = spawnSync(join(CLI_ROOT, "node_modules", ".bin", "stryker"), changedArgs(ranges), { + cwd: CLI_ROOT, + stdio: "inherit", + }); + rmSync(join(CLI_ROOT, ".stryker-tmp"), { recursive: true, force: true }); + fileReports(dir); + if (result.status !== 0) process.exit(result.status ?? 1); + const report = JSON.parse(readFileSync(join(dir, "mutation.json"), "utf8")); + const mutants = Object.values(report.files ?? {}).flatMap((file) => file.mutants).length; + const measured = + mutants === 0 ? "no mutant on those lines" : `score ${scoreOf(report).toFixed(1)}`; + console.log( + `\nReport: reports/mutation/changed/ (${measured}, ${ranges.length} changed range(s) since ${base})` + ); + for (const survivor of survivorsOf(report)) console.log(` survived: ${survivor}`); +} + function main() { const scopes = loadScopes(); const [scope, ...flags] = process.argv.slice(2); + if (scope === "--changed") return mainChanged(flags[0]); if (scope === undefined) usage("No scope given.", scopes); if (!Object.hasOwn(scopes, scope)) usage(`Unknown scope "${scope}".`, scopes); const force = flags.includes("--force"); @@ -109,10 +192,7 @@ function main() { // A sandbox survives an interrupted run and they grow to hundreds of megabytes. rmSync(join(CLI_ROOT, ".stryker-tmp"), { recursive: true, force: true }); - for (const name of WRITTEN_REPORTS) { - const written = join(REPORT_ROOT, name); - if (existsSync(written)) renameSync(written, join(scopeDir, name)); - } + fileReports(scopeDir); if (result.status !== 0) process.exit(result.status ?? 1); diff --git a/cli/tests/architecture/mutation-covers-source.arch.test.ts b/cli/tests/architecture/mutation-covers-source.arch.test.ts index 682406f0..96350b55 100644 --- a/cli/tests/architecture/mutation-covers-source.arch.test.ts +++ b/cli/tests/architecture/mutation-covers-source.arch.test.ts @@ -9,9 +9,12 @@ import { describe, expect, it } from "vitest"; import { HARNESS, scopesToRun } from "../../scripts/mutation-scopes-to-run.mjs"; import { breakVerdict, + changedArgs, + changedRanges, pruneIncremental, scoreOf, strykerArgs, + survivorsOf, } from "../../scripts/run-mutation.mjs"; import { matchesGlob, REPO_ROOT, read, sourceFiles } from "./helpers.js"; @@ -223,6 +226,52 @@ describe("the guard itself", () => { expect(pruned.files?.["a.ts"]?.mutants).toEqual([{ status: "Killed" }]); }); + it("mutates only the lines a diff adds or changes under src/, never a deletion or another file", () => { + const diff = [ + "diff --git a/src/a.ts b/src/a.ts", + "--- a/src/a.ts", + "+++ b/src/a.ts", + "@@ -10,2 +10,3 @@ function a() {", + "@@ -20 +21 @@ function b() {", + "@@ -30,4 +31,0 @@ function c() {", + "diff --git a/src/b.ts b/src/b.ts", + "--- /dev/null", + "+++ b/src/b.ts", + "@@ -0,0 +1,5 @@", + "diff --git a/README.md b/README.md", + "+++ b/README.md", + "@@ -1 +1 @@", + ].join("\n"); + expect(changedRanges(diff)).toEqual(["src/a.ts:10-12", "src/a.ts:21-21", "src/b.ts:1-5"]); + expect(changedArgs(["src/a.ts:10-12", "src/b.ts:1-5"])).toEqual([ + "run", + "--mutate", + "src/a.ts:10-12,src/b.ts:1-5", + ]); + }); + + it("names each mutant a test left alive, with its file and line", () => { + const report = { + files: { + "src/a.ts": { + mutants: [ + { status: "Killed", mutatorName: "BooleanLiteral", location: { start: { line: 3 } } }, + { status: "Survived", mutatorName: "StringLiteral", location: { start: { line: 7 } } }, + { + status: "NoCoverage", + mutatorName: "BlockStatement", + location: { start: { line: 9 } }, + }, + ], + }, + }, + }; + expect(survivorsOf(report)).toEqual([ + "src/a.ts:7 StringLiteral (Survived)", + "src/a.ts:9 BlockStatement (NoCoverage)", + ]); + }); + it("fails a score under the floor and passes one on it", () => { expect(breakVerdict(69.9, scopes.kernel)).toMatch(/below the 70/); expect(breakVerdict(70, scopes.kernel)).toBeNull();