diff --git a/.changeset/ts-share-program-across-compilers.md b/.changeset/ts-share-program-across-compilers.md new file mode 100644 index 00000000..82d3b6ab --- /dev/null +++ b/.changeset/ts-share-program-across-compilers.md @@ -0,0 +1,5 @@ +--- +"diagnostics-webpack-plugin": minor +--- + +Share the TypeScript check's worker, and the program it holds, between the compilers of a `MultiCompiler` whose TypeScript checks are configured alike. Before, each compiler built and kept its own program. Over 600 files, two compilers building one after the other went through a watch with one rebuild in 5.6–5.8 s, 9.8–10.4 CPU-seconds and 467–483 MB, against 9.7–10.3 s, 18.7–20.0 CPU-seconds and 803–853 MB. A run one compiler stops no longer stops another's on the shared worker. diff --git a/.changeset/ts-stop-stale-runs.md b/.changeset/ts-stop-stale-runs.md new file mode 100644 index 00000000..6361e5b2 --- /dev/null +++ b/.changeset/ts-stop-stale-runs.md @@ -0,0 +1,5 @@ +--- +"diagnostics-webpack-plugin": minor +--- + +Stop a TypeScript run that nothing will report any more. A check that `reportAs` keeps off the compilation runs after a watch rebuild. When it is on a worker of its own, it now stops a run of an older rebuild as soon as the next one starts, instead of finishing a check whose report would be dropped. Over 600 files, an edit made during a long check was reported after 2.3–2.6 s instead of 4.2–4.5 s, with about 2–3 CPU-seconds less work. Check adapters can stop such a run too, through an optional `cancel()` on their instance. diff --git a/README.md b/README.md index 65ea1dd2..a7ff0ea0 100644 --- a/README.md +++ b/README.md @@ -342,7 +342,12 @@ pool of workers — so the option means the same thing whatever is being checked and a check added later gets it for nothing. ESLint threads its own work from 9.34.0 under flat config; Stylelint threads none of its own, so it is pooled. TypeScript reads a program whole rather than a file at a time, so it is run on -one worker: what a thread buys it is webpack's own thread back. +one worker: what a thread buys it is webpack's own thread back. The compilers of +a `MultiCompiler` whose TypeScript checks are configured alike share that worker +and the program it holds, rather than each building and keeping one of their +own. Over 600 files, two compilers built one after the other (`parallelism: 1`) +went through a watch with one rebuild in 5.6–5.8 s and 9.8–10.4 s of CPU at +467–483 MB, against 9.7–10.3 s, 18.7–20.0 s and 803–853 MB with a worker each. Prefer `"auto"` over a count. A check sizes `"auto"` against the machine, where a fixed number can end up competing with webpack for the same cores: over three @@ -515,7 +520,13 @@ The work is moved out of the rebuild rather than off the machine: an edit arriving while such a check is still running waits for the rest of it, since a check holds the thread the build runs on. A report a newer rebuild has already overtaken is dropped, and a check never has two of its runs going at once — the -next starts when the last has reported. +next starts when the last has reported. A TypeScript check on a worker of its +own (see [`threads`](#threads)) stops such a run as soon as the next rebuild +starts, between one file and the next: what the program got through is kept, +and the next run checks the rest. An edit made while a long run is going is +then reported once its own run is done, without waiting for the rest of the +old one. Over 600 files, an edit made while an earlier one was still being +checked was reported after 2.3–2.6 s instead of 4.2–4.5 s. #### `ignoreDiagnostics` diff --git a/src/check.js b/src/check.js index be7398d5..4a049e72 100644 --- a/src/check.js +++ b/src/check.js @@ -23,17 +23,20 @@ import { toPosixPath } from "./utils.js"; /** @typedef {Dependencies & { errors?: DiagnosticError, warnings?: DiagnosticError, outputReport?: OutputReportContent }} Report */ /** @typedef {{ lint: (files: string[]) => void, keep: (files: string[]) => void, report: () => Promise, detach: (report: Promise) => void }} Runner */ /** @typedef {Map} ResultStore */ +/** @typedef {{ key: string, run: Promise, joined: boolean }} SharedRun */ /** @type {WeakMap>} */ const resultStores = new WeakMap(); -/** @type {WeakMap>>} */ +/** @type {WeakMap, cancel: () => void }>>} */ const detachedReports = new WeakMap(); // A run of a check that is still going, so that two compilers over the same // files drive the tool once. It is kept only while it runs: what comes after -// belongs to the store of the compiler that made it. -/** @type {Map>} */ +// belongs to the store of the compiler that made it. A run another compiler +// joined is one that compiler waits for too, so it is never stopped, and a run +// that is stopped is no longer there to join. +/** @type {Map} */ const sharedRuns = new Map(); // What only decides which files are checked, or how what is found is reported. @@ -106,6 +109,8 @@ function getResultStore(compilation, check) { /** * The report of a run left to finish on its own, taken so that the next run of * the same check waits for it rather than driving a second tool beside it. + * Nothing prints a report of an older compilation, so a check that can stop + * one is told to. * @param {Compilation} compilation compilation * @param {string} check a key unique to the check within the compiler * @returns {Promise | undefined} the report still to land, if there is one @@ -119,7 +124,11 @@ function takeDetachedReport(compilation, check) { reports.delete(check); - return report; + if (!report) return undefined; + + report.cancel(); + + return report.settled; } /** @@ -249,6 +258,9 @@ function createCheckRunner(key, { id, name, adapter, options }, compilation) { const linted = new Set(); // A check that cannot lint fails the same way for every batch it is given. let failed = false; + // The runs this one started that another compiler could join. + /** @type {SharedRun[]} */ + const runs = []; /** * What the check answers with, and what it says it read to answer — taken @@ -262,7 +274,11 @@ function createCheckRunner(key, { id, name, adapter, options }, compilation) { const key = sharing && `${sharing}\0${files.toSorted().join("\0")}`; const running = key ? sharedRuns.get(key) : undefined; - if (running) return running; + if (running) { + running.joined = true; + + return running.run; + } const started = instance.lintFiles(files).then((results) => ({ results, @@ -274,8 +290,10 @@ function createCheckRunner(key, { id, name, adapter, options }, compilation) { if (key) { const forget = () => sharedRuns.delete(key); + const shared = { key, run: started, joined: false }; - sharedRuns.set(key, started); + sharedRuns.set(key, shared); + runs.push(shared); started.then(forget, forget); } @@ -497,16 +515,30 @@ function createCheckRunner(key, { id, name, adapter, options }, compilation) { detachedReports.set(compilation.compiler, reports); } - reports.set( - checkKey, - report.then( + reports.set(checkKey, { + settled: report.then( () => {}, () => {}, ), - ); + cancel: () => { + if (runs.some((shared) => shared.joined)) return; + + for (const shared of runs) { + if (sharedRuns.get(shared.key) === shared) { + sharedRuns.delete(shared.key); + } + } + + pending.then((instance) => { + if (instance && instance.cancel) instance.cancel(); + }); + }, + }); } return { detach, keep, lint, report }; } +export { sharingKey }; + export default createCheckRunner; diff --git a/src/checks/index.js b/src/checks/index.js index c0d3c20c..1fc78fa1 100644 --- a/src/checks/index.js +++ b/src/checks/index.js @@ -44,6 +44,7 @@ import typescript from "./typescript.js"; * @property {((results: CheckResult[], keep: (message: Message) => boolean) => CheckResult[])=} filterResults keeps what a predicate accepts of every result, which is what `ignore` is applied through * @property {(formatter?: FormatterOption) => Promise} getFormatter loads a formatter, falling back to the tool's default one * @property {() => Promise} cleanup releases whatever the tool holds after a run + * @property {(() => void)=} cancel stops a run of an older compilation that is still going, which nothing reports any more; what it answers with is dropped */ /** diff --git a/src/checks/typescript-program.js b/src/checks/typescript-program.js index bc83cfd5..b2abe049 100644 --- a/src/checks/typescript-program.js +++ b/src/checks/typescript-program.js @@ -15,7 +15,7 @@ import { omitPluginOptions } from "../utils.js"; * hands them back, and the program that type checked them. * What a run of the check answers with: what it found, and what a watcher has * to follow for it to answer the same way again. - * @typedef {{ diagnostics: Diagnostic[], host: EXPECTED_ANY, files: string[], directories: string[], writes: string[] }} Found + * @typedef {{ diagnostics: Diagnostic[], host: EXPECTED_ANY, files: string[], directories: string[], writes: string[], cancelled?: boolean }} Found */ /** @@ -400,9 +400,10 @@ function buildSolution(ts, options, configFile, host, unrecoverable, held) { * every rebuild. * @param {Held} held what the last compilation left behind, with what its declarations said * @param {EXPECTED_ANY} program the program built this time + * @param {EXPECTED_ANY=} cancellation what tells a run nobody waits for any more to stop * @returns {Diagnostic[]} what the declarations of all its files say */ -function declarationDiagnostics(held, program) { +function declarationDiagnostics(held, program, cancellation) { // Asked before the semantic diagnostics are: once those are known the // builder has nothing left that it would check again. let reached = held.declarations.size === 0; @@ -410,9 +411,10 @@ function declarationDiagnostics(held, program) { const affected = new Set(); for ( - let next = held.program.getSemanticDiagnosticsOfNextAffectedFile(); + let next = + held.program.getSemanticDiagnosticsOfNextAffectedFile(cancellation); next; - next = held.program.getSemanticDiagnosticsOfNextAffectedFile() + next = held.program.getSemanticDiagnosticsOfNextAffectedFile(cancellation) ) { // The whole program, where a change reaches every file. if (next.affected === program) reached = true; @@ -432,7 +434,7 @@ function declarationDiagnostics(held, program) { file.fileName, !reached && known && !affected.has(file.fileName) ? known - : program.getDeclarationDiagnostics(file), + : program.getDeclarationDiagnostics(file, cancellation), ); } @@ -448,9 +450,10 @@ function declarationDiagnostics(held, program) { * @param {TypeScript} ts the loaded TypeScript * @param {Options} options options * @param {Held} held what the last compilation left behind + * @param {EXPECTED_ANY=} cancellation what tells a run nobody waits for any more to stop * @returns {Found} what it found, and what it read to find it */ -function check(ts, options, held) { +function check(ts, options, held, cancellation) { const context = String(options.context); const configFile = options.configFile || @@ -557,33 +560,63 @@ function check(ts, options, held) { global: true, ...options.diagnosticOptions, }; - const diagnostics = [ - ...unrecoverable, - ...ts.sortAndDeduplicateDiagnostics([ - ...program.getConfigFileParsingDiagnostics(), - ...program.getOptionsDiagnostics(), - ...(kinds.syntactic ? held.program.getSyntacticDiagnostics() : []), - ...(kinds.global ? program.getGlobalDiagnostics() : []), - ...(kinds.declaration && - (parsed.options.declaration || parsed.options.composite) - ? declarationDiagnostics(held, program) - : []), - ...(kinds.semantic ? held.program.getSemanticDiagnostics() : []), - ]), - ]; - + // The config file decides which files the program holds, so reading it + // again is what a change to it takes. + const files = [configFile, ...extended, ...parsed.fileNames]; + // What `include` covers, which is where a file the program has never held + // can appear. + const directories = Object.keys(parsed.wildcardDirectories || {}); /** @type {string[]} */ - const writes = []; + const writes = overrides.noEmit + ? [] + : [ + parsed.options.outDir, + parsed.options.declarationDir, + parsed.options.tsBuildInfoFile, + ] + .filter(Boolean) + .map(String); + /** @type {Diagnostic[]} */ + let diagnostics; - if (!overrides.noEmit) { - for (const written of [ - parsed.options.outDir, - parsed.options.declarationDir, - parsed.options.tsBuildInfoFile, - ]) { - if (written) writes.push(String(written)); - } + try { + diagnostics = [ + ...unrecoverable, + ...ts.sortAndDeduplicateDiagnostics([ + ...program.getConfigFileParsingDiagnostics(), + ...program.getOptionsDiagnostics(), + ...(kinds.syntactic + ? held.program.getSyntacticDiagnostics(undefined, cancellation) + : []), + ...(kinds.global ? program.getGlobalDiagnostics(cancellation) : []), + ...(kinds.declaration && + (parsed.options.declaration || parsed.options.composite) + ? declarationDiagnostics(held, program, cancellation) + : []), + ...(kinds.semantic + ? held.program.getSemanticDiagnostics(undefined, cancellation) + : []), + ]), + ]; + } catch (err) { + if (!(err instanceof ts.OperationCanceledException)) throw err; + + // The builder keeps what it got through and checks the rest next time. + // Which files the declarations were drained of is lost with the run, so + // the next one works all of them out again. + held.declarations = new Map(); + return { + cancelled: true, + diagnostics: [], + host, + files, + directories, + writes, + }; + } + + if (!overrides.noEmit) { held.program.emit( undefined, ( @@ -601,17 +634,7 @@ function check(ts, options, held) { if (!held.seen.has(file)) held.files.delete(file); } - return { - diagnostics, - host, - // The config file decides which files the program holds, so reading it - // again is what a change to it takes. - files: [configFile, ...extended, ...parsed.fileNames], - // What `include` covers, which is where a file the program has never held - // can appear. - directories: Object.keys(parsed.wildcardDirectories || {}), - writes, - }; + return { diagnostics, host, files, directories, writes }; } /** * What a diagnostic says, as data: a `Diagnostic` holds the source file it was diff --git a/src/checks/typescript-worker.js b/src/checks/typescript-worker.js index 7ffc5bfa..e64c4169 100644 --- a/src/checks/typescript-worker.js +++ b/src/checks/typescript-worker.js @@ -11,6 +11,9 @@ import { check, describe, getHeld } from "./typescript-program.js"; /** @type {Options} */ let checkOptions; +/** @type {Int32Array} */ +let stopped; + /** @type {Promise | null} */ let loading = null; @@ -36,21 +39,34 @@ async function getTypeScript() { /** * @param {Options} options the options the check was configured with + * @param {Int32Array} stop the runs the plugin no longer waits for, each in the slot its number falls in, shared with it */ -function setup(options) { +function setup(options, stop) { checkOptions = options; + stopped = stop; loading = null; } /** * Runs the check and answers with what crosses back: a `Diagnostic` holds the * source file it was found in, so each one is described and formatted here. + * @param {string[]} files nothing: the program is the config file's + * @param {number} run which run of the check this is * @returns {Promise} what it found, and what it read to find it */ -async function lintFiles() { +async function lintFiles(files, run) { const ts = await getTypeScript(); const held = getHeld(compiler, "worker"); - const found = check(ts, checkOptions, held); + // TypeScript asks between one file and the next, which is where a run + // nobody waits for any more stops. + const requested = () => Atomics.load(stopped, run % stopped.length) === run; + const cancellation = { + isCancellationRequested: requested, + throwIfCancellationRequested() { + if (requested()) throw new ts.OperationCanceledException(); + }, + }; + const found = check(ts, checkOptions, held, cancellation); // Each on its own, so that what is left after `ignoreDiagnostics` can be // printed without asking the worker again. Each block carries the newline // TypeScript ends it with, which is what separates one from the next, so diff --git a/src/checks/typescript.js b/src/checks/typescript.js index b819cb5a..947b54bd 100644 --- a/src/checks/typescript.js +++ b/src/checks/typescript.js @@ -3,6 +3,7 @@ import { createRequire } from "node:module"; +import { sharingKey } from "../check.js"; import { countThreads, createSolo } from "../threads.js"; import { importFrom } from "../utils.js"; @@ -26,63 +27,85 @@ export { getTypeScriptOptions } from "./typescript-program.js"; const nodeRequire = createRequire(import.meta.url); -/** @type {WeakMap>} */ -const workersByCompiler = new WeakMap(); +/** + * A worker and the compilers whose checks run on it. + * @typedef {{ worker: Solo, users: Set }} Shared + */ /** + * Workers by what their program is built from rather than by compiler: the + * compilers of a `MultiCompiler` checking the same project under the same + * options share one program instead of holding one each. + * @type {Map} + */ +const workers = new Map(); + +/** @type {WeakMap>} */ +const tapped = new WeakMap(); + +/** + * Lets go of a worker for a compiler, and ends it once no compiler uses it. * @param {EXPECTED_ANY} compiler the compiler the check ran for - * @param {string} id a key unique to the check within that compiler + * @param {string} key what the worker's program is built from + * @returns {Promise} when the worker is done with, if it is */ -function dropWorker(compiler, id) { - const checks = workersByCompiler.get(compiler); +async function release(compiler, key) { + const shared = workers.get(key); + + if (!shared || !shared.users.delete(compiler)) return; + if (shared.users.size > 0) return; - if (checks) checks.delete(id); + workers.delete(key); + + await shared.worker.end(); } /** - * The one worker this check runs on, kept for the compiler it runs for so that - * the program it holds outlives the compilation that built it. + * The one worker this check runs on, kept for as long as a compiler uses it so + * that the program it holds outlives the compilation that built it. * @param {EXPECTED_ANY} compiler the compiler the check runs for - * @param {string} id a key unique to the check within that compiler + * @param {string} key what the worker's program is built from * @param {Options} options the options the check was configured with * @returns {Solo | null} the worker, or nothing when it cannot be handed over */ -function getWorker(compiler, id, options) { - let checks = workersByCompiler.get(compiler); - - if (!checks) { - checks = new Map(); - workersByCompiler.set(compiler, checks); - } - - let worker = checks.get(id); +function getWorker(compiler, key, options) { + let shared = workers.get(key); - if (!worker) { + if (!shared) { const started = createSolo(nodeRequire.resolve("./typescript-worker.js"), [ options, ]); if (!started) return null; - worker = started; - checks.set(id, worker); + shared = { worker: started, users: new Set() }; + workers.set(key, shared); + } - // The program is the worker's, so in a watch it lives as long as the - // compiler does rather than as long as one of its builds. A watch that is - // closed without the compiler ends it too, or the thread outlives the work. - const stop = async () => { - dropWorker(compiler, id); + shared.users.add(compiler); - await /** @type {Solo} */ (worker).end(); - }; + let keys = tapped.get(compiler); - compiler.hooks.shutdown.tapPromise("DiagnosticsPlugin", stop); + if (!keys) { + keys = new Set(); + tapped.set(compiler, keys); + } + + // The program is the worker's, so in a watch it lives as long as the + // compiler does rather than as long as one of its builds. A watch that is + // closed without the compiler lets go of it too, or the thread outlives the + // work. + if (!keys.has(key)) { + keys.add(key); + compiler.hooks.shutdown.tapPromise("DiagnosticsPlugin", () => + release(compiler, key), + ); compiler.hooks.watchClose.tap("DiagnosticsPlugin", () => { - stop(); + release(compiler, key); }); } - return worker; + return shared.worker; } /** @@ -107,8 +130,10 @@ async function create({ key, id: entry, options, compilation }) { // The program is this entry's: another check of the same tool reads the same // config file under options of its own. const id = `${key}\0${entry}`; + // Two checks that would build the same program share the worker holding it. + const program = sharingKey("typescript", options) || id; const worker = canThread(options) - ? getWorker(compilation.compiler, id, options) + ? getWorker(compilation.compiler, program, options) : null; /** @type {TypeScript} */ @@ -134,6 +159,10 @@ async function create({ key, id: entry, options, compilation }) { // The program is the whole project, so it is built once however many batches // of files the plugin hands over. let checked = false; + // The worker's answer to this compilation's run, which is what a newer + // compilation stops when it takes over. + /** @type {Promise | undefined} */ + let answer; /** * @param {EXPECTED_ANY} one a diagnostic, or what a worker said of one @@ -149,7 +178,9 @@ async function create({ key, id: entry, options, compilation }) { checked = true; if (worker) { - const found = await worker.lintFiles(); + answer = worker.lintFiles(); + + const found = await answer; read = found.files; directories = found.directories; @@ -169,6 +200,11 @@ async function create({ key, id: entry, options, compilation }) { return found.diagnostics; }, + // A run of an older compilation is left to finish on the worker while + // webpack builds the next one, which waits for it before checking again. + cancel() { + if (worker && answer) worker.cancel(answer); + }, readFiles() { return read; }, @@ -228,9 +264,7 @@ async function create({ key, id: entry, options, compilation }) { async cleanup() { if (!worker || compilation.compiler.watchMode) return; - dropWorker(compilation.compiler, id); - - await worker.end(); + await release(compilation.compiler, program); }, }; } diff --git a/src/threads.js b/src/threads.js index 14da878d..0304fc17 100644 --- a/src/threads.js +++ b/src/threads.js @@ -10,7 +10,11 @@ import { Worker as JestWorker } from "jest-worker"; /** @typedef {(files: string[]) => Promise} LintTask */ /** @typedef {JestWorker & { lintFiles: LintTask }} Worker */ /** @typedef {{ lintFiles: LintTask, end: () => Promise }} Pool */ -/** @typedef {{ lintFiles: () => Promise, end: () => Promise }} Solo */ +/** @typedef {{ lintFiles: () => Promise, cancel: (answer: Promise) => void, end: () => Promise }} Solo */ + +// More runs than these are never queued on one worker at once: each compiler +// sharing it waits for its own last run before it starts the next. +const STOP_SLOTS = 64; /** * How many threads the user asked for, as a count. A check is spread over one @@ -107,20 +111,42 @@ function createPool(source, size, setupArgs) { function createSolo(source, setupArgs) { if (!isTransferable(setupArgs)) return null; + // Shared with the worker rather than sent to it: a message would wait behind + // the run it is meant to stop. Each run is numbered, and one nobody waits for + // any more finds its number in the slot its number falls in. A worker can be + // shared by checks of several compilers, so stopping one run leaves the rest. + const stopped = new Int32Array(new SharedArrayBuffer(4 * STOP_SLOTS)); + /** @type {WeakMap, number>} */ + const runs = new WeakMap(); + let issued = 0; + let worker = /** @type {Worker | null} */ ( new JestWorker(source, { enableWorkerThreads: true, numWorkers: 1, - setupArgs, + setupArgs: [...setupArgs, stopped], }) ); return { - lintFiles: async () => { + // Not async: the promise handed back is the one `cancel` is given. + lintFiles: () => { /* istanbul ignore next */ - if (!worker) return undefined; + if (!worker) return Promise.resolve(undefined); + + issued += 1; + + const answer = /** @type {EXPECTED_ANY} */ (worker).lintFiles([], issued); + + runs.set(answer, issued); + + return answer; + }, + // Whether the run has started or is still queued. + cancel: (answer) => { + const run = runs.get(answer); - return /** @type {Worker} */ (worker).lintFiles([]); + if (run !== undefined) Atomics.store(stopped, run % STOP_SLOTS, run); }, end: async () => { /* istanbul ignore else */ diff --git a/test/store.test.js b/test/store.test.js index e9059584..09b7f53a 100644 --- a/test/store.test.js +++ b/test/store.test.js @@ -107,3 +107,103 @@ describe("store", () => { assert.deepStrictEqual(await reported(second), []); }); }); + +describe("cancel", () => { + /** + * @param {string[]} cancelled where each stopped run is written down + * @returns {EXPECTED_ANY} an adapter whose runs can be stopped + */ + function adapterCancelled(cancelled) { + const adapter = adapterFinding([]); + const counted = { + ...adapter, + runs: 0, + create: async () => { + const instance = await adapter.create(); + + return { + ...instance, + // Slow enough for a second compiler to join it. + lintFiles: async (/** @type {string[]} */ files) => { + counted.runs += 1; + + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + + return instance.lintFiles(files); + }, + cancel: () => cancelled.push("cancelled"), + }; + }, + }; + + return counted; + } + + /** + * @param {EXPECTED_ANY} runner the runner whose report nobody waits for + * @returns {Promise} once the runner has been left to finish + */ + async function leave(runner) { + runner.detach(runner.report()); + } + + it("should stop a run of an older compilation nothing else waits for", async () => { + const cancelled = []; + const adapter = adapterCancelled(cancelled); + const compiler = { outputPath: "/out" }; + const first = runnerFor(adapter, compiler); + + first.lint(["a.css"]); + await leave(first); + await reported(runnerFor(adapter, compiler)); + + assert.deepStrictEqual(cancelled, ["cancelled"]); + }); + + it("should leave running a run another compiler joined", async () => { + const cancelled = []; + const adapter = adapterCancelled(cancelled); + const one = { outputPath: "/one" }; + const first = runnerFor(adapter, one); + const other = runnerFor(adapter, { outputPath: "/two" }); + + // Both lint the same files at once, so the second joins the first's run. + first.lint(["a.css"]); + other.lint(["a.css"]); + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + await leave(first); + await reported(runnerFor(adapter, one)); + await reported(other); + + assert.deepStrictEqual(cancelled, []); + }); + + it("should not let a compiler join a run that was stopped", async () => { + const cancelled = []; + const adapter = adapterCancelled(cancelled); + const one = { outputPath: "/one" }; + const first = runnerFor(adapter, one); + + first.lint(["a.css"]); + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + await leave(first); + + // The next compilation stops the run while it is still going. + const next = runnerFor(adapter, one); + const other = runnerFor(adapter, { outputPath: "/two" }); + + other.lint(["a.css"]); + await Promise.all([reported(next), reported(other)]); + + assert.deepStrictEqual(cancelled, ["cancelled"]); + // The other compiler ran the files itself rather than taking the + // stopped run's answer. + assert.strictEqual(adapter.runs, 2); + }); +}); diff --git a/test/typescript/cancel.test.js b/test/typescript/cancel.test.js new file mode 100644 index 00000000..3301cddc --- /dev/null +++ b/test/typescript/cancel.test.js @@ -0,0 +1,149 @@ +import assert from "node:assert/strict"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, it } from "node:test"; + +import { countThreads } from "../../src/threads.js"; + +import pack from "./utils/pack.js"; + +const typescriptPath = join(import.meta.dirname, "mock/typescript-stall"); + +const fixture = join(import.meta.dirname, "fixtures", "cancel"); +const checked = join(fixture, "checked.ts"); +const configFile = join(fixture, "tsconfig.json"); +const trigger = join(fixture, "trigger.js"); + +describe("cancel", () => { + let watch; + let outcome; + + afterEach(() => { + if (watch) watch.close(); + + rmSync(checked, { force: true }); + rmSync(configFile, { force: true }); + rmSync(trigger, { force: true }); + rmSync(outcome, { force: true }); + }); + + // The check only runs on a worker of its own where there are threads to + // spare, and only there can a run be stopped while it goes on. + it( + "should stop a run nobody waits for once the next build starts", + { skip: countThreads("auto") < 2 }, + (t, done) => { + outcome = join( + mkdtempSync(join(tmpdir(), "typescript-stall-")), + "outcome", + ); + // Read by the stalling TypeScript, on the worker the check runs on. + process.env.TYPESCRIPT_STALL_OUTCOME = outcome; + + writeFileSync( + configFile, + `${JSON.stringify({ + compilerOptions: { strict: true, skipLibCheck: true }, + include: ["*.ts"], + })}\n`, + ); + writeFileSync(trigger, "export const trigger = 1;\n"); + writeFileSync(checked, "export const fine = 1;\n"); + + /** @type {string[]} */ + const logged = []; + const quiet = () => {}; + const started = Date.now(); + let step = 0; + + const compiler = pack( + "cancel", + // Logged rather than reported, which is what leaves a run to finish + // after the build it was started for. + { threads: "auto", typescriptPath, reportAs: "log" }, + { + infrastructureLogging: { + level: "error", + console: { + error: (/** @type {string} */ message) => { + logged.push(message); + + if (!/TS2322/u.test(message)) return; + + assert.strictEqual(readFileSync(outcome, "utf8"), "cancelled"); + // What the stale run found is never printed. + assert.strictEqual(logged.length, 1); + assert.ok(Date.now() - started < 30000); + done(); + }, + warn: quiet, + info: quiet, + log: quiet, + debug: quiet, + trace: quiet, + group: quiet, + groupEnd: quiet, + groupCollapsed: quiet, + status: quiet, + clear: quiet, + profile: quiet, + profileEnd: quiet, + }, + }, + }, + ); + + watch = compiler.watch({}, (err, stats) => { + assert.strictEqual(err, null); + + const changed = [...(stats.compilation.compiler.modifiedFiles || [])]; + + if (step === 0) { + // A rebuild webpack starts for what was written before it watched. + if (changed.length > 0) return; + + step = 1; + // The run this build leaves behind stalls until it is cancelled. + writeFileSync(checked, "// stall\nexport const fine = 2;\n"); + + return; + } + + if (step === 1) { + // Webpack names the folder that holds it too. + const others = changed.filter((file) => file !== fixture); + + if (others.length !== 1 || others[0] !== checked) { + // A rebuild for what was written before the watch, late: the + // edit is made again so that a run is left with it. + writeFileSync( + checked, + `// stall\nexport const fine = ${Date.now()};\n`, + ); + + return; + } + + step = 2; + + // The worker reads the file when its run gets to it, so the next + // edit waits until that run is stuck on this one. + const stuck = setInterval(() => { + if (!existsSync(outcome)) return; + + clearInterval(stuck); + assert.strictEqual(readFileSync(outcome, "utf8"), "stalling"); + writeFileSync(checked, "export const wrong: string = 42;\n"); + }, 10); + } + }); + }, + ); +}); diff --git a/test/typescript/fixtures/cancel/index.js b/test/typescript/fixtures/cancel/index.js new file mode 100644 index 00000000..aa624046 --- /dev/null +++ b/test/typescript/fixtures/cancel/index.js @@ -0,0 +1 @@ +import "./trigger.js"; diff --git a/test/typescript/mock/typescript-counter/index.js b/test/typescript/mock/typescript-counter/index.js new file mode 100644 index 00000000..9f64f866 --- /dev/null +++ b/test/typescript/mock/typescript-counter/index.js @@ -0,0 +1,25 @@ +// Real TypeScript, writing down whether each program was built from nothing or +// from the one before it, wherever the check runs. +const { appendFileSync } = require("node:fs"); + +const typescript = require("typescript"); + +const build = typescript.createSemanticDiagnosticsBuilderProgram; + +module.exports = { + ...typescript, + createSemanticDiagnosticsBuilderProgram( + rootNames, + options, + host, + old, + ...rest + ) { + appendFileSync( + process.env.TYPESCRIPT_COUNTER_FILE, + old ? "reused\n" : "fresh\n", + ); + + return build(rootNames, options, host, old, ...rest); + }, +}; diff --git a/test/typescript/mock/typescript-stall/index.js b/test/typescript/mock/typescript-stall/index.js new file mode 100644 index 00000000..d35035a9 --- /dev/null +++ b/test/typescript/mock/typescript-stall/index.js @@ -0,0 +1,50 @@ +// Real TypeScript, except that a program holding a file marked to stall keeps +// checking it until the run is cancelled, and says which way it ended. +const { writeFileSync } = require("node:fs"); + +const typescript = require("typescript"); + +const LIMIT = 30000; +const build = typescript.createSemanticDiagnosticsBuilderProgram; +const pause = new Int32Array(new SharedArrayBuffer(4)); + +/** + * @param {import("typescript").CancellationToken=} token what a run is cancelled through + */ +function stall(token) { + const started = Date.now(); + + writeFileSync(process.env.TYPESCRIPT_STALL_OUTCOME, "stalling"); + + while (Date.now() - started < LIMIT) { + if (token && token.isCancellationRequested()) { + writeFileSync(process.env.TYPESCRIPT_STALL_OUTCOME, "cancelled"); + token.throwIfCancellationRequested(); + } + + Atomics.wait(pause, 0, 0, 5); + } + + writeFileSync(process.env.TYPESCRIPT_STALL_OUTCOME, "timed out"); +} + +module.exports = { + ...typescript, + createSemanticDiagnosticsBuilderProgram(...args) { + const builder = build(...args); + const check = builder.getSemanticDiagnostics.bind(builder); + + builder.getSemanticDiagnostics = (file, token) => { + const stalls = builder + .getProgram() + .getSourceFiles() + .some((source) => source.text.includes("// stall")); + + if (stalls) stall(token); + + return check(file, token); + }; + + return builder; + }, +}; diff --git a/test/typescript/share.test.js b/test/typescript/share.test.js new file mode 100644 index 00000000..78a8824f --- /dev/null +++ b/test/typescript/share.test.js @@ -0,0 +1,56 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, it } from "node:test"; + +import webpack from "webpack"; + +import { countThreads } from "../../src/threads.js"; + +import conf from "./utils/conf.js"; + +const typescriptPath = join(import.meta.dirname, "mock/typescript-counter"); + +describe("share", () => { + // Only a worker outlives the compilation that started it, which is what a + // second compiler can find the program on. + it( + "should build one program for the compilers checking the same project", + { skip: countThreads("auto") < 2 }, + (t, done) => { + const counter = join( + mkdtempSync(join(tmpdir(), "typescript-counter-")), + "programs", + ); + + // Read by the counting TypeScript, on the worker the check runs on. + process.env.TYPESCRIPT_COUNTER_FILE = counter; + + const configs = ["one", "two"].map((name) => { + const config = conf("good", { threads: "auto", typescriptPath }); + + return { + ...config, + name, + output: { path: join(config.output.path, name) }, + }; + }); + + // One after the other, so that the second does not join the first's run. + configs.parallelism = 1; + + const watch = webpack(configs).watch({}, (err) => { + assert.strictEqual(err, null); + + watch.close(() => { + assert.deepStrictEqual( + readFileSync(counter, "utf8").trim().split("\n"), + ["fresh", "reused"], + ); + done(); + }); + }); + }, + ); +}); diff --git a/types/check.d.ts b/types/check.d.ts index ea7125ad..8449e39e 100644 --- a/types/check.d.ts +++ b/types/check.d.ts @@ -33,6 +33,22 @@ export type Runner = { detach: (report: Promise) => void; }; export type ResultStore = Map; +export type SharedRun = { + key: string; + run: Promise; + joined: boolean; +}; +/** + * What tells one check's run of a tool from another's, so that only two runs + * that would do the same work are shared. + * @param {string} name the name of the check + * @param {EnabledCheck["options"]} options the options it runs under + * @returns {string | undefined} the key, or nothing when they cannot be compared + */ +export function sharingKey( + name: string, + options: EnabledCheck["options"], +): string | undefined; /** * Creates the runner synchronously so that the compilation hooks are tapped * before webpack starts building modules, whatever the tool takes to load. A diff --git a/types/checks/index.d.ts b/types/checks/index.d.ts index 513ad84b..9398736c 100644 --- a/types/checks/index.d.ts +++ b/types/checks/index.d.ts @@ -86,6 +86,10 @@ export type CheckInstance = { * releases whatever the tool holds after a run */ cleanup: () => Promise; + /** + * stops a run of an older compilation that is still going, which nothing reports any more; what it answers with is dropped + */ + cancel?: (() => void) | undefined; }; /** * What a check shipped outside this package has to provide; everything the @@ -241,6 +245,7 @@ export type CheckAdapter = { * @property {((results: CheckResult[], keep: (message: Message) => boolean) => CheckResult[])=} filterResults keeps what a predicate accepts of every result, which is what `ignore` is applied through * @property {(formatter?: FormatterOption) => Promise} getFormatter loads a formatter, falling back to the tool's default one * @property {() => Promise} cleanup releases whatever the tool holds after a run + * @property {(() => void)=} cancel stops a run of an older compilation that is still going, which nothing reports any more; what it answers with is dropped */ /** * What a check shipped outside this package has to provide; everything the diff --git a/types/checks/typescript-program.d.ts b/types/checks/typescript-program.d.ts index 0f2bbe4f..cd75c34e 100644 --- a/types/checks/typescript-program.d.ts +++ b/types/checks/typescript-program.d.ts @@ -14,6 +14,7 @@ export type Found = { files: string[]; directories: string[]; writes: string[]; + cancelled?: boolean; }; export type Held = { signature: string; @@ -39,9 +40,15 @@ export type Held = { * @param {TypeScript} ts the loaded TypeScript * @param {Options} options options * @param {Held} held what the last compilation left behind + * @param {EXPECTED_ANY=} cancellation what tells a run nobody waits for any more to stop * @returns {Found} what it found, and what it read to find it */ -export function check(ts: TypeScript, options: Options, held: Held): Found; +export function check( + ts: TypeScript, + options: Options, + held: Held, + cancellation?: EXPECTED_ANY | undefined, +): Found; /** * What a diagnostic says, as data: a `Diagnostic` holds the source file it was * found in, so this is what can be read once it has left the check. diff --git a/types/checks/typescript-worker.d.ts b/types/checks/typescript-worker.d.ts index c2bbbf9c..d0abbeef 100644 --- a/types/checks/typescript-worker.d.ts +++ b/types/checks/typescript-worker.d.ts @@ -4,10 +4,13 @@ export type TypeScript = EXPECTED_ANY; /** * Runs the check and answers with what crosses back: a `Diagnostic` holds the * source file it was found in, so each one is described and formatted here. + * @param {string[]} files nothing: the program is the config file's + * @param {number} run which run of the check this is * @returns {Promise} what it found, and what it read to find it */ -export function lintFiles(): Promise; +export function lintFiles(files: string[], run: number): Promise; /** * @param {Options} options the options the check was configured with + * @param {Int32Array} stop the runs the plugin no longer waits for, each in the slot its number falls in, shared with it */ -export function setup(options: Options): void; +export function setup(options: Options, stop: Int32Array): void; diff --git a/types/checks/typescript.d.ts b/types/checks/typescript.d.ts index df7dd7f6..18c85530 100644 --- a/types/checks/typescript.d.ts +++ b/types/checks/typescript.d.ts @@ -30,6 +30,13 @@ export type Described = { text: string; formatted: string; }; +/** + * A worker and the compilers whose checks run on it. + */ +export type Shared = { + worker: Solo; + users: Set; +}; /** * @param {CheckContext} context check context * @returns {Promise} typescript check diff --git a/types/threads.d.ts b/types/threads.d.ts index 3252c9da..5bdef1c7 100644 --- a/types/threads.d.ts +++ b/types/threads.d.ts @@ -1,3 +1,4 @@ +export type EXPECTED_ANY = any; export type CheckOptions = import("./options.js").CheckOptions; export type CheckResult = import("./checks/index.js").CheckResult; export type LintTask = (files: string[]) => Promise; @@ -10,15 +11,9 @@ export type Pool = { }; export type Solo = { lintFiles: () => Promise; + cancel: (answer: Promise) => void; end: () => Promise; }; -export type EXPECTED_ANY = any; -/** @typedef {import("./options.js").CheckOptions} CheckOptions */ -/** @typedef {import("./checks/index.js").CheckResult} CheckResult */ -/** @typedef {(files: string[]) => Promise} LintTask */ -/** @typedef {JestWorker & { lintFiles: LintTask }} Worker */ -/** @typedef {{ lintFiles: LintTask, end: () => Promise }} Pool */ -/** @typedef {{ lintFiles: () => Promise, end: () => Promise }} Solo */ /** * How many threads the user asked for, as a count. A check is spread over one * fewer thread than the machine has, leaving webpack the one it builds on, and