diff --git a/README.md b/README.md index 325773bb9..abf035b21 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ scriptc is experimental and targets macOS, Linux, Windows, and WebAssembly via W ## Installation -The compiler requires Node.js 20 or newer and clang. The executables it produces do not require Node. +The compiler requires Node.js 24 or newer and clang. The executables it produces do not require Node. ```console $ npm install -g scriptc diff --git a/packages/cli/package.json b/packages/cli/package.json index 3f8726c26..89ad7661b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -11,13 +11,13 @@ }, "type": "module", "bin": { - "scriptc": "dist/main.js" + "scriptc": "dist/bootstrap.js" }, "files": [ "dist" ], "engines": { - "node": ">=20" + "node": ">=24" }, "scripts": { "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json" diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts new file mode 100644 index 000000000..32f9b0dad --- /dev/null +++ b/packages/cli/src/bootstrap.ts @@ -0,0 +1,16 @@ +#!/usr/bin/env node + +import { enableCompileCache } from "node:module"; + +// Node 24 can persist V8's compiled module bytecode. scriptc's CLI imports +// the compiler and its lowering/backend graph before handling any command, so +// enabling this in the tiny bootstrap avoids reparsing that graph on every +// edit/build invocation. +try { + enableCompileCache(); +} catch { + // Bytecode caching is an optimization boundary. A read-only temp directory + // must never prevent the compiler from running. +} + +await import("./main.js"); diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 619bfb6ea..127d7e27f 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -1,4 +1,3 @@ -#!/usr/bin/env node import { spawn } from "node:child_process"; import { existsSync, readFileSync, rmSync, statSync } from "node:fs"; import { basename, dirname, join, resolve } from "node:path"; diff --git a/packages/compiler/src/backend/cc-cache.test.ts b/packages/compiler/src/backend/cc-cache.test.ts index fdebf6f78..241a126ed 100644 --- a/packages/compiler/src/backend/cc-cache.test.ts +++ b/packages/compiler/src/backend/cc-cache.test.ts @@ -1,7 +1,7 @@ import { execFileSync, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { existsSync } from "node:fs"; -import { chmod, cp, mkdir, mkdtemp, readdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises"; +import { chmod, cp, mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { delimiter, join } from "node:path"; import { afterAll, afterEach, expect, test as vitestTest } from "vitest"; @@ -268,6 +268,125 @@ test("the runtime fingerprint includes the textually included Ryū sources", asy expect(await runtimeFingerprint(rtDir)).not.toBe(first); }); +test("the runtime fingerprint includes newly added nested headers", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-fingerprint-nested-")); + scratch.push(dir); + const rtDir = join(dir, "src"); + const ryuDir = join(dir, "vendor", "ryu"); + await Promise.all([ + mkdir(join(rtDir, "sys"), { recursive: true }), + mkdir(ryuDir, { recursive: true }), + ]); + await writeFile(join(rtDir, "scr_number.c"), "int scriptc_probe;\n"); + const first = await runtimeFingerprint(rtDir); + + // A new file below an existing include root can win resolution without + // changing any path selected by the previous dependency scan. + await writeFile(join(rtDir, "sys", "types.h"), "#define SCRIPTC_SHADOW 1\n"); + expect(await runtimeFingerprint(rtDir)).not.toBe(first); +}); + +test.skipIf(process.platform === "win32")( + "the runtime fingerprint follows symlinked source files", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-fingerprint-symlink-")); + scratch.push(dir); + const rtDir = join(dir, "src"); + const ryuDir = join(dir, "vendor", "ryu"); + const target = join(dir, "runtime-target.h"); + await Promise.all([ + mkdir(rtDir, { recursive: true }), + mkdir(ryuDir, { recursive: true }), + writeFile(target, "#define SCRIPTC_SYMLINK 1\n"), + ]); + await symlink(target, join(rtDir, "linked.h")); + const first = await runtimeFingerprint(rtDir); + + await writeFile(target, "#define SCRIPTC_SYMLINK 2\n"); + expect(await runtimeFingerprint(rtDir)).not.toBe(first); + }, +); + +test.skipIf(process.platform === "win32")( + "new nested runtime headers invalidate complete and output-local artifacts", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-shadow-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const fakeRuntime = join(dir, "runtime", "src"); + const projectDir = join(dir, "project"); + const originalRuntime = runtimeSrcDir(); + const cPath = join(projectDir, "program.c"); + const firstOut = join(projectDir, "first"); + const crossOutput = join(projectDir, "cross-output"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const oldRuntimeDir = process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"]; + + try { + await Promise.all([ + cp(originalRuntime, fakeRuntime, { recursive: true }), + mkdir(projectDir), + mkdir(join(dir, "runtime", "vendor"), { recursive: true }).then(() => + cp( + join(originalRuntime, "..", "vendor", "ryu"), + join(dir, "runtime", "vendor", "ryu"), + { recursive: true }, + ) + ), + ]); + await mkdir(join(fakeRuntime, "sys")); + const numberSource = join(fakeRuntime, "scr_number.c"); + await writeFile( + numberSource, + `${await readFile(numberSource, "utf8")}\n` + + "#ifndef SCRIPTC_SHADOW_VALUE\n#define SCRIPTC_SHADOW_VALUE 1\n#endif\n" + + "int scriptc_shadow_value(void) { return SCRIPTC_SHADOW_VALUE; }\n", + ); + await writeFile( + cPath, + "#include \nint scriptc_shadow_value(void);\n" + + 'int main(void) { printf("%d\\n", scriptc_shadow_value()); return 0; }\n', + ); + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"] = fakeRuntime; + delete process.env["SCRIPTC_NO_CACHE"]; + + await compileC({ cPath, outPath: firstOut, cacheIdentity: "scriptc-generated-v1" }); + expect(execFileSync(firstOut, { encoding: "utf8" }).trim()).toBe("1"); + + // scr_runtime.h includes . This newly created header wins + // the existing -I runtime search without changing any dependency path + // selected during the first build. + await writeFile( + join(fakeRuntime, "sys", "types.h"), + "#include_next \n#define SCRIPTC_SHADOW_VALUE 2\n", + ); + + // A new output path bypasses the output-local stamp and probes the + // cross-output complete-artifact cache directly. + await compileC({ + cPath, + outPath: crossOutput, + cacheIdentity: "scriptc-generated-v1", + }); + expect(execFileSync(crossOutput, { encoding: "utf8" }).trim()).toBe("2"); + + // The original output has an output-local stamp from the first build; + // it must invalidate for the same namespace change. + await compileC({ cPath, outPath: firstOut, cacheIdentity: "scriptc-generated-v1" }); + expect(execFileSync(firstOut, { encoding: "utf8" }).trim()).toBe("2"); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + if (oldRuntimeDir === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"]; + else process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"] = oldRuntimeDir; + } + }, +); + test("implicit dependency seeds include separately compiled vendor system headers", async () => { const includes = await implicitDependencyProbeIncludes(runtimeSrcDir()); // zlib's crc32.c is compiled independently and is not textually reachable @@ -2351,6 +2470,249 @@ test("system libraries relink after an in-place rebuild while runtime objects re } }); +test("frontend-generated same-output builds no-op only while output and dependency stamp are valid", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-local-artifact-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const cPath = join(dir, "program.c"); + const outPath = join(dir, "program"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const oldUmask = process.umask(); + + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + await writeFile(cPath, '#include \nint main(void) { puts("valid"); return 0; }\n'); + + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + const stampPath = join( + cacheRoot, + "local", + createHash("sha256").update(outPath).digest("hex"), + ); + const stamp = JSON.parse(await readFile(stampPath, "utf8")) as { + dependencies: { path: string; kind: "file" | "directory"; size: number; mtimeMs: number; ctimeMs: number }[]; + integrity: string; + }; + expect(stamp.dependencies.length).toBeGreaterThan(0); + + const pinnedTime = new Date("2001-01-01T00:00:00.000Z"); + await utimes(outPath, pinnedTime, pinnedTime); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect((await stat(outPath)).mtimeMs).toBe(pinnedTime.getTime()); + + process.umask(0o077); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect((await stat(outPath)).mode & 0o777).toBe(0o700); + process.umask(oldUmask); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect((await stat(outPath)).mode & 0o777).toBe(0o777 & ~oldUmask); + + // Generated TU bytes join the key: an actual source edit must replace the + // output even though its path, runtime, and toolchain are unchanged. + await writeFile(cPath, '#include \nint main(void) { puts("edited"); return 0; }\n'); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect(execFileSync(outPath, { encoding: "utf8" }).trim()).toBe("edited"); + await writeFile(cPath, '#include \nint main(void) { puts("valid"); return 0; }\n'); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + + // A damaged output cannot no-op even when every build input is unchanged. + await writeFile(outPath, "damaged\n"); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect(execFileSync(outPath, { encoding: "utf8" }).trim()).toBe("valid"); + + // The stamp is disposable cache data. A changed SDK/linker dependency + // metadata record must take the strict CAS path, which reinstalls output. + const damagedStamp = JSON.parse(await readFile(stampPath, "utf8")) as typeof stamp; + damagedStamp.dependencies[0]!.size++; + await writeFile(stampPath, `${JSON.stringify(damagedStamp)}\n`); + await utimes(outPath, pinnedTime, pinnedTime); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect((await stat(outPath)).mtimeMs).toBeGreaterThan(pinnedTime.getTime()); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + process.umask(oldUmask); + } +}); + +test.skipIf(process.platform === "win32")( + "output-local hits follow symlinked header targets", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-local-artifact-symlink-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const cPath = join(dir, "program.c"); + const header = join(dir, "value.h"); + const target = join(dir, "target.h"); + const outPath = join(dir, "program"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + await writeFile(target, '#define VALUE "one"\n'); + await symlink(target, header); + await writeFile( + cPath, + '#include \n#include "value.h"\nint main(void) { puts(VALUE); return 0; }\n', + ); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect(execFileSync(outPath, { encoding: "utf8" }).trim()).toBe("one"); + + // clang's dependency file names the symlink path. The local stamp must + // also follow that path and invalidate when only the target changes. + await writeFile(target, '#define VALUE "two"\n'); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect(execFileSync(outPath, { encoding: "utf8" }).trim()).toBe("two"); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + } + }, +); + +test.skipIf(process.platform === "win32")( + "output-local hits detect newly shadowing nested headers", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-local-artifact-shadow-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const cPath = join(dir, "program.c"); + const outPath = join(dir, "program"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + await mkdir(join(dir, "sys")); + await writeFile( + cPath, + '#include \n#include "sys/param.h"\nint main(void) { printf("%d\\n", MAXPATHLEN); return 0; }\n', + ); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect(execFileSync(outPath, { encoding: "utf8" }).trim()).not.toBe("7"); + + // The source directory itself does not necessarily change metadata when + // a child directory gains a file. Its recursive namespace digest must. + await writeFile(join(dir, "sys", "param.h"), "#define MAXPATHLEN 7\n"); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect(execFileSync(outPath, { encoding: "utf8" }).trim()).toBe("7"); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + } + }, +); + +test("fresh processes preserve output-local dependency coverage after source edits", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-local-artifact-process-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const cPath = join(dir, "program.c"); + const outPath = join(dir, "program"); + const helperPath = join(dir, "compile.mjs"); + const childEnv: NodeJS.ProcessEnv = { ...process.env, SCRIPTC_CACHE_DIR: cacheRoot }; + delete childEnv["SCRIPTC_NO_CACHE"]; + delete childEnv["SCRIPTC_TEST_STABLE_TOOLCHAIN"]; + delete childEnv["SCRIPTC_TEST_TRUST_COMPILER_WRAPPER"]; + + await writeFile( + helperPath, + `import { compileC } from ${JSON.stringify(new URL("./cc.ts", import.meta.url).href)};\n` + + `await compileC({ cPath: process.argv[2], outPath: process.argv[3], cacheIdentity: "scriptc-generated-v1" });\n`, + ); + const compileInFreshProcess = (): void => { + execFileSync(process.execPath, ["--import", "tsx", helperPath, cPath, outPath], { + env: childEnv, + stdio: "pipe", + }); + }; + const stampPath = join( + cacheRoot, + "local", + createHash("sha256").update(outPath).digest("hex"), + ); + const dependencyPaths = async (): Promise => { + const stamp = JSON.parse(await readFile(stampPath, "utf8")) as { + dependencies: { path: string }[]; + }; + return stamp.dependencies.map((dependency) => dependency.path); + }; + + await writeFile(cPath, '#include \nint main(void) { puts("one"); return 0; }\n'); + compileInFreshProcess(); + const firstPaths = await dependencyPaths(); + expect(firstPaths.length).toBeGreaterThan(2); + + // The metadata fingerprints are restored from cache files in this second + // Node process; their in-memory fingerprint-to-path map starts empty. + await writeFile(cPath, '#include \nint main(void) { puts("two"); return 0; }\n'); + compileInFreshProcess(); + expect(execFileSync(outPath, { encoding: "utf8" }).trim()).toBe("two"); + expect(await dependencyPaths()).toEqual(firstPaths); +}); + +test("native metadata snapshots survive source edits and repair after tampering", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-native-metadata-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const cPath = join(dir, "program.c"); + const outPath = join(dir, "program"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + await writeFile(cPath, '#include \nint main(void) { puts("one"); return 0; }\n'); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + + const metadataDir = join(cacheRoot, "meta"); + const metadataNames = await readdir(metadataDir); + expect(metadataNames.length).toBeGreaterThanOrEqual(3); + const pinnedTime = new Date("2002-01-01T00:00:00.000Z"); + await Promise.all(metadataNames.map((name) => utimes(join(metadataDir, name), pinnedTime, pinnedTime))); + + // Program bytes are not toolchain identity. A source edit should reuse the + // validated target/compiler/link snapshots without republishing them. + await writeFile(cPath, '#include \nint main(void) { puts("two"); return 0; }\n'); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect(execFileSync(outPath, { encoding: "utf8" }).trim()).toBe("two"); + for (const name of metadataNames) { + expect((await stat(join(metadataDir, name))).mtimeMs).toBe(pinnedTime.getTime()); + } + + // Snapshot files are disposable cache data. Tampering invalidates one, + // strict discovery repairs it, and the requested source edit still lands. + const damagedPath = join(metadataDir, metadataNames[0]!); + const damaged = JSON.parse(await readFile(damagedPath, "utf8")) as { integrity: string }; + damaged.integrity = "0".repeat(64); + await writeFile(damagedPath, `${JSON.stringify(damaged)}\n`); + await utimes(damagedPath, pinnedTime, pinnedTime); + await writeFile(cPath, '#include \nint main(void) { puts("three"); return 0; }\n'); + await compileC({ cPath, outPath, cacheIdentity: "scriptc-generated-v1" }); + expect(execFileSync(outPath, { encoding: "utf8" }).trim()).toBe("three"); + expect((await stat(damagedPath)).mtimeMs).toBeGreaterThan(pinnedTime.getTime()); + const repaired = JSON.parse(await readFile(damagedPath, "utf8")) as { integrity: string }; + expect(repaired.integrity).not.toBe("0".repeat(64)); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + } +}); + test("damaged complete artifacts are rejected and rebuilt", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-artifact-integrity-")); scratch.push(dir); diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 0ae232321..7291b8e62 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -2,9 +2,9 @@ import { execFile, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { constants as fsConstants, existsSync, readdirSync } from "node:fs"; import { createRequire } from "node:module"; -import { access, chmod, copyFile, link, mkdir, mkdtemp, readdir, readFile, realpath, rename, rm, stat, unlink, utimes, writeFile } from "node:fs/promises"; +import { access, chmod, copyFile, link, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rename, rm, stat, unlink, utimes, writeFile } from "node:fs/promises"; import { availableParallelism, homedir, tmpdir } from "node:os"; -import { basename, delimiter, dirname, extname, isAbsolute, join, resolve } from "node:path"; +import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path"; import { promisify } from "node:util"; import { localizeElfObject, mergeAndLocalizeCoffObjects } from "./object-localize.js"; @@ -1405,6 +1405,8 @@ export interface LibArchiveOptions { * matching compileC's arbitrary-input safety boundary. */ cacheIdentity?: string; sanitize?: boolean; + /** Native optimization posture: release = -O2, dev = -O0. */ + optimization?: "release" | "dev"; /** Multi-instance library mode (the profile's abi.localize_runtime): the * external symbols to KEEP global — every other scriptc external * definition in the archive (the runtime's internals, the program TU's @@ -1444,6 +1446,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise const rtDir = runtimeSrcDir(); const driver = resolveCc(); const sanitize = opts.sanitize ?? false; + const optimization = opts.optimization ?? "release"; const regex = opts.regex ?? false; const sources = [ ...LIB_RUNTIME_SOURCES, @@ -1471,7 +1474,9 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise const cflags = [ "-std=c11", ...driver.targetArgs, - ...(sanitize ? ["-O1", "-fsanitize=address", "-DSCR_RC_AUDIT"] : ["-O2"]), + ...(sanitize + ? ["-O1", "-fsanitize=address", "-DSCR_RC_AUDIT"] + : [optimization === "dev" ? "-O0" : "-O2"]), "-fno-math-errno", "-fno-strict-aliasing", // the emitted object model type-puns — see compileC's buildArgs "-Wno-deprecated-declarations", @@ -2177,6 +2182,19 @@ async function resolvedToolIdentity( } const directCompilerDriverMemos = new Map(); +const directCompilerSelections = new Map(); + +function compilerDriverProbeKey( + driver: Pick, + environmentFingerprint: string, +): string { + return [ + environmentFingerprint, + driver.argv.join("\x1f"), + driver.target ?? "", + driver.targetArgs.join("\x1f"), + ].join("\0"); +} /** `/usr/bin/clang` on Darwin is Apple's immutable driver shim: its `-###` * trace names the selected Xcode/CommandLineTools clang rather than the shim @@ -2206,12 +2224,7 @@ async function compilerDriverSupportsPersistentCache( // wrapper. This is deliberately undocumented and test-scoped. if (process.env["SCRIPTC_TEST_TRUST_COMPILER_WRAPPER"] === "1") return true; - const driverKey = [ - environmentFingerprint, - driver.argv.join("\x1f"), - driver.target ?? "", - driver.targetArgs.join("\x1f"), - ].join("\0"); + const driverKey = compilerDriverProbeKey(driver, environmentFingerprint); const compiler = driver.argv[0] ?? "clang"; const resolvedDriver = await resolvedTool(compiler); // A prior dependency list cannot prove that name resolution is unchanged: @@ -2254,6 +2267,7 @@ async function compilerDriverSupportsPersistentCache( direct = effectiveCompiler !== null && (effectiveCompiler.fileIdentity === resolvedDriver.fileIdentity || isAppleSystemClangHandoff(resolvedDriver, effectiveCompiler)); + if (direct) directCompilerSelections.set(driverKey, effectiveCompiler!); } } catch { direct = false; @@ -2261,6 +2275,7 @@ async function compilerDriverSupportsPersistentCache( await rm(probeDir, { recursive: true, force: true }).catch(() => undefined); } directCompilerDriverMemos.set(probeKey, direct); + if (!direct) directCompilerSelections.delete(driverKey); return direct; } @@ -2368,7 +2383,8 @@ interface ImplicitToolchainProbe { compilerInvocation: string; dependencies: string[]; dependencyFingerprint: string; - tools: { spelling: string; identity: string | null }[]; + invocationPaths: string[]; + tools: { spelling: string; identity: string | null; path: string | null }[]; } interface ImplicitLinkerProbe { @@ -2376,7 +2392,45 @@ interface ImplicitLinkerProbe { linkerInvocation: string; dependencies: string[]; dependencyFingerprint: string; - linker: { spelling: string; identity: string | null }; + invocationPaths: string[]; + linker: { spelling: string; identity: string | null; path: string | null }; +} + +interface FingerprintDependencies { + paths: string[]; + /** Content identity computed while the enclosing fingerprint was minted. + * Null means the fingerprint has no file-content component. */ + contentPaths: string[]; + contentFingerprint: string | null; +} + +/** Dependency paths discovered while computing one strict fingerprint. The + * output-local cache stamp snapshots these exact files/directories after a + * validated build, then can prove a later same-output no-op without spawning + * clang again. The content identity closes the gap between hashing and that + * metadata snapshot: a file changed in the gap cannot make new bytes ride an + * old key. Keep the map bounded for long-lived corpus/test processes. */ +const fingerprintDependencies = new Map(); +function rememberFingerprintDependencies( + fingerprint: string, + paths: readonly string[], + contentPaths: readonly string[] = [], + contentFingerprint: string | null = null, +): string { + fingerprintDependencies.set(fingerprint, { + paths: [...new Set(paths)].sort(), + contentPaths: [...new Set(contentPaths)].sort(), + contentFingerprint, + }); + if (fingerprintDependencies.size > 256) { + const oldest = fingerprintDependencies.keys().next().value as string | undefined; + if (oldest !== undefined) fingerprintDependencies.delete(oldest); + } + return fingerprint; +} + +function fingerprintDependencyPaths(fingerprint: string): string[] { + return fingerprintDependencies.get(fingerprint)?.paths ?? []; } function parseMakeDependencies(output: string, cwd: string = process.cwd()): string[] { @@ -2425,6 +2479,7 @@ interface EffectiveCompilerInvocationProbe { invocation: string; dependencies: string[]; dependencyFingerprint: string; + invocationPaths: string[]; } /** The effective cc1 invocation and injected dependencies for the flags used @@ -2481,13 +2536,18 @@ async function effectiveCompilerInvocationFingerprintFresh( invocation: normalizedProbeInvocation(invocation, probeDir), dependencies, dependencyFingerprint: await fingerprintDependencyFiles(dependencies), + invocationPaths: await existingDriverTracePaths( + `${invocation.stdout}\n${invocation.stderr}`, + probeDir, + probeDir, + ), }; } finally { await rm(probeDir, { recursive: true, force: true }).catch(() => undefined); } } - return createHash("sha256") + const fingerprint = createHash("sha256") .update("effective-compiler-invocation-v2\0") .update(environmentFingerprint) .update("\0") @@ -2505,6 +2565,12 @@ async function effectiveCompilerInvocationFingerprintFresh( .update("\0") .update(probe.dependencyFingerprint) .digest("hex"); + return rememberFingerprintDependencies( + fingerprint, + [...probe.dependencies, ...probe.invocationPaths], + probe.dependencies, + probe.dependencyFingerprint, + ); } const stableEffectiveCompilerInvocationMemos = new Map>(); @@ -2536,15 +2602,25 @@ async function nativeSourceFiles( include: (name: string) => boolean = (name) => name.endsWith(".c") || name.endsWith(".h"), ): Promise { const files: string[] = []; - const walk = async (current: string): Promise => { + const walk = async (current: string, ancestors: ReadonlySet): Promise => { + const canonical = await realpath(current).catch(() => resolve(current)); + if (ancestors.has(canonical)) return; + const nestedAncestors = new Set(ancestors).add(canonical); const entries = await readdir(current, { withFileTypes: true }).catch(() => []); for (const entry of entries) { const path = join(current, entry.name); - if (entry.isDirectory() && recursive) await walk(path); - else if (entry.isFile() && include(entry.name)) files.push(path); + if (entry.isDirectory() && recursive) { + await walk(path, nestedAncestors); + } else if (entry.isSymbolicLink()) { + const target = await stat(path).catch(() => null); + if (target?.isDirectory() && recursive) await walk(path, nestedAncestors); + else if (target?.isFile() && include(entry.name)) files.push(path); + } else if (entry.isFile() && include(entry.name)) { + files.push(path); + } } }; - await walk(directory); + await walk(directory, new Set()); return files.sort(); } @@ -2609,6 +2685,23 @@ async function fingerprintDependencyFiles(paths: readonly string[]): Promise { + const distinct = [...new Set(fingerprints)]; + const identities = distinct + .map((fingerprint) => fingerprintDependencies.get(fingerprint)) + // A restored native-metadata stamp carries and revalidates the dependency + // snapshot from the process that minted this fingerprint. Only identities + // computed in this process have an additional content hash to recheck here. + .filter((identity): identity is FingerprintDependencies => identity !== undefined); + return (await Promise.all( + identities.map(async (identity) => + identity.contentFingerprint === null || + await fingerprintDependencyFiles(identity.contentPaths).catch(() => null) === + identity.contentFingerprint + ), + )).every(Boolean); +} + interface TranslationUnitDependencyProbe { compilerIdentity: string; dependencies: string[]; @@ -2629,7 +2722,10 @@ async function translationUnitDependencyFingerprintFresh( environmentFingerprint: string, ): Promise { if (sourcePath.endsWith(".ll")) { - return createHash("sha256").update("translation-unit-dependencies-v1\0llvm-ir").digest("hex"); + return rememberFingerprintDependencies( + createHash("sha256").update("translation-unit-dependencies-v1\0llvm-ir").digest("hex"), + [], + ); } const compiler = driver.argv[0] ?? "clang"; @@ -2673,7 +2769,7 @@ async function translationUnitDependencyFingerprintFresh( } } - return createHash("sha256") + const fingerprint = createHash("sha256") .update("translation-unit-dependencies-v1\0") .update(environmentFingerprint) .update("\0") @@ -2683,6 +2779,12 @@ async function translationUnitDependencyFingerprintFresh( .update("\0") .update(probe.dependencyFingerprint) .digest("hex"); + return rememberFingerprintDependencies( + fingerprint, + probe.dependencies, + probe.dependencies, + probe.dependencyFingerprint, + ); } const stableTranslationUnitDependencyMemos = new Map>(); @@ -2809,11 +2911,20 @@ async function implicitToolchainFingerprintFresh( compilerInvocation: normalizedProbeInvocation(compilerInvocation, probeDir), dependencies: dependencyPaths, dependencyFingerprint: await fingerprintDependencyFiles(dependencyPaths), + invocationPaths: await existingDriverTracePaths( + `${compilerInvocation.stdout}\n${compilerInvocation.stderr}`, + probeDir, + probeDir, + ), tools: await Promise.all( - toolSpellings.map(async (spelling) => ({ - spelling, - identity: await resolvedToolIdentity(spelling), - })), + toolSpellings.map(async (spelling) => { + const resolved = await resolvedTool(spelling); + return { + spelling, + identity: resolved?.cacheIdentity ?? null, + path: resolved?.canonicalPath ?? null, + }; + }), ), }; } finally { @@ -2845,7 +2956,16 @@ async function implicitToolchainFingerprintFresh( .update(currentIdentity ?? tool.identity ?? "") .update("\0"); } - return hash.digest("hex"); + return rememberFingerprintDependencies( + hash.digest("hex"), + [ + ...probe.dependencies, + ...probe.invocationPaths, + ...probe.tools.flatMap((tool) => tool.path === null ? [] : [tool.path]), + ], + probe.dependencies, + probe.dependencyFingerprint, + ); } const stableImplicitToolchainMemos = new Map>(); @@ -2888,6 +3008,63 @@ function driverTraceCandidates(line: string): string[] { return candidates; } +async function existingDriverTracePaths( + output: string, + cwd: string, + excludedRoot: string, +): Promise { + const candidates = new Set(); + for (const line of output.split(/\r?\n/)) { + const tokens = driverTraceCandidates(line); + for (let index = 0; index < tokens.length; index++) { + const token = tokens[index]!; + const joinedPathOption = ["-I", "-L", "-F"].find( + (option) => token.startsWith(option) && token.length > option.length, + ); + const equalsPathOption = ["--sysroot=", "-resource-dir="].find((option) => + token.startsWith(option) + ); + const separatePathOptions = [ + "-I", + "-L", + "-F", + "--sysroot", + "-isysroot", + "-resource-dir", + "-isystem", + "-iquote", + "-internal-isystem", + "-internal-externc-isystem", + "-internal-iframework", + ]; + const optionPath = joinedPathOption !== undefined + ? token.slice(joinedPathOption.length) + : equalsPathOption !== undefined + ? token.slice(equalsPathOption.length) + : separatePathOptions.includes(token) + ? tokens[index + 1] ?? "" + : token; + if (!isAbsolute(optionPath)) continue; + const path = resolve(cwd, optionPath); + if ( + path === excludedRoot || + path.startsWith(`${excludedRoot}/`) || + path.startsWith(`${excludedRoot}\\`) + ) { + continue; + } + candidates.add(path); + } + } + const existing = await Promise.all( + [...candidates].map(async (path) => [path, await lstat(path).catch(() => null)] as const), + ); + return existing + .filter((entry): entry is readonly [string, NonNullable<(typeof entry)[1]>] => entry[1] !== null) + .map(([path]) => path) + .sort(); +} + export async function parseLinkTraceFiles( output: string, cwd: string, @@ -3008,9 +3185,21 @@ async function implicitLinkerFingerprintFresh( linkerInvocation: normalizedProbeInvocation(driverInvocation, probeDir), dependencies, dependencyFingerprint: await fingerprintDependencyFiles(dependencies), + invocationPaths: await existingDriverTracePaths( + `${driverInvocation.stdout}\n${driverInvocation.stderr}`, + probeDir, + probeDir, + ), linker: { spelling: linkerSpelling, - identity: linkerSpelling === "" ? null : await resolvedToolIdentity(linkerSpelling), + identity: + linkerSpelling === "" + ? null + : (await resolvedTool(linkerSpelling))?.cacheIdentity ?? null, + path: + linkerSpelling === "" + ? null + : (await resolvedTool(linkerSpelling))?.canonicalPath ?? null, }, }; } finally { @@ -3022,7 +3211,7 @@ async function implicitLinkerFingerprintFresh( probe.linker.spelling === "" ? null : await resolvedToolIdentity(probe.linker.spelling); - return createHash("sha256") + const fingerprint = createHash("sha256") .update("implicit-linker-v3\0") .update(environmentFingerprint) .update("\0") @@ -3048,6 +3237,16 @@ async function implicitLinkerFingerprintFresh( .update("\0") .update(linkerIdentity ?? probe.linker.identity ?? "") .digest("hex"); + return rememberFingerprintDependencies( + fingerprint, + [ + ...probe.dependencies, + ...probe.invocationPaths, + ...(probe.linker.path === null ? [] : [probe.linker.path]), + ], + probe.dependencies, + probe.dependencyFingerprint, + ); } const stableImplicitLinkerMemos = new Map>(); @@ -3089,23 +3288,18 @@ function ccacheAvailable(): Promise { return ccacheMemo; } -/** Content hash of every .c/.h in the runtime src dir, the Ryū sources +/** Content hash of every .c/.h in the runtime src tree, the Ryū sources * textually included by scr_number.c, and the separately-built vendor pins — * everything a binary links that the emitted C bytes don't already cover * (npm-embedded C rides inside the emitted C; the engine and standalone * vendor archives are pinned by their version constants). The small source - * tree is hashed on every identity calculation: a stat-only memo can miss a - * same-size edit whose timestamp was preserved by a copy/sync tool. */ + * tree is hashed on every identity calculation: recursive enumeration is + * required because a newly added nested header can begin shadowing a system + * include without changing any previously selected dependency, while content + * hashing catches same-size edits whose timestamp was preserved by a copy/sync + * tool. */ async function runtimeFingerprintFresh(rtDir: string): Promise { - const groups = await Promise.all( - [ - { label: "runtime", dir: rtDir }, - { label: "ryu", dir: join(rtDir, "..", "vendor", "ryu") }, - ].map(async (group) => { - const names = (await readdir(group.dir)).filter((n) => n.endsWith(".c") || n.endsWith(".h")).sort(); - return { ...group, names }; - }), - ); + const groups = await runtimeFingerprintInputGroups(rtDir); const h = createHash("sha256").update(QJS_COMMIT).update(MBEDTLS_VERSION).update(ZLIB_VERSION); for (const group of groups) { for (const n of group.names) { @@ -3115,6 +3309,28 @@ async function runtimeFingerprintFresh(rtDir: string): Promise { return h.digest("hex"); } +async function runtimeFingerprintInputGroups( + rtDir: string, +): Promise<{ label: string; dir: string; names: string[] }[]> { + return Promise.all( + [ + { label: "runtime", dir: rtDir }, + { label: "ryu", dir: join(rtDir, "..", "vendor", "ryu") }, + ].map(async (group) => { + const names = (await nativeSourceFiles(group.dir, true)) + .map((path) => relative(group.dir, path)) + .sort(); + return { ...group, names }; + }), + ); +} + +async function runtimeFingerprintInputPaths(rtDir: string): Promise { + return (await runtimeFingerprintInputGroups(rtDir)).flatMap((group) => + group.names.map((name) => join(group.dir, name)) + ); +} + const stableRuntimeFingerprintMemos = new Map>(); export function runtimeFingerprint(rtDir: string): Promise { const key = resolve(rtDir); @@ -3206,6 +3422,401 @@ async function publishCachedFile(source: string, destination: string): Promise { + const hash = createHash("sha256").update("native-dependency-tree-v1\0"); + const visited = new Set(); + const excluded = excludedPaths.map((path) => resolve(path)); + const walk = async (directory: string, relative: string): Promise => { + const canonical = await realpath(directory); + if (visited.has(canonical)) { + hash.update(relative).update("\0cycle\0").update(canonical).update("\0"); + return; + } + visited.add(canonical); + const entries = await readdir(directory, { withFileTypes: true }); + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + const path = join(directory, entry.name); + const absolute = resolve(path); + if (excluded.some((candidate) => + absolute === candidate || + absolute.startsWith(`${candidate}/`) || + absolute.startsWith(`${candidate}\\`) + )) continue; + const child = relative === "" ? entry.name : `${relative}/${entry.name}`; + const info = await lstat(path); + const kind = info.isDirectory() + ? "directory" + : info.isFile() + ? "file" + : info.isSymbolicLink() + ? "symlink" + : "other"; + hash.update(child).update("\0").update(kind).update("\0"); + if (kind === "symlink") { + const target = await realpath(path).catch(() => ""); + hash.update(target).update("\0"); + const targetInfo = await stat(path).catch(() => null); + if (targetInfo?.isDirectory()) await walk(path, child); + } + if (kind === "directory") await walk(path, child); + } + visited.delete(canonical); + }; + await walk(root, ""); + return hash.digest("hex"); +} + +function localDependencyKind( + info: Awaited>, +): LocalArtifactDependency["kind"] | null { + return info.isFile() + ? "file" + : info.isDirectory() + ? "directory" + : info.isSymbolicLink() + ? "symlink" + : null; +} + +async function snapshotLocalArtifactDependency( + path: string, + treeExclusions: readonly string[] | null = null, +): Promise { + const info = await lstat(path); + const kind = localDependencyKind(info); + if (kind === null) throw new Error(`unsupported local artifact dependency: ${path}`); + const dependency: LocalArtifactDependency = { + path, + kind, + dev: info.dev, + ino: info.ino, + size: info.size, + mtimeMs: info.mtimeMs, + ctimeMs: info.ctimeMs, + }; + if (kind === "directory" && treeExclusions !== null) { + dependency.treeExclusions = [...new Set(treeExclusions.map((entry) => resolve(entry)))].sort(); + dependency.treeDigest = await directoryTreeDigest(path, dependency.treeExclusions); + } + if (kind === "symlink") { + const targetPath = await realpath(path); + const target = await stat(path); + const targetKind = target.isFile() ? "file" : target.isDirectory() ? "directory" : null; + if (targetKind === null) throw new Error(`unsupported symlink target dependency: ${path}`); + dependency.targetPath = targetPath; + dependency.targetKind = targetKind; + dependency.targetDev = target.dev; + dependency.targetIno = target.ino; + dependency.targetSize = target.size; + dependency.targetMtimeMs = target.mtimeMs; + dependency.targetCtimeMs = target.ctimeMs; + if (targetKind === "directory" && treeExclusions !== null) { + dependency.treeExclusions = [...new Set(treeExclusions.map((entry) => resolve(entry)))].sort(); + dependency.treeDigest = await directoryTreeDigest(path, dependency.treeExclusions); + } + } + return dependency; +} + +async function snapshotLocalArtifactDependencies( + dependencyPaths: readonly string[], + recursiveDirectories: readonly string[] = [], + recursiveExclusions: readonly string[] = [], +): Promise { + const recursive = new Set(recursiveDirectories.map((path) => resolve(path))); + return Promise.all( + [...new Set(dependencyPaths)].sort().map((path) => + snapshotLocalArtifactDependency( + path, + recursive.has(resolve(path)) ? recursiveExclusions : null, + ) + ), + ); +} + +async function localArtifactDependenciesStillMatch( + dependencies: readonly LocalArtifactDependency[], +): Promise { + return (await Promise.all( + dependencies.map(async (dependency) => { + const current = await snapshotLocalArtifactDependency( + dependency.path, + dependency.treeDigest === undefined ? null : dependency.treeExclusions ?? [], + ).catch(() => null); + return current !== null && JSON.stringify(current) === JSON.stringify(dependency); + }), + )).every(Boolean); +} + +interface NativeMetadataStamp { + version: 2; + key: string; + values: Record; + dependencies: LocalArtifactDependency[]; + integrity: string; +} + +function nativeMetadataStampPath(root: string, key: string): string { + return join(root, "meta", createHash("sha256").update(key).digest("hex")); +} + +function nativeMetadataStampIntegrity( + stamp: Pick, +): string { + return createHash("sha256") + .update("native-metadata-stamp-v2\0") + .update(JSON.stringify(stamp)) + .digest("hex"); +} + +async function readNativeMetadataStamp( + root: string, + key: string, +): Promise { + try { + const stamp = JSON.parse( + await readFile(nativeMetadataStampPath(root, key), "utf8"), + ) as NativeMetadataStamp; + if ( + stamp.version !== 2 || + stamp.key !== key || + stamp.values === null || + typeof stamp.values !== "object" || + !Array.isArray(stamp.dependencies) || + !/^[0-9a-f]{64}$/.test(stamp.integrity) || + nativeMetadataStampIntegrity({ + version: stamp.version, + key: stamp.key, + values: stamp.values, + dependencies: stamp.dependencies, + }) !== stamp.integrity || + !(await localArtifactDependenciesStillMatch(stamp.dependencies)) + ) { + return null; + } + return stamp; + } catch { + return null; + } +} + +async function publishNativeMetadataStamp( + root: string, + key: string, + values: Record, + dependencyPaths: readonly string[], + fingerprints: readonly string[] = [], +): Promise { + const destination = nativeMetadataStampPath(root, key); + await mkdir(dirname(destination), { recursive: true }); + const dependencies = await snapshotLocalArtifactDependencies(dependencyPaths); + if (!(await fingerprintDependenciesStillMatch(fingerprints))) { + throw new CacheInputsChangedError(); + } + const unsigned = { version: 2, key, values, dependencies } as const; + const stamp: NativeMetadataStamp = { + ...unsigned, + integrity: nativeMetadataStampIntegrity(unsigned), + }; + const tmp = `${destination}.tmp-${process.pid}-${Math.random().toString(36).slice(2)}`; + try { + await writeFile(tmp, `${JSON.stringify(stamp)}\n`, { mode: 0o600 }); + await rename(tmp, destination); + } finally { + await rm(tmp, { force: true }).catch(() => undefined); + } + return stamp; +} + +function nativeMetadataKey( + kind: string, + parts: readonly (string | readonly string[])[], +): string { + const hash = createHash("sha256").update(`native-metadata-${kind}-v2\0`); + for (const part of parts) { + hash.update(typeof part === "string" ? part : part.join("\x1f")).update("\0"); + } + return `${kind}-${hash.digest("hex")}`; +} + +/** The caller-visible output is itself the cheapest safe cache tier. Once a + * generated TU has produced this exact binary, an unchanged rebuild need not + * rediscover every SDK header and linker input merely to copy equivalent bytes + * back onto the same path. This tier is deliberately narrower than the CAS: + * only frontend-generated programs with no caller-owned native inputs opt in. + * The generated TU bytes, every scriptc runtime source, the selected direct + * compiler inode, target/options/environment, and the output path all join the + * key. A digest rejects a modified/truncated output before the no-op hit. */ +function localArtifactIdentity( + opts: CcOptions, + driver: CcDriver, + environmentFingerprint: string, + compilerIdentity: string, + runtimeHash: string, + programBytes: Buffer, +): string { + const normalizedOptions = Object.fromEntries( + Object.entries(opts) + .filter(([, value]) => value !== undefined) + .sort(([a], [b]) => a.localeCompare(b)), + ); + return createHash("sha256") + .update("local-artifact-v1\0") + .update(cacheTargetIdentity(driver)).update("\0") + .update(environmentFingerprint).update("\0") + .update(compilerIdentity).update("\0") + .update(runtimeHash).update("\0") + .update(driver.argv.join("\x1f")).update("\0") + .update(driver.targetArgs.join("\x1f")).update("\0") + .update(driver.linkArgs.join("\x1f")).update("\0") + .update(process.env["SCRIPTC_FETCH_CURL"] === "1" ? "fetch-curl" : "fetch-native").update("\0") + .update(JSON.stringify(normalizedOptions)).update("\0") + .update(resolve(opts.cPath)).update("\0") + .update(resolve(opts.outPath)).update("\0") + .update(programBytes) + .digest("hex"); +} + +function localArtifactStampPath(root: string, outPath: string): string { + const outputKey = createHash("sha256").update(resolve(outPath)).digest("hex"); + return join(root, "local", outputKey); +} + +function localArtifactStampIntegrity( + stamp: Pick, +): string { + return createHash("sha256") + .update("local-artifact-stamp-v2\0") + .update(JSON.stringify(stamp)) + .digest("hex"); +} + +async function localArtifactHit( + stampPath: string, + outPath: string, + key: string, +): Promise { + try { + const stamp = JSON.parse(await readFile(stampPath, "utf8")) as Partial; + const output = await lstat(outPath); + const expectedMode = 0o777 & ~process.umask(); + if ( + stamp.version !== 2 || + stamp.key !== key || + !/^[0-9a-f]{64}$/.test(stamp.digest ?? "") || + !Array.isArray(stamp.dependencies) || + !/^[0-9a-f]{64}$/.test(stamp.integrity ?? "") || + localArtifactStampIntegrity({ + version: stamp.version, + key: stamp.key, + digest: stamp.digest!, + dependencies: stamp.dependencies, + }) !== stamp.integrity || + !output.isFile() || + (output.mode & 0o777) !== expectedMode || + stamp.dependencies.some((dependency) => + dependency === null || + typeof dependency !== "object" || + typeof dependency.path !== "string" || + dependency.kind !== "file" && + dependency.kind !== "directory" && + dependency.kind !== "symlink" || + typeof dependency.dev !== "number" || + typeof dependency.ino !== "number" || + typeof dependency.size !== "number" || + typeof dependency.mtimeMs !== "number" || + typeof dependency.ctimeMs !== "number" || + dependency.treeDigest !== undefined && typeof dependency.treeDigest !== "string" || + dependency.kind === "symlink" && ( + typeof dependency.targetPath !== "string" || + dependency.targetKind !== "file" && dependency.targetKind !== "directory" || + typeof dependency.targetDev !== "number" || + typeof dependency.targetIno !== "number" || + typeof dependency.targetSize !== "number" || + typeof dependency.targetMtimeMs !== "number" || + typeof dependency.targetCtimeMs !== "number" + ) + ) || + !(await localArtifactDependenciesStillMatch(stamp.dependencies)) || + await fileDigest(outPath) !== stamp.digest + ) { + return false; + } + const now = new Date(); + await utimes(stampPath, now, now).catch(() => undefined); + return true; + } catch { + return false; + } +} + +async function publishLocalArtifactStamp( + stampPath: string, + outPath: string, + key: string, + dependencyPaths: readonly string[], + recursiveDirectories: readonly string[] = [], + recursiveExclusions: readonly string[] = [], +): Promise { + await mkdir(dirname(stampPath), { recursive: true }); + const tmp = `${stampPath}.tmp-${process.pid}-${Math.random().toString(36).slice(2)}`; + try { + const dependencies = await snapshotLocalArtifactDependencies( + dependencyPaths, + recursiveDirectories, + recursiveExclusions, + ); + const unsigned = { + version: 2, + key, + digest: await fileDigest(outPath), + dependencies, + } as const; + const stamp: LocalArtifactStamp = { + ...unsigned, + integrity: localArtifactStampIntegrity(unsigned), + }; + await writeFile(tmp, `${JSON.stringify(stamp)}\n`, { mode: 0o600 }); + await rename(tmp, stampPath); + } finally { + await rm(tmp, { force: true }).catch(() => undefined); + } +} + /** The cached .o set for one flag flavor, compiled on first need. Concurrent * first builds (parallel test workers on a cold cache) may duplicate work; * per-file atomic renames make every winner equivalent. Publication is held @@ -3248,7 +3859,22 @@ async function ensureRuntimeObjects( missing.slice(i, i + width).map(async (src) => { const tmpObj = join(tmpDir, `${basename(src, ".c")}.o`); const argv = [...(useCcache ? ["ccache"] : []), ...ccArgv, ...cflags, "-c", src, "-o", tmpObj]; - await execFileAsync(argv[0] ?? "clang", argv.slice(1)); + await execFileAsync(argv[0] ?? "clang", argv.slice(1), useCcache + ? { + // ccache direct mode remembers only the headers selected by + // its previous manifest and can miss a newly created, + // higher-priority header. The scriptc object-set key already + // includes the recursive runtime namespace fingerprint, so + // carry it into ccache's own keyspace as well. + env: { + ...process.env, + CCACHE_NAMESPACE: [ + process.env["CCACHE_NAMESPACE"], + `scriptc-${setKey}`, + ].filter((value) => value !== undefined && value !== "").join(":"), + }, + } + : undefined); compiled.set(src, { object: tmpObj, digest: await fileDigest(tmpObj) }); }), ); @@ -3469,10 +4095,114 @@ export async function compileC(opts: CcOptions): Promise { root = null; } } + let localArtifact: { + stampPath: string; + key: string; + runtimeHash: string; + programBytes: Buffer; + compilerPath: string; + } | null = null; + // Generated executable TUs are closed over scriptc's own runtime tree. A + // same-output rebuild can therefore check those bytes directly before the + // broader cross-output CAS performs its compiler/SDK/linker rediscovery. + // FFI/native-input builds and the public arbitrary-C cache API stay on the + // strict path because their dependency graphs are caller-owned. + if ( + root !== null && + cachePolicy.completeArtifacts && + opts.cacheIdentity === "scriptc-generated-v1" && + (opts.linkInputs?.length ?? 0) === 0 && + (opts.systemLibraries?.length ?? 0) === 0 && + process.env["SCRIPTC_TEST_TRUST_COMPILER_WRAPPER"] !== "1" + ) { + try { + const [compiler, runtimeHash, programBytes] = await Promise.all([ + resolvedTool(driver.argv[0] ?? "clang"), + runtimeFingerprint(rtDir), + readFile(opts.cPath), + ]); + if (compiler !== null) { + const effectiveCompiler = directCompilerSelections.get( + compilerDriverProbeKey(driver, toolchainEnv), + ) ?? compiler; + const key = localArtifactIdentity( + opts, + driver, + toolchainEnv, + `${compiler.cacheIdentity}\0${effectiveCompiler.cacheIdentity}`, + runtimeHash, + programBytes, + ); + const stampPath = localArtifactStampPath(root, opts.outPath); + localArtifact = { + stampPath, + key, + runtimeHash, + programBytes, + compilerPath: effectiveCompiler.canonicalPath, + }; + if (await localArtifactHit(stampPath, opts.outPath, key)) return; + } + } catch { + // The output-local tier is only an optimization; the fully validated + // CAS below remains the source of truth on any metadata trouble. + localArtifact = null; + } + } let implicitToolchain: string | null = null; + let toolchainMetadataStamp: NativeMetadataStamp | null = null; + const metadataCompiler = root === null + ? null + : await resolvedTool(driver.argv[0] ?? "clang"); + const metadataEffectiveCompiler = directCompilerSelections.get( + compilerDriverProbeKey(driver, toolchainEnv), + ) ?? metadataCompiler; + const toolchainMetadataKey = + root === null || + metadataCompiler === null || + metadataEffectiveCompiler === null || + process.env["SCRIPTC_TEST_TRUST_COMPILER_WRAPPER"] === "1" + ? null + : nativeMetadataKey("toolchain", [ + cacheTargetIdentity(driver), + toolchainEnv, + driver.argv, + driver.targetArgs, + metadataCompiler.cacheIdentity, + metadataEffectiveCompiler.cacheIdentity, + rtDir, + ]); if (persistentDriverCache) { try { - implicitToolchain = await implicitToolchainFingerprint(driver, toolchainEnv); + toolchainMetadataStamp = + root === null || toolchainMetadataKey === null + ? null + : await readNativeMetadataStamp(root, toolchainMetadataKey); + implicitToolchain = toolchainMetadataStamp?.values["implicitToolchain"] ?? + await implicitToolchainFingerprint(driver, toolchainEnv); + let compilerVersion = toolchainMetadataStamp?.values["compilerVersion"]; + if (compilerVersion === undefined) { + compilerVersion = await ccVersionOnce(driver.argv, toolchainEnv, true); + } + if ( + toolchainMetadataStamp === null && + root !== null && + toolchainMetadataKey !== null + ) { + if (metadataCompiler !== null && metadataEffectiveCompiler !== null) { + toolchainMetadataStamp = await publishNativeMetadataStamp( + root, + toolchainMetadataKey, + { implicitToolchain, compilerVersion }, + [ + metadataCompiler.canonicalPath, + metadataEffectiveCompiler.canonicalPath, + ...fingerprintDependencyPaths(implicitToolchain), + ], + [implicitToolchain], + ); + } + } } catch { // Cache discovery is best-effort. In particular, a compiler wrapper can // compile successfully without implementing the metadata probes. @@ -3790,21 +4520,77 @@ export async function compileC(opts: CcOptions): Promise { let runtimeCompilerInvocation: string | null = null; let programCompilerInvocation: string | null = null; - if (root !== null) { - try { - runtimeCompilerInvocation = await effectiveCompilerInvocationFingerprint( - driver, + let payloadMetadata: Promise<[string, string, Buffer]> | null = null; + let compileMetadataStamp: NativeMetadataStamp | null = null; + const compileMetadataKey = + root === null || process.env["SCRIPTC_TEST_TRUST_COMPILER_WRAPPER"] === "1" + ? null + : nativeMetadataKey("compile", [ + cacheTargetIdentity(driver), toolchainEnv, + implicitToolchain ?? "", + driver.argv, cflags, - ); - programCompilerInvocation = programSourceExtension === ".ll" - ? await effectiveCompilerInvocationFingerprint( - driver, - toolchainEnv, - programCompilerArgs, - programSourceExtension, - ) - : runtimeCompilerInvocation; + programCompilerArgs, + programSourceExtension, + ]); + if (root !== null) { + try { + // These probes inspect disjoint inputs. Start the payload reads here as + // well so runtime hashing and clang's dry-run traces overlap instead of + // forming a serial prelude before every cache lookup. + payloadMetadata = Promise.all([ + Promise.resolve( + toolchainMetadataStamp?.values["compilerVersion"] ?? + ccVersionOnce(driver.argv, toolchainEnv, true), + ), + localArtifact === null + ? runtimeFingerprint(rtDir) + : Promise.resolve(localArtifact.runtimeHash), + localArtifact === null + ? readFile(opts.cPath) + : Promise.resolve(localArtifact.programBytes), + ]); + compileMetadataStamp = compileMetadataKey === null + ? null + : await readNativeMetadataStamp(root, compileMetadataKey); + if (compileMetadataStamp !== null) { + runtimeCompilerInvocation = compileMetadataStamp.values["runtimeInvocation"] ?? null; + programCompilerInvocation = compileMetadataStamp.values["programInvocation"] ?? null; + if (runtimeCompilerInvocation === null || programCompilerInvocation === null) { + compileMetadataStamp = null; + } + } + if (compileMetadataStamp === null) { + const [runtimeInvocation, programInvocation] = await Promise.all([ + effectiveCompilerInvocationFingerprint(driver, toolchainEnv, cflags), + programSourceExtension === ".ll" + ? effectiveCompilerInvocationFingerprint( + driver, + toolchainEnv, + programCompilerArgs, + programSourceExtension, + ) + : Promise.resolve(null), + ]); + runtimeCompilerInvocation = runtimeInvocation; + programCompilerInvocation = programInvocation ?? runtimeInvocation; + if (compileMetadataKey !== null) { + compileMetadataStamp = await publishNativeMetadataStamp( + root, + compileMetadataKey, + { + runtimeInvocation: runtimeCompilerInvocation, + programInvocation: programCompilerInvocation, + }, + [ + ...fingerprintDependencyPaths(runtimeCompilerInvocation), + ...fingerprintDependencyPaths(programCompilerInvocation), + ], + [runtimeCompilerInvocation, programCompilerInvocation], + ); + } + } } catch { // Preserve the uncached build for wrappers that compile successfully but // cannot provide a dry-run trace for the real build flavor. @@ -3829,11 +4615,11 @@ export async function compileC(opts: CcOptions): Promise { let fingerprint: string; let cBytes: Buffer; try { - [cv, fingerprint, cBytes] = await Promise.all([ + [cv, fingerprint, cBytes] = await (payloadMetadata ?? Promise.all([ ccVersionOnce(driver.argv, toolchainEnv, true), runtimeFingerprint(rtDir), readFile(opts.cPath), - ]); + ])); } catch { // A version/fingerprint probe is an optimization boundary. If the compiler // itself can still compile, preserve the pre-cache behavior instead of @@ -3852,21 +4638,6 @@ export async function compileC(opts: CcOptions): Promise { (opts.linkInputs?.length ?? 0) === 0 && (opts.systemLibraries?.length ?? 0) === 0; let programDependencies: string | null = null; - if (cacheCompleteArtifact) { - try { - programDependencies = await translationUnitDependencyFingerprint( - driver, - cflags, - opts.cPath, - cBytes, - toolchainEnv, - ); - } catch { - // Program-header discovery is needed only for a complete hit. Runtime - // objects remain safe because they never contain the caller's TU. - cacheCompleteArtifact = false; - } - } const linkProbeArgs = [ ...(sanitize ? ["-fsanitize=address"] : []), ...threadArgs, @@ -3908,20 +4679,108 @@ export async function compileC(opts: CcOptions): Promise { ? effectiveLinkInvocationArgs : effectiveLinkInvocationArgs.filter((arg) => arg !== `-L${curlStubDir}`); let implicitLinker: string | null = null; - if (cacheCompleteArtifact) { - try { - implicitLinker = await implicitLinkerFingerprint( - driver, + let preBuildDependencies: LocalArtifactDependency[] | null = null; + let localArtifactDependencyPaths: string[] | null = null; + let linkMetadataStamp: NativeMetadataStamp | null = null; + const linkMetadataKey = + root === null || process.env["SCRIPTC_TEST_TRUST_COMPILER_WRAPPER"] === "1" + ? null + : nativeMetadataKey("link", [ + cacheTargetIdentity(driver), toolchainEnv, + implicitToolchain ?? "", + runtimeCompilerInvocation ?? "", + programCompilerInvocation ?? "", + driver.argv, linkProbeArgs, effectiveLinkInvocationArgs, linkTraceInvocationArgs, - ); + ]); + if (cacheCompleteArtifact) { + try { + // Header discovery and linker tracing are independent subprocess trees. + // Running them together removes one complete probe round-trip from both + // cache hits and ordinary edit/build misses without changing either key. + linkMetadataStamp = linkMetadataKey === null + ? null + : await readNativeMetadataStamp(root, linkMetadataKey); + if (linkMetadataStamp !== null) { + implicitLinker = linkMetadataStamp.values["implicitLinker"] ?? null; + if (implicitLinker === null) linkMetadataStamp = null; + } + [programDependencies, implicitLinker] = await Promise.all([ + translationUnitDependencyFingerprint( + driver, + cflags, + opts.cPath, + cBytes, + toolchainEnv, + ), + linkMetadataStamp === null + ? implicitLinkerFingerprint( + driver, + toolchainEnv, + linkProbeArgs, + effectiveLinkInvocationArgs, + linkTraceInvocationArgs, + ) + : Promise.resolve(implicitLinker!), + ]); + if (linkMetadataStamp === null && linkMetadataKey !== null) { + linkMetadataStamp = await publishNativeMetadataStamp( + root, + linkMetadataKey, + { implicitLinker }, + fingerprintDependencyPaths(implicitLinker), + [implicitLinker], + ); + } + preBuildDependencies = await snapshotLocalArtifactDependencies([ + ...(await runtimeFingerprintInputPaths(rtDir)), + ...(toolchainMetadataStamp?.dependencies.map((dependency) => dependency.path) ?? + fingerprintDependencyPaths(implicitToolchain!)), + ...(compileMetadataStamp?.dependencies.map((dependency) => dependency.path) ?? []), + ...(linkMetadataStamp?.dependencies.map((dependency) => dependency.path) ?? + fingerprintDependencyPaths(implicitLinker!)), + ...fingerprintDependencyPaths(programDependencies), + ]); + // Every content-bearing fingerprint was computed before this metadata + // snapshot. Re-read those exact files now so the snapshot cannot certify + // bytes that changed after hashing but before the final compile starts. + if (!(await fingerprintDependenciesStillMatch([ + implicitToolchain!, + runtimeCompilerInvocation!, + programCompilerInvocation!, + implicitLinker!, + programDependencies, + ])) || + await runtimeFingerprint(rtDir).catch(() => null) !== fingerprint || + !(await Promise.all( + [toolchainMetadataStamp, compileMetadataStamp, linkMetadataStamp] + .filter((stamp): stamp is NativeMetadataStamp => stamp !== null) + .map((stamp) => localArtifactDependenciesStillMatch(stamp.dependencies)), + )).every(Boolean)) { + throw new CacheInputsChangedError(); + } + if (localArtifact !== null) { + // Native metadata stamps persist their dependency paths across CLI + // processes; fingerprintDependencyPaths deliberately does not. Build + // the output-local stamp from this complete validated snapshot so a + // source edit in a fresh process cannot replace it with only the two + // process-local fallback paths. + localArtifactDependencyPaths = [ + localArtifact.compilerPath, + dirname(resolve(opts.cPath)), + ...preBuildDependencies.map((dependency) => dependency.path), + ]; + } } catch { - // Some compiler wrappers/linkers do not implement trace mode. Runtime - // objects remain safely cacheable, but a complete executable cannot be - // keyed without knowing every implicit link input. + // Program-header discovery and linker tracing are both required for a + // complete hit. Runtime objects remain safely cacheable if either probe + // is unavailable. cacheCompleteArtifact = false; + preBuildDependencies = null; + localArtifactDependencyPaths = null; } } const binDir = join(root, "bin"); @@ -3976,6 +4835,16 @@ export async function compileC(opts: CcOptions): Promise { // cache entry populated by a less restrictive shell must not widen access. await chmod(tmpOut, 0o777 & ~process.umask()); await rename(tmpOut, opts.outPath); + if (localArtifact !== null && localArtifactDependencyPaths !== null) { + await publishLocalArtifactStamp( + localArtifact.stampPath, + opts.outPath, + localArtifact.key, + localArtifactDependencyPaths, + [dirname(resolve(opts.cPath))], + [root], + ).catch(() => undefined); + } return; // hit: the program/runtime payload compile and link were skipped } catch { await rm(tmpOut, { force: true }).catch(() => undefined); @@ -4016,18 +4885,26 @@ export async function compileC(opts: CcOptions): Promise { let objects: Map | null = null; let cacheInputsStable = true; - let objectImplicitVerification: Promise | null = null; + let strictObjectVerification: Promise | null = null; const objectImplicitToolchainStillMatches = (): Promise => { - objectImplicitVerification ??= Promise.all([ + if (preBuildDependencies !== null) { + return localArtifactDependenciesStillMatch(preBuildDependencies); + } + // Complete-artifact caching can be disabled by caller-owned native + // inputs while the safe runtime-object tier remains active. Preserve its + // strict discovery fallback in that posture. + strictObjectVerification ??= Promise.all([ + runtimeFingerprint(rtDir), implicitToolchainFingerprint(driver, toolchainEnv), effectiveCompilerInvocationFingerprint(driver, toolchainEnv, cflags), ]).then( - ([currentImplicit, currentInvocation]) => + ([currentRuntime, currentImplicit, currentInvocation]) => + currentRuntime === fingerprint && currentImplicit === implicitToolchain && currentInvocation === runtimeCompilerInvocation, () => false, ); - return objectImplicitVerification; + return strictObjectVerification; }; try { const cached = await ensureRuntimeObjects( @@ -4037,8 +4914,7 @@ export async function compileC(opts: CcOptions): Promise { rtInputs, `obj-v5\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${implicitToolchain}\0${runtimeCompilerInvocation}\0${ccName}\0${cv}\0${fingerprint}\0`, async () => - (await runtimeFingerprint(rtDir)) === fingerprint && - (await objectImplicitToolchainStillMatches()), + await objectImplicitToolchainStillMatches(), ); objects = await stageRuntimeObjects(cached, join(buildDir, "runtime-objects")); } catch (err) { @@ -4055,10 +4931,11 @@ export async function compileC(opts: CcOptions): Promise { await installArtifact(privateOut, opts.outPath); if (cachedBin !== null && keyHex !== null) { - // This check is deliberately fresh rather than the object-publication - // verification above. The final program compile/link can itself race an - // SDK header, compiler, linker, CRT, or system-library replacement; its - // bytes must not be published under the pre-build identity in that case. + // Metadata comparison catches ordinary changes cheaply, but cannot by + // itself prove the snapshot was taken from the same bytes hashed into + // the key. Recompute every content-bearing identity after the final link + // so a header/SDK/compiler change in either pre-build gap cannot publish + // new output under an old key. const [currentRuntime, currentImplicit, currentRuntimeInvocation, currentProgramInvocation, currentProgramDependencies, currentLinker, currentCompiler] = await Promise.all([ runtimeFingerprint(rtDir).catch(() => null), @@ -4090,6 +4967,8 @@ export async function compileC(opts: CcOptions): Promise { ]); cacheInputsStable = cacheInputsStable && + preBuildDependencies !== null && + await localArtifactDependenciesStillMatch(preBuildDependencies) && currentRuntime === fingerprint && currentImplicit === implicitToolchain && currentRuntimeInvocation === runtimeCompilerInvocation && @@ -4108,6 +4987,21 @@ export async function compileC(opts: CcOptions): Promise { /* publishing is best-effort */ } } + if ( + localArtifact !== null && + localArtifactDependencyPaths !== null && + cacheCompleteArtifact && + cacheInputsStable + ) { + await publishLocalArtifactStamp( + localArtifact.stampPath, + opts.outPath, + localArtifact.key, + localArtifactDependencyPaths, + [dirname(resolve(opts.cPath))], + [root], + ).catch(() => undefined); + } } finally { await rm(buildDir, { recursive: true, force: true }).catch(() => undefined); } diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 5cc456b5f..050254017 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -1152,6 +1152,8 @@ export class Lowerer { * IrModule.unions. An arm's index in the canonical list is its runtime * tag. */ readonly unions = new UnionRegistry(); + /** Context-free successful type mappings. See TypeMapperCtx.typeMemo. */ + readonly typeMemo = new Map(); readonly ambient = ambientDtsPath(); readonly overridesAmbient = overridesDtsPath(); readonly fallbackAmbient = fallbackDtsPath(); @@ -1331,6 +1333,11 @@ export class Lowerer { isExternalTypeFile: (sf) => this.externalTypeSpecifiersByFile.has(tsgoPath(resolve(sf.fileName))), dynamic: this.dynamic, + typeMemo: this.typeMemo, + canMemoizeType: () => + this.typeParamBindings === null && + this.typeParamTsBindings === null && + this.mixinTypeContext === null, // fileTag is filled just below; the hook is only ever CALLED during // lowering, long after the constructor completes. isProgramFile: (sf) => this.fileTag.has(sf), diff --git a/packages/compiler/src/frontend/ts7/checker.ts b/packages/compiler/src/frontend/ts7/checker.ts index 879f2f3b7..b3c6258cb 100644 --- a/packages/compiler/src/frontend/ts7/checker.ts +++ b/packages/compiler/src/frontend/ts7/checker.ts @@ -44,7 +44,7 @@ import type { TypeReference, } from "typescript/unstable/sync"; import { walkPreorder } from "./ast.js"; -import { SignatureKind, TypeFlags } from "./enums.js"; +import { SignatureKind, SyntaxKind, TypeFlags } from "./enums.js"; /** Array-overload chunk size: large enough that per-request overhead * vanishes, small enough to keep any single JSON-RPC payload modest. */ @@ -92,11 +92,23 @@ function chunked(items: readonly T[], fetch: (chunk: readonly T[]) => read * memoized per-node call. */ const PREFETCH_MAX_DEPTH = 512; +/** Node kinds the lowering routinely asks getTypeAtLocation about. The + * fallback path remains correct for every other kind, but bulk-querying the + * entire AST was severe overfetch on generated facades (191k nodes fetched, + * only 29k ever requested). */ +const TYPE_PREFETCH_KINDS = new Set([ + SyntaxKind.Identifier, + SyntaxKind.PropertyAccessExpression, + SyntaxKind.ObjectLiteralExpression, + SyntaxKind.ArrayLiteralExpression, + SyntaxKind.ConditionalExpression, +]); + /** Preorder sweep of the whole file, ITERATIVE (walkPreorder): the obvious * recursive forEachChild walk overflowed the stack HERE, in the prefetch * sweep, on the binderBinaryExpressionStress chains — before lowering could * answer with its SC1090 nesting fence. */ -function collectNodes(sf: SourceFile): Node[] { +function collectNodes(sf: Node): Node[] { const nodes: Node[] = []; walkPreorder(sf, (n, depth) => { nodes.push(n); @@ -245,7 +257,13 @@ export class CheckerFacade { private prefetchTypes(sf: SourceFile): void { if (this.prefetchedTypes.has(sf)) return; this.prefetchedTypes.add(sf); - const nodes = collectNodes(sf).filter((n) => !this.typeAtLocation.has(n)); + this.prefetchTypesIn(sf); + } + + private prefetchTypesIn(root: Node): void { + const nodes = collectNodes(root).filter( + (n) => TYPE_PREFETCH_KINDS.has(n.kind) && !this.typeAtLocation.has(n), + ); const types = chunked(nodes, (chunk) => this.typesWithPanicFence(chunk)); nodes.forEach((n, i) => this.typeAtLocation.set(n, types[i])); } @@ -259,7 +277,13 @@ export class CheckerFacade { private prefetchSymbols(sf: SourceFile): void { if (this.prefetchedSymbols.has(sf)) return; this.prefetchedSymbols.add(sf); - const nodes = collectNodes(sf).filter((n) => !this.symbolAtLocation.has(n)); + this.prefetchSymbolsIn(sf); + } + + private prefetchSymbolsIn(root: Node): void { + const nodes = collectNodes(root).filter( + (n) => n.kind === SyntaxKind.Identifier && !this.symbolAtLocation.has(n), + ); // The same bisecting panic fence as the type sweep: tsgo panics on // SYMBOL queries too (observed: GetSymbolAtLocation over an // `import.defer(...)` callee — the sweep's batch must not turn one diff --git a/packages/compiler/src/frontend/types.ts b/packages/compiler/src/frontend/types.ts index f855932e2..c78f8d2eb 100644 --- a/packages/compiler/src/frontend/types.ts +++ b/packages/compiler/src/frontend/types.ts @@ -587,6 +587,14 @@ export interface TypeMapperCtx { /** --dynamic: `any` maps to the island handle type (jsval). Off, `any` * stays unmapped and the requires-dynamic diagnostic fires per site. */ dynamic: boolean; + /** Successful context-free mappings, keyed by dynamic posture + the + * TypeScript server's stable type id. A generated facade can reference the + * same large record/function type thousands of times; remapping its full + * member graph each time is quadratic work. Generic/mixin contexts opt out + * through canMemoizeType because the same checker type can map differently + * under different instantiation bindings. */ + typeMemo?: Map; + canMemoizeType?: () => boolean; /** True for files the Lowerer actually compiles (its module order). A * class type can reach the entry through the TYPE world alone — a jsdoc * `typeof import('./mod')` over a module never imported at value level @@ -678,9 +686,11 @@ function genChannels( * fallback for it. */ const MAP_TYPE_MAX_DEPTH = 64; let mapTypeDepth = 0; -/** Bumped whenever mapType resolves a type through CONTEXT-SENSITIVE hooks - * (a generic body's type parameter, a mixin instantiation) — mappings that - * make the same ts.Type answer differently across instantiation contexts. +/** Bumped whenever mapType resolves a type through CONTEXT-SENSITIVE or + * collection-sensitive hooks (a generic body's type parameter, a mixin + * instantiation, a generic class instance) — mappings that can make the same + * ts.Type answer differently across instantiation contexts or collection + * phases. * The recursive-shape machinery keys shape identity by checker type, which * is sound only for context-FREE mappings: a recursive frame that observes * a bump between entry and exit stays fenced (recursive generic-open types @@ -689,9 +699,28 @@ let contextResolutions = 0; export function mapType(type: ts.Type, ctx: TypeMapperCtx): IrType | null { if (mapTypeDepth >= MAP_TYPE_MAX_DEPTH) return null; + const topLevel = mapTypeDepth === 0; + const typeId = (type as { id?: number }).id; + const memoizable = typeId !== undefined && (ctx.canMemoizeType?.() ?? true); + const memoKey = memoizable ? `${ctx.dynamic ? 1 : 0}:${typeId}` : null; + if (memoKey !== null) { + const memo = ctx.typeMemo?.get(memoKey); + if (memo !== undefined) return memo; + } + const sensitivityAtEntry = contextResolutions; mapTypeDepth++; try { - return mapTypeInner(type, ctx); + const mapped = mapTypeInner(type, ctx); + if ( + mapped !== null && + memoKey !== null && + topLevel && + contextResolutions === sensitivityAtEntry && + (ctx.canMemoizeType?.() ?? true) + ) { + ctx.typeMemo?.set(memoKey, mapped); + } + return mapped; } finally { mapTypeDepth--; } @@ -1163,7 +1192,14 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { // INSTANTIATION's class (`Box%0`), registered on demand — the Lowerer // hook owns the instance table (monomorphization by flow). if (classDecl.typeParameters) { - return ctx.genericClassInstance ? ctx.genericClassInstance(classDecl, widened) : null; + const instance = ctx.genericClassInstance + ? ctx.genericClassInstance(classDecl, widened) + : null; + // Before the generic declaration's collection turn, the hook can only + // return its family shell; after collection the same checker type names + // a concrete registered instance. Never memoize either phase's answer. + if (instance !== null) contextResolutions++; + return instance; } return { kind: "object", className: classNamer(classDecl) }; } diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 1b55371a6..948d7a4f6 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1500,7 +1500,25 @@ function mergeSidecarIntSlots( } export async function compileLibrary(opts: CompileLibraryOptions): Promise { + const timingOn = process.env["SCRIPTC_TIMING"] === "1"; + const timingStart = performance.now(); + let timingLast = timingStart; + const timing = (phase: string, detail: Record = {}): void => { + if (!timingOn) return; + const now = performance.now(); + process.stderr.write( + `scriptc timing ${JSON.stringify({ + phase, + phase_ms: Math.round((now - timingLast) * 10) / 10, + total_ms: Math.round((now - timingStart) * 10) / 10, + rss_mb: Math.round(process.memoryUsage().rss / 1024 / 1024), + ...detail, + })}\n`, + ); + timingLast = now; + }; const loadedProfile = loadLibraryProfile(resolve(opts.profilePath)); + timing("profile-load"); if (!loadedProfile.ok) { return { ok: false, diagnostics: loadedProfile.diagnostics, sourceTexts: new Map() }; } @@ -1582,6 +1600,10 @@ export async function compileLibrary(opts: CompileLibraryOptions): Promise; @@ -1691,6 +1713,9 @@ export async function compileLibrary(opts: CompileLibraryOptions): Promise ({ ok: false, @@ -1772,6 +1798,7 @@ export async function compileLibrary(opts: CompileLibraryOptions): Promise libIntBoundaryDiag(v.path, v.cls, v.obligation!, v.detail!, v.fix!, v.loc))); } } + timing("integer-proof"); const validation = validateModule(mod); if (validation.length > 0) return fail(validation.map((v) => iceDiag(v.message, v.loc))); + timing("ir-validate"); await mkdir(opts.outDir, { recursive: true }); const stem = basename(entryPath).replace(/\.(ts|js|mjs|cjs)$/, ""); @@ -1797,8 +1826,10 @@ export async function compileLibrary(opts: CompileLibraryOptions): Promise", * "entry": "src/lib.ts", // ONE module, profile-relative * "emission": "llvm" | "c", // pins the emission; no fallback + * "optimization": "release" | "dev", // optional; default release * "abi": { * "prefix": "_", * "init_symbol": "_init", @@ -362,6 +363,9 @@ export interface LibraryProfile { entry: string; /** The pinned emission — no fallback concept exists on the library path. */ emission: "llvm" | "c"; + /** Native optimizer posture. release preserves the production -O2 archive; + * dev uses -O0 for fast iterative embedding builds. */ + optimization: "release" | "dev"; prefix: string; initSymbol: string; sinkRegisterSymbol: string; @@ -486,7 +490,7 @@ export function loadLibraryProfile( // the root would otherwise be silently inert — the exact footgun the // fence machinery refuses everywhere else. for (const k of Object.keys(p)) { - if (["profile_format", "name", "entry", "emission", "abi", "exports", "callbacks", "sidecar", "determinism"].includes(k)) continue; + if (["profile_format", "name", "entry", "emission", "optimization", "abi", "exports", "callbacks", "sidecar", "determinism"].includes(k)) continue; if (k === "fences" || k === "teachings" || k === "remediations") { throw new ProfileError( `'${k}' at the profile root does nothing — the ask-5 determinism surface lives under 'determinism.${k}'; move it there`, @@ -502,6 +506,12 @@ export function loadLibraryProfile( if (emission !== "llvm" && emission !== "c") { throw new ProfileError(`'emission' must be "llvm" or "c", got '${emission}'`); } + const optimization = p["optimization"] === undefined + ? "release" + : req(p["optimization"], "optimization", "string"); + if (optimization !== "release" && optimization !== "dev") { + throw new ProfileError(`'optimization' must be "release" or "dev", got '${optimization}'`); + } const abi = p["abi"]; if (abi === null || typeof abi !== "object" || Array.isArray(abi)) { throw new ProfileError("'abi' must be an object"); @@ -866,6 +876,7 @@ export function loadLibraryProfile( profilePath, entry, emission, + optimization, prefix, initSymbol, sinkRegisterSymbol, diff --git a/packages/compiler/test/ts7/facade.test.ts b/packages/compiler/test/ts7/facade.test.ts index 7fd36df2e..cbb7c4ab5 100644 --- a/packages/compiler/test/ts7/facade.test.ts +++ b/packages/compiler/test/ts7/facade.test.ts @@ -54,28 +54,29 @@ function collectNodes(w: TwoWorlds): Node[] { return nodes; } -test("the walk's queries batch: N nodes cost O(1) raw requests, repeats cost zero", () => { +test("hot expression and identifier queries batch; uncommon kinds fall back once", () => { const { facade, counts, w } = build(); const nodes = collectNodes(w); expect(nodes.length).toBeGreaterThan(300); for (const n of nodes) facade.getTypeAtLocation(n); - // One chunked array request for the whole file (fixture < chunk size), - // triggered by the first miss — never one request per node. - expect(counts["getTypeAtLocation"]).toBe(1); + // The first miss bulk-fetches only lowering's hot expression kinds. The + // uncommon declaration/token kinds then use the direct memoized fallback. + expect(counts["getTypeAtLocation"] ?? 0).toBeLessThan(nodes.length); for (const n of nodes) facade.getSymbolAtLocation(n); - expect(counts["getSymbolAtLocation"]).toBe(1); + expect(counts["getSymbolAtLocation"] ?? 0).toBeLessThan(nodes.length); // Symbol prefetch batch-fetches the symbols' types too... const typeOfSymbolBatches = counts["getTypeOfSymbol"] ?? 0; expect(typeOfSymbolBatches).toBe(1); - // ...so getTypeOfSymbol over every symbol the file surfaced is free. + // ...so hot identifier symbols are free. Symbols surfaced only by uncommon + // direct-fallback nodes pay one memoized query each. for (const n of nodes) { const s = facade.getSymbolAtLocation(n); if (s) facade.getTypeOfSymbol(s); } - expect(counts["getTypeOfSymbol"]).toBe(typeOfSymbolBatches); + expect(counts["getTypeOfSymbol"] ?? 0).toBeLessThan(nodes.length); // Warm repeats of everything: zero further raw traffic. const before = { ...counts }; @@ -150,18 +151,24 @@ test("isTupleType agrees with the raw checker; only object types round-trip, onc expect(counts["isTupleType"] ?? 0).toBe(before); }); -test("explicit prefetchSourceFile primes everything ahead of the walk", () => { +test("explicit prefetchSourceFile primes hot kinds and direct fallbacks memoize", () => { const { facade, counts, w } = build(); const sf = w.p7.getSourceFile(w.files[0]!)!; facade.prefetchSourceFile(sf); - const after = { ...counts }; const nodes = collectNodes(w); for (const n of nodes) { facade.getTypeAtLocation(n); const s = facade.getSymbolAtLocation(n); if (s) facade.getTypeOfSymbol(s); } - expect(counts).toEqual(after); + expect(counts["getTypeAtLocation"] ?? 0).toBeLessThan(nodes.length); + expect(counts["getSymbolAtLocation"] ?? 0).toBeLessThan(nodes.length); + const afterWalk = { ...counts }; + for (const n of nodes) { + facade.getTypeAtLocation(n); + facade.getSymbolAtLocation(n); + } + expect(counts).toEqual(afterWalk); }); test("autoPrefetch: false degrades to per-call queries (the escape hatch works)", () => { diff --git a/tests/corpus/1951-generic-classes-basics.ts b/tests/corpus/1951-generic-classes-basics.ts index 9c515c747..b1fbbe29d 100644 --- a/tests/corpus/1951-generic-classes-basics.ts +++ b/tests/corpus/1951-generic-classes-basics.ts @@ -1,6 +1,18 @@ // Generic classes, monomorphized per instantiation: fields, methods, // accessors, inference and explicit type arguments, instanceof against the // generic name (one runtime class in Node — the family interval here). +// The early reference pins collection-order independence: its checker type is +// first mapped before Box's family has collected, then must remap to Box%N. +class EarlyHolder { + readonly box: Box; + constructor(box: Box) { + this.box = box; + } + read(): number { + return this.box.get(); + } +} + class Box { v: T; constructor(v: T) { @@ -20,6 +32,7 @@ class Box { const a = new Box(41); // inferred Box a.set(a.get() + 1); console.log(a.get(), a.describe()); +console.log(new EarlyHolder(a).read()); const b = new Box("hi"); // explicit console.log(b.get().toUpperCase(), b.describe()); diff --git a/tests/harness/library-profile.test.ts b/tests/harness/library-profile.test.ts index bcdb4207e..8b35d8026 100644 --- a/tests/harness/library-profile.test.ts +++ b/tests/harness/library-profile.test.ts @@ -55,6 +55,7 @@ describe("library profile validation", () => { if (!r.ok) return; expect(r.profile.name).toBe("conformance-test"); expect(r.profile.emission).toBe("llvm"); + expect(r.profile.optimization).toBe("release"); expect(r.profile.prefix).toBe("kx_"); expect(r.profile.initSymbol).toBe("kx_init"); expect(r.profile.collectSymbol).toBe("kx_collect"); @@ -148,6 +149,12 @@ describe("library profile validation", () => { expectSc4001(rest, "'name'"); }); test("bad emission", () => expectSc4001({ ...good, emission: "wasm" }, "emission")); + test("optimization defaults to release and admits dev explicitly", () => { + const dev = loadLibraryProfile(writeProfile({ ...good, optimization: "dev" })); + expect(dev.ok).toBe(true); + if (dev.ok) expect(dev.profile.optimization).toBe("dev"); + expectSc4001({ ...good, optimization: "fast" }, "optimization"); + }); test("bad prefix identifier", () => expectSc4001({ ...good, abi: { ...good.abi, prefix: "9bad_" } }, "abi.prefix")); test("symbol without the prefix", () =>