From 8c6b05106498bd4de9b8ffe2074183e97ea6c0bd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 18 Aug 2026 08:14:40 -0500 Subject: [PATCH 1/5] Cache unchanged library frontends - Restore generated library artifacts before TypeScript and lowering on exact repeats - Track source, configuration, package-resolution, and missing-candidate inputs for safe invalidation - Add integrity, replacement, and cache-disable coverage plus cache documentation --- packages/cli/README.md | 2 +- packages/compiler/src/backend/cc.ts | 25 ++ .../src/frontend/input-tracker.test.ts | 61 ++++ .../compiler/src/frontend/input-tracker.ts | 241 ++++++++++++++ .../src/frontend/lowering/lower-builtins.ts | 9 +- .../src/frontend/npm-static-rewrite.ts | 37 +-- packages/compiler/src/frontend/npm-static.ts | 24 +- packages/compiler/src/frontend/npm.ts | 20 +- packages/compiler/src/frontend/program.ts | 6 +- packages/compiler/src/frontend/resolve.ts | 33 +- packages/compiler/src/frontend/ts7/program.ts | 31 +- packages/compiler/src/index.ts | 191 +++++++++-- .../compiler/src/library/early-cache.test.ts | 215 ++++++++++++ packages/compiler/src/library/early-cache.ts | 308 ++++++++++++++++++ tests/harness/README.md | 1 + 15 files changed, 1069 insertions(+), 135 deletions(-) create mode 100644 packages/compiler/src/frontend/input-tracker.test.ts create mode 100644 packages/compiler/src/frontend/input-tracker.ts create mode 100644 packages/compiler/src/library/early-cache.test.ts create mode 100644 packages/compiler/src/library/early-cache.ts diff --git a/packages/cli/README.md b/packages/cli/README.md index 4044bfa74..92b68d808 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -24,7 +24,7 @@ $ npm install -g scriptc Requires clang on the PATH (Xcode Command Line Tools on macOS, `clang` package on Linux). -Builds use a bounded persistent cache by default. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. The compiler remains required so dependency selection is rediscovered on every invocation. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass the cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions. +Builds use a bounded persistent cache by default. Exact unchanged library builds validate their recorded TypeScript/module-resolution inputs and restore the generated C/LLVM unit before starting the frontend; the native cache then applies its independent toolchain checks. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. Experimental provenance-source builds bypass the early frontend tier because their fetched-source registry is process state. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent native artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass every cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions. ## Commands diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 7291b8e62..8af648ae5 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -2075,6 +2075,31 @@ export function resolveBuildCacheRoot( : resolve(userHome, ".cache", "scriptc", "build"); } +/** Shared persistent-cache root for compiler-level tiers. The early library + * cache deliberately follows the native cache's activation and hard-disable + * contract, while native compilation retains ownership of toolchain safety. */ +export function buildCacheRoot(): string | null { + return resolveBuildCacheRoot(); +} + +/** Harden/create a compiler-level cache root using the same privacy policy as + * the artifact caches. Failure disables only the optional caller's tier. */ +export async function prepareBuildCacheRoot(root: string | null): Promise { + if (root === null) return null; + try { + await ensurePrivateCacheRoot(root, process.env["SCRIPTC_CACHE_DIR"] === undefined); + return root; + } catch { + return null; + } +} + +/** Register a successful compiler-level cache write with the shared bounded + * LRU policy. */ +export async function pruneBuildCache(root: string | null): Promise { + if (root !== null) await pruneCache(root).catch(() => undefined); +} + function cacheRootDir(): string | null { return resolveBuildCacheRoot(); } diff --git a/packages/compiler/src/frontend/input-tracker.test.ts b/packages/compiler/src/frontend/input-tracker.test.ts new file mode 100644 index 000000000..9cd0081b6 --- /dev/null +++ b/packages/compiler/src/frontend/input-tracker.test.ts @@ -0,0 +1,61 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { + FrontendInputTracker, + frontendInputsStillMatch, + trackedAccessibleEntries, + trackedFileExists, + trackedReadFile, + validFrontendInputSnapshot, +} from "./input-tracker.js"; + +const scratch: string[] = []; + +afterEach(async () => { + await Promise.all(scratch.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +test("tracked frontend reads invalidate on byte edits", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const file = join(dir, "entry.ts"); + await writeFile(file, "export const answer = 1;\n"); + + const tracker = new FrontendInputTracker(); + tracker.run(() => expect(trackedReadFile(file)).toContain("answer")); + const snapshot = tracker.snapshot(); + expect(validFrontendInputSnapshot(snapshot)).toBe(true); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + + await writeFile(file, "export const answer = 2;\n"); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + +test("failed resolution candidates invalidate when a file appears", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const candidate = join(dir, "dependency.ts"); + + const tracker = new FrontendInputTracker(); + tracker.run(() => expect(trackedFileExists(candidate)).toBe(false)); + const snapshot = tracker.snapshot(); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + + await writeFile(candidate, "export const loaded = true;\n"); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + +test("directory enumeration invalidates workspace discovery", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const packages = join(dir, "packages"); + await mkdir(packages); + + const tracker = new FrontendInputTracker(); + tracker.run(() => expect(trackedAccessibleEntries(packages)?.directories).toEqual([])); + const snapshot = tracker.snapshot(); + await mkdir(join(packages, "new-member")); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts new file mode 100644 index 000000000..b056b97f3 --- /dev/null +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -0,0 +1,241 @@ +import { createHash } from "node:crypto"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; +import { resolve } from "node:path"; + +/** + * Filesystem observations made while constructing and lowering one frontend + * program. The early library cache replays these probes before trusting a + * generated translation unit: successful reads are content-addressed, while + * failed candidate probes are retained so a newly-created module cannot hide + * behind an old resolution answer. + */ +export type FrontendInputProbe = + | { op: "file"; path: string; digest: string } + | { op: "kind"; path: string; kind: "file" | "directory" | "other" | "missing" } + | { op: "entries"; path: string; files: string[]; directories: string[] } + | { op: "realpath"; path: string; target: string | null }; + +export interface FrontendInputSnapshot { + version: 1; + probes: FrontendInputProbe[]; + stable: boolean; +} + +function digest(text: string): string { + return createHash("sha256").update(text).digest("hex"); +} + +function pathKind(path: string): Extract["kind"] { + try { + const info = statSync(path); + return info.isFile() + ? "file" + : info.isDirectory() + ? "directory" + : "other"; + } catch { + return "missing"; + } +} + +const activeTracker = new AsyncLocalStorage(); + +export class FrontendInputTracker { + private readonly probes = new Map(); + private stable = true; + + run(fn: () => T): T { + const parent = activeTracker.getStore(); + if (parent === undefined || parent === this) return activeTracker.run(this, fn); + return activeTracker.run(this, () => { + const result = fn(); + for (const probe of this.probes.values()) parent.record(probe); + return result; + }); + } + + record(probe: FrontendInputProbe): void { + const key = `${probe.op}\0${probe.path}`; + const previous = this.probes.get(key); + if (previous !== undefined && JSON.stringify(previous) !== JSON.stringify(probe)) { + // An input changed while this frontend was running. The build may + // still finish normally, but it is not safe to publish as a cache key. + this.stable = false; + } + this.probes.set(key, probe); + if (probe.op === "file") { + // A successful content read supersedes earlier existence probes for the + // same path. Retaining both would make the post-build stability check + // reject a perfectly stable input (`kind` vs `file` are two views of + // the same current file, not two historical states). + this.probes.delete(`kind\0${probe.path}`); + } + } + + snapshot(): FrontendInputSnapshot { + return { + version: 1, + probes: [...this.probes.values()].sort((a, b) => + a.path === b.path ? a.op.localeCompare(b.op) : a.path.localeCompare(b.path) + ), + stable: this.stable, + }; + } +} + +export function frontendInputTrackingActive(): boolean { + return activeTracker.getStore() !== undefined; +} + +function record(probe: FrontendInputProbe): void { + activeTracker.getStore()?.record(probe); +} + +export function trackedReadFile(path: string): string | null { + path = resolve(path); + try { + const text = readFileSync(path, "utf8"); + record({ op: "file", path, digest: digest(text) }); + return text; + } catch { + record({ op: "kind", path, kind: pathKind(path) }); + return null; + } +} + +export function trackedFileExists(path: string): boolean { + path = resolve(path); + let exists = false; + try { + exists = statSync(path).isFile(); + } catch { + // The exact failed candidate is part of the resolution answer. + } + record({ op: "kind", path, kind: pathKind(path) }); + return exists; +} + +export function trackedDirectoryExists(path: string): boolean { + path = resolve(path); + let exists = false; + try { + exists = statSync(path).isDirectory(); + } catch { + // The exact failed candidate is part of the resolution answer. + } + record({ op: "kind", path, kind: pathKind(path) }); + return exists; +} + +export function trackedExists(path: string): boolean { + path = resolve(path); + const exists = existsSync(path); + record({ op: "kind", path, kind: pathKind(path) }); + return exists; +} + +export function trackedRealpath(path: string): string | null { + path = resolve(path); + try { + const target = realpathSync(path); + record({ op: "realpath", path, target }); + return target; + } catch { + record({ op: "realpath", path, target: null }); + return null; + } +} + +export function trackedAccessibleEntries( + path: string, +): { files: string[]; directories: string[] } | null { + path = resolve(path); + try { + const files: string[] = []; + const directories: string[] = []; + for (const entry of readdirSync(path, { withFileTypes: true })) { + const kind = entry.isSymbolicLink() ? pathKind(resolve(path, entry.name)) : null; + if (entry.isFile() || kind === "file") files.push(entry.name); + if (entry.isDirectory() || kind === "directory") directories.push(entry.name); + } + files.sort(); + directories.sort(); + const answer = { files, directories }; + record({ op: "entries", path, ...answer }); + return answer; + } catch { + record({ op: "kind", path, kind: pathKind(path) }); + return null; + } +} + +/** Re-run every recorded probe against the current filesystem. */ +export function frontendInputsStillMatch(snapshot: FrontendInputSnapshot): boolean { + if (snapshot.version !== 1 || snapshot.stable !== true || !Array.isArray(snapshot.probes)) { + return false; + } + return snapshot.probes.every((probe) => { + switch (probe.op) { + case "file": { + try { + return digest(readFileSync(probe.path, "utf8")) === probe.digest; + } catch { + return false; + } + } + case "kind": + return pathKind(probe.path) === probe.kind; + case "entries": { + try { + const files: string[] = []; + const directories: string[] = []; + for (const entry of readdirSync(probe.path, { withFileTypes: true })) { + const kind = entry.isSymbolicLink() ? pathKind(resolve(probe.path, entry.name)) : null; + if (entry.isFile() || kind === "file") files.push(entry.name); + if (entry.isDirectory() || kind === "directory") directories.push(entry.name); + } + files.sort(); + directories.sort(); + return JSON.stringify(files) === JSON.stringify(probe.files) && + JSON.stringify(directories) === JSON.stringify(probe.directories); + } catch { + return false; + } + } + case "realpath": { + try { + return realpathSync(probe.path) === probe.target; + } catch { + return probe.target === null; + } + } + } + }); +} + +/** Pure validator used by the persistent-cache reader before any path probes. */ +export function validFrontendInputSnapshot(snapshot: unknown): snapshot is FrontendInputSnapshot { + if (snapshot === null || typeof snapshot !== "object") return false; + const candidate = snapshot as Partial; + if (candidate.version !== 1 || candidate.stable !== true || !Array.isArray(candidate.probes)) return false; + return candidate.probes.every((probe) => { + if (probe === null || typeof probe !== "object") return false; + const value = probe as Partial; + if (typeof value.path !== "string" || typeof value.op !== "string") return false; + switch (value.op) { + case "file": + return typeof value.digest === "string" && /^[0-9a-f]{64}$/.test(value.digest); + case "kind": + return value.kind === "file" || value.kind === "directory" || value.kind === "other" || + value.kind === "missing"; + case "entries": + return Array.isArray(value.files) && value.files.every((entry) => typeof entry === "string") && + Array.isArray(value.directories) && value.directories.every((entry) => typeof entry === "string"); + case "realpath": + return value.target === null || typeof value.target === "string"; + default: + return false; + } + }); +} diff --git a/packages/compiler/src/frontend/lowering/lower-builtins.ts b/packages/compiler/src/frontend/lowering/lower-builtins.ts index 908366ded..7d1f36f25 100644 --- a/packages/compiler/src/frontend/lowering/lower-builtins.ts +++ b/packages/compiler/src/frontend/lowering/lower-builtins.ts @@ -2,7 +2,6 @@ * crypto, child_process spawn/spawnSync and child/stats/spawn-result * methods), JSON.parse/stringify, process properties/methods and * process.env access, and console.log detection. */ -import { readFileSync } from "node:fs"; import { builtinModules } from "node:module"; import { dirname, resolve } from "node:path"; import * as ts from "../ts7/adapter.js"; @@ -12,6 +11,7 @@ import { canonicalBuiltinModule, isJsSourceFile, locOf, requireSpecOf } from ".. import { isRelativeSpecifier } from "../shared.js"; import { probeNodeRequireRefusal } from "../npm.js"; import { isNpmStaticPackage } from "../npm-static.js"; +import { trackedReadFile } from "../input-tracker.js"; import { invalidJsonModuleDiag, requiresDynamicImportDiag } from "../../diagnostics/diagnostic.js"; import { BuiltinModuleFn, @@ -408,12 +408,7 @@ function lowerBuiltinOptionalDefault( const abs = spec.startsWith("/") ? spec : resolve(dirname(cr.baseFile.fileName), spec); - let text: string | null = null; - try { - text = readFileSync(abs, "utf8"); - } catch { - /* the fence below speaks */ - } + const text = trackedReadFile(abs); if (text === null) { L.noLowering( `createRequire's require of '${spec}' (no file at ${abs})`, diff --git a/packages/compiler/src/frontend/npm-static-rewrite.ts b/packages/compiler/src/frontend/npm-static-rewrite.ts index da1a09e51..5bf6e358d 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite.ts @@ -73,11 +73,11 @@ * untouched (null): its import-site errors stay, and the frontend's * offender attribution degrades the PACKAGE to the island with a note. */ -import { existsSync, readFileSync, statSync } from "node:fs"; import { dirname, join, resolve as resolvePath } from "node:path"; import ts from "typescript5"; import { cjsLexedExportsOf, cjsLexerVisibleNames } from "./cjs-lexer.js"; import { resolveExports } from "./npm.js"; +import { trackedDirectoryExists, trackedFileExists, trackedReadFile } from "./input-tracker.js"; /** True when `e` is exactly the `exports` identifier. */ function isExportsIdent(e: ts.Expression): boolean { @@ -233,26 +233,23 @@ function definePropertyExportOf( function resolveCjsBase(base: string): string | null { const candidates = [base, `${base}.js`, `${base}.cjs`]; for (const c of candidates) { - try { - if (existsSync(c) && statSync(c).isFile() && /\.(js|cjs)$/.test(c)) return c; - } catch { - /* keep probing */ - } + if (trackedFileExists(c) && /\.(js|cjs)$/.test(c)) return c; } try { - if (existsSync(base) && statSync(base).isDirectory()) { + if (trackedDirectoryExists(base)) { const pkgPath = resolvePath(base, "package.json"); - if (existsSync(pkgPath)) { - const main = (JSON.parse(readFileSync(pkgPath, "utf8")) as { main?: unknown }).main; + const pkgText = trackedReadFile(pkgPath); + if (pkgText !== null) { + const main = (JSON.parse(pkgText) as { main?: unknown }).main; if (typeof main === "string") { const m = resolvePath(base, main); for (const c of [m, `${m}.js`, `${m}.cjs`, resolvePath(m, "index.js")]) { - if (existsSync(c) && statSync(c).isFile() && /\.(js|cjs)$/.test(c)) return c; + if (trackedFileExists(c) && /\.(js|cjs)$/.test(c)) return c; } } } const idx = resolvePath(base, "index.js"); - if (existsSync(idx)) return idx; + if (trackedFileExists(idx)) return idx; } } catch { /* unresolved */ @@ -281,13 +278,14 @@ function resolveBareRequireCjs(fromFile: string, spec: string): string | null { for (let dir = dirname(fromFile); ; ) { const pkgDir = join(dir, "node_modules", name); try { - if (existsSync(pkgDir) && statSync(pkgDir).isDirectory()) { + if (trackedDirectoryExists(pkgDir)) { const pkgPath = join(pkgDir, "package.json"); let exports: unknown; try { - exports = existsSync(pkgPath) - ? (JSON.parse(readFileSync(pkgPath, "utf8")) as { exports?: unknown }).exports - : undefined; + const pkgText = trackedReadFile(pkgPath); + exports = pkgText === null + ? undefined + : (JSON.parse(pkgText) as { exports?: unknown }).exports; } catch { return null; } @@ -329,9 +327,10 @@ function requireTargetEsModuleStamped(fromFile: string, spec: string, depth = 0) // the nearest package.json "type" decides the .js format for (let dir = dirname(file); ; ) { const pkgPath = join(dir, "package.json"); - if (existsSync(pkgPath)) { + const pkgText = trackedReadFile(pkgPath); + if (pkgText !== null) { try { - if ((JSON.parse(readFileSync(pkgPath, "utf8")) as { type?: unknown }).type === "module") return true; + if ((JSON.parse(pkgText) as { type?: unknown }).type === "module") return true; } catch { /* unreadable — treat as CJS */ } @@ -344,7 +343,7 @@ function requireTargetEsModuleStamped(fromFile: string, spec: string, depth = 0) } let src: string; try { - src = readFileSync(file, "utf8"); + src = trackedReadFile(file)!; } catch { return false; } @@ -366,7 +365,7 @@ function starTargetNames(file: string): Set { try { return cjsLexerVisibleNames( file, - (f) => readFileSync(f, "utf8"), + (f) => trackedReadFile(f)!, (from, spec) => resolveRelativeCjs(from, spec), ); } catch { diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 992cd9d72..09788d3bd 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -54,10 +54,10 @@ * the state, so a flagless compile after a flagged one sees a clean * slate. */ -import { existsSync, readFileSync, realpathSync } from "node:fs"; import { dirname } from "node:path"; import { rewriteBundlerCjsExports } from "./npm-static-rewrite.js"; import { npmPackageNameOf, registerWorkspacePackage, workspacePackageOfPath } from "./shared.js"; +import { trackedExists, trackedReadFile, trackedRealpath } from "./input-tracker.js"; let activePackages: ReadonlySet = new Set(); @@ -129,11 +129,9 @@ const realpathProbed = new Set(); function registerWorkspaceRealpath(pkg: string, nmDir: string): void { if (realpathProbed.has(nmDir)) return; realpathProbed.add(nmDir); - try { - const real = realpathSync(nmDir).split("\\").join("/"); - if (real !== nmDir && !real.includes("/node_modules/")) registerWorkspacePackage(pkg, real); - } catch { - /* dangling symlink / missing dir — nothing to register */ + const real = trackedRealpath(nmDir)?.split("\\").join("/"); + if (real !== undefined && real !== nmDir && !real.includes("/node_modules/")) { + registerWorkspacePackage(pkg, real); } } @@ -287,7 +285,7 @@ function packageIsUntyped(path: string): boolean { if (hit !== undefined) return hit; let untyped = true; try { - const pkg = JSON.parse(readFileSync(`${pkgDir}/package.json`, "utf8")) as Record; + const pkg = JSON.parse(trackedReadFile(`${pkgDir}/package.json`)!) as Record; if (pkg["types"] !== undefined || pkg["typings"] !== undefined) untyped = false; if (untyped && typeof pkg["exports"] === "object" && pkg["exports"] !== null) { // a "types" condition anywhere inside exports is a claim too @@ -296,13 +294,13 @@ function packageIsUntyped(path: string): boolean { } catch { /* no package.json — keep probing */ } - if (untyped && existsSync(`${pkgDir}/index.d.ts`)) untyped = false; + if (untyped && trackedExists(`${pkgDir}/index.d.ts`)) untyped = false; if (untyped) { // the @types twin, hoisted anywhere up the realm chain const mangled = mangledTypesName(dirName); for (let dir = dirname(pkgDir); ; ) { const parent = dirname(dir); - if (existsSync(`${dir}/node_modules/@types/${mangled}/package.json`) || existsSync(`${dir}/@types/${mangled}/package.json`)) { + if (trackedExists(`${dir}/node_modules/@types/${mangled}/package.json`) || trackedExists(`${dir}/@types/${mangled}/package.json`)) { untyped = false; break; } @@ -344,7 +342,7 @@ export function npmStaticFsShadow(): NpmStaticFsShadow | null { if ( (path.endsWith(".js") || path.endsWith(".cjs")) && isNodeModulesPathNorm(path) && - !existsSync(path.replace(/\.(js|cjs)$/, ".d.ts")) && // a sibling .d.ts types this very file + !trackedExists(path.replace(/\.(js|cjs)$/, ".d.ts")) && // a sibling .d.ts types this very file packageIsUntyped(path) ) { return "module.exports = (() => { let u; return u; })();\n"; @@ -354,7 +352,7 @@ export function npmStaticFsShadow(): NpmStaticFsShadow | null { if (target.viaTypes) return undefined; if (path.endsWith("/package.json") || path.endsWith("\\package.json")) { try { - return npmStaticTransformPkgJsonText(readFileSync(path, "utf8")); + return npmStaticTransformPkgJsonText(trackedReadFile(path)!); } catch { return undefined; } @@ -373,7 +371,7 @@ export function npmStaticFsShadow(): NpmStaticFsShadow | null { if (hit !== undefined) return hit ?? undefined; let rewritten: string | null = null; try { - const answer = rewriteBundlerCjsExports(readFileSync(path, "utf8"), path); + const answer = rewriteBundlerCjsExports(trackedReadFile(path)!, path); if (answer !== null && typeof answer === "object") { reportNpmStaticOffender(target.pkg, answer.degrade); } else { @@ -446,7 +444,7 @@ export function npmStaticIneligibleReason( if (jsEntry === null) return "no runtime JS entry resolves"; let source: string; try { - source = readFileSync(jsEntry, "utf8"); + source = trackedReadFile(jsEntry)!; } catch { return `its runtime entry ${jsEntry} cannot be read`; } diff --git a/packages/compiler/src/frontend/npm.ts b/packages/compiler/src/frontend/npm.ts index 8b919cd08..142e699d7 100644 --- a/packages/compiler/src/frontend/npm.ts +++ b/packages/compiler/src/frontend/npm.ts @@ -85,10 +85,10 @@ * lives under its key (requireHelperOriginOf follows the import and * re-export hops). */ -import { readFileSync, realpathSync } from "node:fs"; import { dirname, extname, join, resolve } from "node:path"; import ts from "typescript5"; import { cjsLexedExportsOf } from "./cjs-lexer.js"; +import { trackedDirectoryExists, trackedFileExists, trackedReadFile, trackedRealpath } from "./input-tracker.js"; export type EmbeddedFormat = "esm" | "cjs" | "json"; @@ -732,21 +732,11 @@ interface Host { const realHost: Host = { readFile: (path) => { - try { - return readFileSync(path, "utf8"); - } catch { - return null; - } - }, - isFile: (path) => ts.sys.fileExists(path), - isDirectory: (path) => ts.sys.directoryExists(path), - realpath: (path) => { - try { - return realpathSync(path); - } catch { - return path; - } + return trackedReadFile(path); }, + isFile: (path) => trackedFileExists(path), + isDirectory: (path) => trackedDirectoryExists(path), + realpath: (path) => trackedRealpath(path) ?? path, }; interface PkgJson { diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 89e0e58d3..8a4c41e1c 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -40,7 +40,6 @@ * that path (no snapshot pins it). */ import { builtinModules } from "node:module"; -import { statSync } from "node:fs"; import { dirname, resolve } from "node:path"; import * as ts from "./ts7/adapter.js"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; @@ -76,6 +75,7 @@ import { unsupportedModuleFeatureOf, workspacePackageOfPath, } from "./shared.js"; +import { trackedFileExists } from "./input-tracker.js"; const BASE_OPTIONS: ts.Ts7CompilerOptions = { strict: true, @@ -401,9 +401,7 @@ export function loadProgram( ); } const declarationPath = resolve(file); - try { - if (!statSync(declarationPath).isFile()) throw new Error("not a file"); - } catch { + if (!trackedFileExists(declarationPath)) { throw new TypeError( `external type declaration for ${JSON.stringify(specifier)} does not name a readable file: ${declarationPath}`, ); diff --git a/packages/compiler/src/frontend/resolve.ts b/packages/compiler/src/frontend/resolve.ts index bdf2f0c4f..68958a953 100644 --- a/packages/compiler/src/frontend/resolve.ts +++ b/packages/compiler/src/frontend/resolve.ts @@ -14,33 +14,21 @@ * implementations and requires identical answers. Change 5.9.3's options * and these tables are wrong — that is what the suite is for. */ -import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; import { dirname, isAbsolute, join, resolve } from "node:path"; import { isNpmStaticPackage, npmStaticPackageOfPath, npmStaticTransformPkgJson } from "./npm-static.js"; import { provenanceEntryFor } from "./provenance-registry.js"; +import { trackedAccessibleEntries, trackedDirectoryExists, trackedExists, trackedFileExists, trackedReadFile, trackedRealpath } from "./input-tracker.js"; function isFile(path: string): boolean { - try { - return statSync(path).isFile(); - } catch { - return false; - } + return trackedFileExists(path); } function isDirectory(path: string): boolean { - try { - return statSync(path).isDirectory(); - } catch { - return false; - } + return trackedDirectoryExists(path); } function realpathOr(path: string): string { - try { - return realpathSync(path); - } catch { - return path; - } + return trackedRealpath(path) ?? path; } interface PkgJson { @@ -63,9 +51,9 @@ function pkgJsonOf(dir: string): PkgJson | null { if (cached === undefined) { const path = join(dir, "package.json"); let parsed: PkgJson | null = null; - if (existsSync(path)) { + if (trackedExists(path)) { try { - parsed = JSON.parse(readFileSync(path, "utf8")) as PkgJson; + parsed = JSON.parse(trackedReadFile(path)!) as PkgJson; } catch { parsed = null; } @@ -114,12 +102,7 @@ function expandWorkspacePattern(root: string, pattern: string): string[] { const next: string[] = []; for (const dir of dirs) { if (seg === "*") { - let entries: string[]; - try { - entries = readdirSync(dir); - } catch { - continue; - } + const entries = trackedAccessibleEntries(dir)?.directories ?? []; for (const e of entries) { if (e === "node_modules" || e.startsWith(".")) continue; const child = join(dir, e); @@ -463,7 +446,7 @@ function nearestPkgDir(dir: string): string | null { function nearestPackageConfigDir(dir: string): string | null { let d = dir; for (;;) { - if (existsSync(join(d, "package.json"))) return d; + if (trackedExists(join(d, "package.json"))) return d; const parent = dirname(d); if (parent === d) return null; d = parent; diff --git a/packages/compiler/src/frontend/ts7/program.ts b/packages/compiler/src/frontend/ts7/program.ts index 74c91b0dd..f319016f9 100644 --- a/packages/compiler/src/frontend/ts7/program.ts +++ b/packages/compiler/src/frontend/ts7/program.ts @@ -15,7 +15,7 @@ * 387 cold). createProgram() takes an optional shared host — the default * spawns a private one that dispose() closes. */ -import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { API } from "typescript/unstable/sync"; import type { @@ -28,6 +28,7 @@ import type { SourceFile } from "typescript/unstable/ast"; import { CheckerFacade } from "./checker.js"; import { enumKeyOf, ModuleDetectionKind, ModuleKind, ModuleResolutionKind, ScriptTarget } from "./enums.js"; import { tsgoPath } from "../shared.js"; +import { trackedAccessibleEntries, trackedDirectoryExists, trackedFileExists, trackedReadFile, trackedRealpath } from "../input-tracker.js"; /** The compiler options our createProgram accepts: TS7's CompilerOptions * shape (numeric enums for target/module/moduleResolution — the enums module @@ -116,18 +117,22 @@ export class Ts7Host { readFile: (fileName) => { const virtual = virtualFiles.get(tsgoPath(fileName)); if (virtual !== undefined) return virtual; - if (shadow === null) return undefined; - if (shadow.hideFile(fileName)) return null; - return shadow.readFile(fileName); + if (shadow !== null) { + if (shadow.hideFile(fileName)) return null; + const replacement = shadow.readFile(fileName); + if (replacement !== undefined) return replacement; + } + return trackedReadFile(fileName); }, fileExists: (fileName) => { if (virtualFiles.has(tsgoPath(fileName))) return true; if (shadow !== null && shadow.hideFile(fileName)) return false; - return undefined; + return trackedFileExists(fileName); }, - directoryExists: () => undefined, - realpath: (path) => (virtualFiles.has(tsgoPath(path)) ? path : undefined), - getAccessibleEntries: () => undefined, + directoryExists: (path) => trackedDirectoryExists(path), + realpath: (path) => + virtualFiles.has(tsgoPath(path)) ? path : (trackedRealpath(path) ?? path), + getAccessibleEntries: (path) => trackedAccessibleEntries(path) ?? { files: [], directories: [] }, }, }); } @@ -309,20 +314,16 @@ export function findConfigFile( * are the direct implementations. */ export const sys = { fileExists(path: string): boolean { - return existsSync(path) && statSync(path).isFile(); + return trackedFileExists(path); }, readFile(path: string): string | undefined { - try { - return readFileSync(path, "utf8"); - } catch { - return undefined; - } + return trackedReadFile(path) ?? undefined; }, writeFile(path: string, data: string): void { writeFileSync(path, data); }, directoryExists(path: string): boolean { - return existsSync(path) && statSync(path).isDirectory(); + return trackedDirectoryExists(path); }, getCurrentDirectory(): string { return process.cwd(); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 948d7a4f6..702c84eb9 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,6 +1,6 @@ import { mkdir, rm, writeFile } from "node:fs/promises"; import { basename, dirname, join, resolve } from "node:path"; -import { CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, resolveCc, targetPlatform } from "./backend/cc.js"; +import { buildCacheRoot, CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform } from "./backend/cc.js"; import { emitModule } from "./backend/emission/emitter.js"; import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js"; import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; @@ -27,12 +27,14 @@ import { validateModule } from "./ir/validate.js"; import { canonicalBuiltinModule, checkPreflight, isNodeTypesPath, loadProgram, locOf, requiresOf, resolveNpmImport, type LoadResult } from "./frontend/program.js"; import { npmStaticIneligibleReason, npmStaticOffenders, npmStaticPackageOfPath } from "./frontend/npm-static.js"; import { provenanceSources } from "./frontend/provenance-registry.js"; -import { resolveBareModule } from "./frontend/resolve.js"; +import { clearResolveCaches, resolveBareModule } from "./frontend/resolve.js"; import { isJsSourceFileName, isRelativeSpecifier } from "./frontend/shared.js"; import { lowerToIr, type LowerOptions, type LowerResult } from "./frontend/lowering/lowerer.js"; import type { CoverageInput, NpmStaticStatus } from "./coverage/report.js"; import { loadFfiProfile, type FfiProfile } from "./ffi/profile.js"; import { hasForeignFfiCallback } from "./backend/ffi-callbacks.js"; +import { FrontendInputTracker, trackedReadFile } from "./frontend/input-tracker.js"; +import { libraryFrontendImplementationFingerprint, publishEarlyLibraryCache, readEarlyLibraryCache, type EarlyLibraryCacheOptions, type EarlyLibraryCachePublish, type EarlyLibraryNativeFeatures } from "./library/early-cache.js"; export const VERSION = "0.0.1"; @@ -517,6 +519,11 @@ function runFrontend( npmStatic?: readonly string[] | "auto" | "lib", externalTypes?: Readonly>, ): Frontend { + // Resolver package/workspace metadata is intentionally shared across the + // several load attempts of ONE auto-detection fixpoint, but never across + // separate compiles in a long-lived process. A cache miss must observe + // package.json edits before it can publish a new early-library entry. + clearResolveCaches(); const statuses: NpmStaticStatus[] = []; const npmSites = new Map(); const judged = new Set(); @@ -1499,7 +1506,77 @@ function mergeSidecarIntSlots( return { ok: true, config: cfg }; } +function libraryNativeFeatures( + mod: IrModule, + backend: "c" | "llvm", +): EarlyLibraryNativeFeatures { + return { + backend, + regex: moduleUsesRegex(mod), + assert: moduleUsesAssert(mod), + inspect: moduleUsesInspect(mod), + symbol: moduleUsesSymbol(mod), + searchParams: moduleUsesSearchParams(mod), + emitter: moduleUsesEmitter(mod), + zlib: moduleUsesZlib(mod), + copying: moduleUsesCopying(mod), + textDecoderLegacy: moduleUsesLegacyTextDecoder(mod), + }; +} + +function libraryLocalizeSymbols(profile: LibraryProfile): string[] | undefined { + return profile.localizeRuntime + ? [ + profile.initSymbol, + profile.sinkRegisterSymbol, + ...(profile.collectSymbol !== null ? [profile.collectSymbol] : []), + ...(profile.resultResetSymbol !== null ? [profile.resultResetSymbol] : []), + ...(profile.callbackRegisterSymbol !== null ? [profile.callbackRegisterSymbol] : []), + ...(profile.sidecar !== null + ? [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol] + : []), + ...profile.exports.map((entry) => entry.symbol), + ] + : undefined; +} + +async function compileLibraryNative( + profile: LibraryProfile, + cPath: string, + archivePath: string, + sanitize: boolean, + features: EarlyLibraryNativeFeatures, +): Promise { + const localizeSymbols = libraryLocalizeSymbols(profile); + await compileLibArchive({ + cPath, + outPath: archivePath, + cacheIdentity: "scriptc-generated-library-v1", + sanitize, + optimization: profile.optimization, + ...(localizeSymbols !== undefined ? { localizeSymbols } : {}), + ...(profile.instancePerThread ? { threadInstances: true } : {}), + regex: features.regex, + assert: features.assert, + inspect: features.inspect, + symbol: features.symbol, + searchParams: features.searchParams, + emitter: features.emitter, + zlib: features.zlib, + copying: features.copying, + textDecoderLegacy: features.textDecoderLegacy, + }); +} + export async function compileLibrary(opts: CompileLibraryOptions): Promise { + const frontendInputs = new FrontendInputTracker(); + return frontendInputs.run(() => compileLibraryTracked(opts, frontendInputs)); +} + +async function compileLibraryTracked( + opts: CompileLibraryOptions, + frontendInputs: FrontendInputTracker, +): Promise { const timingOn = process.env["SCRIPTC_TIMING"] === "1"; const timingStart = performance.now(); let timingLast = timingStart; @@ -1525,6 +1602,23 @@ export async function compileLibrary(opts: CompileLibraryOptions): Promise { + const path = join(directory, name); + trackedReadFile(path); + }); + } + if (directory === profileDir || dirname(directory) === directory) break; + } + const archivePath = opts.outPath ?? join( + opts.outDir, + `${basename(entryPath).replace(/\.(ts|js|mjs|cjs)$/, "")}.lib.a`, + ); // Mobile-target admission first — a pure env/host check, so a refused // pairing never reaches toolchain discovery. iOS targets (device and @@ -1593,6 +1687,48 @@ export async function compileLibrary(opts: CompileLibraryOptions): Promise e.symbol), - ] - : undefined; - await compileLibArchive({ + const nativeFeatures = libraryNativeFeatures(mod, profile.emission); + await compileLibraryNative( + profile, cPath, - outPath: archivePath, - cacheIdentity: "scriptc-generated-library-v1", - sanitize: opts.sanitize ?? false, - optimization: profile.optimization, - ...(localizeSymbols !== undefined ? { localizeSymbols } : {}), - ...(profile.instancePerThread ? { threadInstances: true } : {}), - regex: moduleUsesRegex(mod), - assert: moduleUsesAssert(mod), - inspect: moduleUsesInspect(mod), - symbol: moduleUsesSymbol(mod), - searchParams: moduleUsesSearchParams(mod), - emitter: moduleUsesEmitter(mod), - zlib: moduleUsesZlib(mod), - copying: moduleUsesCopying(mod), - textDecoderLegacy: moduleUsesLegacyTextDecoder(mod), - }); + archivePath, + opts.sanitize ?? false, + nativeFeatures, + ); timing("native-archive"); // The sidecar lands beside the compiled object, written by the same @@ -1895,6 +2004,16 @@ export async function compileLibrary(opts: CompileLibraryOptions): Promise undefined); + await pruneBuildCache(cacheRoot); + timing("early-cache-publish"); timing("complete"); return { ok: true, diff --git a/packages/compiler/src/library/early-cache.test.ts b/packages/compiler/src/library/early-cache.test.ts new file mode 100644 index 000000000..3d73478d3 --- /dev/null +++ b/packages/compiler/src/library/early-cache.test.ts @@ -0,0 +1,215 @@ +import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { FrontendInputTracker, trackedFileExists, trackedReadFile } from "../frontend/input-tracker.js"; +import { + publishEarlyLibraryCache, + readEarlyLibraryCache, + type EarlyLibraryCacheOptions, +} from "./early-cache.js"; + +const scratch: string[] = []; + +afterEach(async () => { + await Promise.all(scratch.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +async function fixture(): Promise<{ + root: string; + options: EarlyLibraryCacheOptions; + source: string; + missing: string; + cPath: string; + irPath: string; + sidecarPath: string; +}> { + const dir = await mkdtemp(join(tmpdir(), "scriptc-early-cache-")); + scratch.push(dir); + const source = join(dir, "entry.ts"); + const profilePath = join(dir, "profile.json"); + const cPath = join(dir, "entry.lib.ll"); + const irPath = join(dir, "entry.lib.ir.json"); + const sidecarPath = join(dir, "entry.lib.a.contract.json"); + await Promise.all([ + writeFile(source, "export function value(): number { return 1; }\n"), + writeFile(profilePath, "{}\n"), + writeFile(cPath, "; generated llvm\n"), + writeFile(irPath, "{\"irVersion\":6}\n"), + writeFile(sidecarPath, "{\"contract\":true}\n"), + ]); + return { + root: join(dir, "cache"), + source, + missing: join(dir, "missing.ts"), + cPath, + irPath, + sidecarPath, + options: { + profilePath, + profileBytes: new TextEncoder().encode("{}\n"), + entryPath: source, + outDir: dir, + emitIr: true, + sanitize: false, + target: "test", + compiler: ["clang"], + implementation: "test-implementation", + }, + }; +} + +test("early library cache restores generated artifacts and metadata", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => { + trackedReadFile(f.source); + trackedFileExists(f.missing); + }); + const native = { + backend: "llvm" as const, + regex: false, + assert: true, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }; + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native, + frontend: tracker.snapshot(), + }); + await Promise.all([rm(f.cPath), rm(f.irPath), rm(f.sidecarPath)]); + + const hit = await readEarlyLibraryCache(f.root, f.options, null); + expect(hit).not.toBeNull(); + expect(hit?.native).toEqual(native); + expect(await readFile(f.cPath, "utf8")).toBe("; generated llvm\n"); + expect(await readFile(f.irPath, "utf8")).toContain("irVersion"); + expect(await readFile(f.sidecarPath, "utf8")).toContain("contract"); +}); + +test("early library cache misses on source edits and newly-resolved candidates", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => { + trackedReadFile(f.source); + trackedFileExists(f.missing); + }); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }); + + await writeFile(f.source, "export function value(): number { return 2; }\n"); + expect(await readEarlyLibraryCache(f.root, f.options, null)).toBeNull(); + await writeFile(f.cPath, "; generated llvm v2\n"); + const editedTracker = new FrontendInputTracker(); + editedTracker.run(() => { + trackedReadFile(f.source); + trackedFileExists(f.missing); + }); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: editedTracker.snapshot(), + }); + expect((await readEarlyLibraryCache(f.root, f.options, null))?.cPath).toBe(f.cPath); + expect(await readFile(f.cPath, "utf8")).toBe("; generated llvm v2\n"); + + await writeFile(f.source, "export function value(): number { return 1; }\n"); + await writeFile(f.missing, "export const appeared = true;\n"); + expect(await readEarlyLibraryCache(f.root, f.options, null)).toBeNull(); +}); + +test("early library cache rejects corrupted artifacts and metadata", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }); + const earlyRoot = join(f.root, "early-lib"); + const [key] = await readdir(earlyRoot); + await writeFile(join(earlyRoot, key!, "program.tu"), "corrupt\n"); + expect(await readEarlyLibraryCache(f.root, f.options, null)).toBeNull(); + + const stamp = join(earlyRoot, key!, "stamp.json"); + await writeFile(stamp, "{\"version\":1}\n"); + expect(await readEarlyLibraryCache(f.root, f.options, null)).toBeNull(); +}); + +test("disabled early library cache performs no reads or writes", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + const publish = { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm" as const, + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }; + await publishEarlyLibraryCache(null, f.options, publish); + await expect(readdir(f.root)).rejects.toThrow(); + expect(await readEarlyLibraryCache(null, f.options, null)).toBeNull(); +}); diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts new file mode 100644 index 000000000..46cf03c9e --- /dev/null +++ b/packages/compiler/src/library/early-cache.ts @@ -0,0 +1,308 @@ +import { createHash } from "node:crypto"; +import { chmod, copyFile, mkdir, readFile, readdir, rename, rm, utimes, writeFile } from "node:fs/promises"; +import { basename, dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { compilerReleaseVersion } from "./sidecar.js"; +import { frontendInputsStillMatch, validFrontendInputSnapshot, type FrontendInputSnapshot } from "../frontend/input-tracker.js"; + +interface CachedLibraryFile { + name: string; + digest: string; +} + +interface EarlyLibraryCacheStamp { + version: 1; + key: string; + frontend: FrontendInputSnapshot; + files: { + translationUnit: CachedLibraryFile; + ir: CachedLibraryFile | null; + sidecar: CachedLibraryFile | null; + }; + native: { + backend: "c" | "llvm"; + regex: boolean; + assert: boolean; + inspect: boolean; + symbol: boolean; + searchParams: boolean; + emitter: boolean; + zlib: boolean; + copying: boolean; + textDecoderLegacy: boolean; + }; + integrity: string; +} + +export interface EarlyLibraryNativeFeatures { + backend: "c" | "llvm"; + regex: boolean; + assert: boolean; + inspect: boolean; + symbol: boolean; + searchParams: boolean; + emitter: boolean; + zlib: boolean; + copying: boolean; + textDecoderLegacy: boolean; +} + +export interface EarlyLibraryCacheOptions { + profilePath: string; + profileBytes: Uint8Array; + entryPath: string; + outDir: string; + outPath?: string; + emitIr: boolean; + sanitize: boolean; + target: string; + compiler: string[]; + implementation: string; +} + +export interface EarlyLibraryCacheHit { + cPath: string; + irPath?: string; + sidecarPath?: string; + native: EarlyLibraryNativeFeatures; +} + +export interface EarlyLibraryCachePublish extends EarlyLibraryCacheHit { + frontend: FrontendInputSnapshot; +} + +function validNativeFeatures(value: unknown): value is EarlyLibraryNativeFeatures { + if (value === null || typeof value !== "object") return false; + const native = value as Partial; + return (native.backend === "c" || native.backend === "llvm") && + [ + native.regex, + native.assert, + native.inspect, + native.symbol, + native.searchParams, + native.emitter, + native.zlib, + native.copying, + native.textDecoderLegacy, + ].every((flag) => typeof flag === "boolean"); +} + +function digest(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function cacheKey(options: EarlyLibraryCacheOptions): string { + return createHash("sha256") + .update("early-library-v1\0") + .update(compilerReleaseVersion()).update("\0") + .update(resolve(options.profilePath)).update("\0") + .update(options.profileBytes).update("\0") + .update(resolve(options.entryPath)).update("\0") + .update(resolve(options.outDir)).update("\0") + .update(options.outPath === undefined ? "" : resolve(options.outPath)).update("\0") + .update(options.emitIr ? "emit-ir" : "no-ir").update("\0") + .update(options.sanitize ? "sanitize" : "plain").update("\0") + .update(options.target).update("\0") + .update(options.compiler.join("\x1f")).update("\0") + .update(options.implementation) + .digest("hex"); +} + +/** + * Release versions separate published compilers. A source checkout keeps the + * package version unchanged while a branch is under test, so the built + * implementation directory also joins the key. Published packages execute + * the same `dist` tree and therefore get the same protection from partial or + * mixed installs without hashing the TypeScript source tree beside it. + */ +export async function libraryFrontendImplementationFingerprint(): Promise { + const moduleDir = dirname(fileURLToPath(import.meta.url)); + const implementationRoot = resolve(moduleDir, "..", ".."); + const files: string[] = []; + const walk = async (directory: string): Promise => { + const entries = await readdir(directory, { withFileTypes: true }); + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + const path = join(directory, entry.name); + if (entry.isDirectory()) await walk(path); + else if (entry.isFile()) files.push(path); + } + }; + await walk(implementationRoot); + const hash = createHash("sha256").update("scriptc-frontend-implementation-v1\0"); + for (const file of files) { + hash.update(relative(implementationRoot, file)).update("\0").update(await readFile(file)).update("\0"); + } + return hash.digest("hex"); +} + +function stampPath(root: string, options: EarlyLibraryCacheOptions): string { + return join(root, "early-lib", cacheKey(options), "stamp.json"); +} + +function stampIntegrity(stamp: Omit): string { + return createHash("sha256") + .update("early-library-stamp-v1\0") + .update(JSON.stringify(stamp)) + .digest("hex"); +} + +function outputPaths(options: EarlyLibraryCacheOptions, backend: "c" | "llvm"): { + cPath: string; + irPath: string; +} { + const stem = basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, ""); + return { + cPath: join(options.outDir, `${stem}.lib.${backend === "llvm" ? "ll" : "c"}`), + irPath: join(options.outDir, `${stem}.lib.ir.json`), + }; +} + +function sidecarOutputPath(options: EarlyLibraryCacheOptions, configured: string | null): string { + const archivePath = options.outPath ?? join( + options.outDir, + `${basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, "")}.lib.a`, + ); + return configured !== null ? resolve(dirname(archivePath), configured) : `${archivePath}.contract.json`; +} + +async function readCachedFile(path: string, expected: string): Promise { + try { + const bytes = await readFile(path); + return digest(bytes) === expected ? bytes : null; + } catch { + return null; + } +} + +async function installBytes(bytes: Uint8Array, destination: string): Promise { + await mkdir(dirname(destination), { recursive: true }); + const tmp = join(dirname(destination), `.scriptc-early-hit-${process.pid}-${Math.random().toString(36).slice(2)}`); + try { + await writeFile(tmp, bytes, { mode: 0o600 }); + await chmod(tmp, 0o666 & ~process.umask()); + await rename(tmp, destination); + } finally { + await rm(tmp, { force: true }).catch(() => undefined); + } +} + +export async function readEarlyLibraryCache( + root: string | null, + options: EarlyLibraryCacheOptions, + sidecarConfiguredPath: string | null | undefined, +): Promise { + if (root === null) return null; + const path = stampPath(root, options); + try { + const stamp = JSON.parse(await readFile(path, "utf8")) as EarlyLibraryCacheStamp; + const { integrity, ...unsigned } = stamp; + if ( + stamp.version !== 1 || + stamp.key !== cacheKey(options) || + !validFrontendInputSnapshot(stamp.frontend) || + !validNativeFeatures(stamp.native) || + stamp.files?.translationUnit?.name !== "program.tu" || + !/^[0-9a-f]{64}$/.test(stamp.files.translationUnit.digest) || + (stamp.files.ir !== null && ( + stamp.files.ir?.name !== "program.ir.json" || !/^[0-9a-f]{64}$/.test(stamp.files.ir.digest) + )) || + (stamp.files.sidecar !== null && ( + stamp.files.sidecar?.name !== "contract.json" || !/^[0-9a-f]{64}$/.test(stamp.files.sidecar.digest) + )) || + stampIntegrity(unsigned) !== integrity || + !frontendInputsStillMatch(stamp.frontend) || + (stamp.files.ir !== null) !== options.emitIr || + (stamp.files.sidecar !== null) !== (sidecarConfiguredPath !== undefined) + ) return null; + + const directory = dirname(path); + const [translationUnit, ir, sidecar] = await Promise.all([ + readCachedFile(join(directory, stamp.files.translationUnit.name), stamp.files.translationUnit.digest), + stamp.files.ir === null + ? Promise.resolve(null) + : readCachedFile(join(directory, stamp.files.ir.name), stamp.files.ir.digest), + stamp.files.sidecar === null + ? Promise.resolve(null) + : readCachedFile(join(directory, stamp.files.sidecar.name), stamp.files.sidecar.digest), + ]); + if ( + translationUnit === null || + stamp.files.ir !== null && ir === null || + stamp.files.sidecar !== null && sidecar === null + ) return null; + + const paths = outputPaths(options, stamp.native.backend); + await installBytes(translationUnit, paths.cPath); + if (ir !== null) await installBytes(ir, paths.irPath); + let sidecarPath: string | undefined; + if (sidecar !== null) { + sidecarPath = sidecarOutputPath(options, sidecarConfiguredPath ?? null); + await installBytes(sidecar, sidecarPath); + } + const now = new Date(); + await utimes(path, now, now).catch(() => undefined); + return { + cPath: paths.cPath, + native: stamp.native, + ...(ir !== null ? { irPath: paths.irPath } : {}), + ...(sidecarPath !== undefined ? { sidecarPath } : {}), + }; + } catch { + return null; + } +} + +export async function publishEarlyLibraryCache( + root: string | null, + options: EarlyLibraryCacheOptions, + result: EarlyLibraryCachePublish, +): Promise { + if (root === null || !result.frontend.stable) return; + const destination = dirname(stampPath(root, options)); + const parent = dirname(destination); + const stage = join(parent, `.tmp-${basename(destination).slice(0, 12)}-${process.pid}-${Math.random().toString(36).slice(2)}`); + try { + await mkdir(stage, { recursive: true, mode: 0o700 }); + const publishFile = async (source: string, name: string): Promise => { + const target = join(stage, name); + await copyFile(source, target); + await chmod(target, 0o600); + return { name, digest: digest(await readFile(target)) }; + }; + const [translationUnit, ir, sidecar] = await Promise.all([ + publishFile(result.cPath, "program.tu"), + result.irPath === undefined ? Promise.resolve(null) : publishFile(result.irPath, "program.ir.json"), + result.sidecarPath === undefined ? Promise.resolve(null) : publishFile(result.sidecarPath, "contract.json"), + ]); + if (!frontendInputsStillMatch(result.frontend)) return; + const unsigned: Omit = { + version: 1, + key: cacheKey(options), + frontend: result.frontend, + files: { translationUnit, ir, sidecar }, + native: result.native, + }; + const stamp: EarlyLibraryCacheStamp = { ...unsigned, integrity: stampIntegrity(unsigned) }; + await writeFile(join(stage, "stamp.json"), `${JSON.stringify(stamp)}\n`, { mode: 0o600 }); + await mkdir(destination, { recursive: true, mode: 0o700 }); + const install = async (name: string): Promise => { + const source = join(stage, name); + const target = join(destination, name); + await rename(source, target).catch(async () => { + // Windows does not replace an existing destination through rename. + // A racing reader sees either the old file or a miss; the stamp lands + // last, so no mixed artifact set can validate as a hit. + await rm(target, { force: true }); + await rename(source, target); + }); + }; + await install(translationUnit.name); + if (ir !== null) await install(ir.name); + if (sidecar !== null) await install(sidecar.name); + await install("stamp.json"); + } finally { + await rm(stage, { recursive: true, force: true }).catch(() => undefined); + } +} diff --git a/tests/harness/README.md b/tests/harness/README.md index 965913d73..a7781b158 100644 --- a/tests/harness/README.md +++ b/tests/harness/README.md @@ -73,6 +73,7 @@ Test runs are dominated by clang (~275 corpus programs × two lanes at -O2/-O1+A - **binaries** (`bin/`, cc.ts): key = resolved clang identity/version + target/compiler environment + implicit system-header dependency bytes + linker/assembler identities + runtime fingerprint (every runtime .c/.h + the vendor pin) + the full normalized command line + the emitted C bytes (byte-stable by project invariant). A hit skips native code generation and linking; the binary still RUNS live, so no comparison or sanitizer coverage is ever skipped. Each hit is checksum-verified. The sanitized lane's flags land in naturally distinct keys. FFI archive/object inputs and ambient system libraries always relink because their named files can hide mutable transitive dependencies. - **library archives** (`lib/`, cc.ts): key = resolved clang and archiver identities/versions + target/compiler environment and implicit dependencies + runtime fingerprint + target/flags + gated runtime-source set + emitted program-TU bytes. A checksum-verified hit skips native code generation and `ar`. +- **early library frontend** (`early-lib/`, library/early-cache.ts): exact library repeats validate content hashes for every file the TypeScript frontend read plus recorded failed-resolution and directory-enumeration probes, then restore the generated C/LLVM unit, optional IR, sidecar, and native feature gates without spawning TypeScript or lowering again. Source/config/package edits and newly appearing resolution candidates miss; the native archive tier still performs its own toolchain/runtime checks. - **runtime objects** (`obj/`, cc.ts): per-flavor .o for the runtime sources, including a distinct `-DSCR_LIB` flavor, so an edited executable or library recompiles only the program's own translation unit before linking or archiving. Each object carries a verified digest; a damaged entry is rebuilt before it reaches the linker or archiver. Publication rechecks the runtime and implicit-toolchain fingerprints after compilation so a concurrent source/header edit cannot place new bytes under an old key. Compiles route through ccache when installed, silently falling back when not. - **oracle results** (`oracle/`, differential.test.ts): Node's stdout/exit per program, keyed by program bytes + the spawned node's version + shim contents + invocation shape. Only the spawn is skipped; the comparison never changes. Real-time programs (setTimeout/setInterval/Promise.race — 18 of 298) are excluded and always spawn Node live: their stdout is a timer interleave that Node and the native binary only agree on under the same instantaneous load, so a cached verdict from one run must never meet a live native run from another. From ef8dc1dd4cdc0387635c58ed1d27db2370b72910 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 18 Aug 2026 08:52:51 -0500 Subject: [PATCH 2/5] Fix early frontend cache invalidation --- .../src/frontend/input-tracker.test.ts | 24 +++++++++++- .../compiler/src/frontend/input-tracker.ts | 23 +++++++++++- .../src/frontend/npm-static-rewrite.ts | 14 +++---- .../compiler/src/frontend/npm-static.test.ts | 16 ++++++++ packages/compiler/src/frontend/npm-static.ts | 37 ++++++++++--------- .../compiler/src/library/early-cache.test.ts | 3 ++ packages/compiler/src/library/early-cache.ts | 3 ++ 7 files changed, 94 insertions(+), 26 deletions(-) create mode 100644 packages/compiler/src/frontend/npm-static.test.ts diff --git a/packages/compiler/src/frontend/input-tracker.test.ts b/packages/compiler/src/frontend/input-tracker.test.ts index 9cd0081b6..f4291340b 100644 --- a/packages/compiler/src/frontend/input-tracker.test.ts +++ b/packages/compiler/src/frontend/input-tracker.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test } from "vitest"; @@ -33,6 +33,28 @@ test("tracked frontend reads invalidate on byte edits", async () => { expect(frontendInputsStillMatch(snapshot)).toBe(false); }); +test("failed frontend reads invalidate when the same file becomes readable", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const file = join(dir, "unreadable.ts"); + await writeFile(file, "export const repaired = true;\n"); + await chmod(file, 0o000); + + const tracker = new FrontendInputTracker(); + const result = tracker.run(() => trackedReadFile(file)); + if (result !== null) { + // Windows and privileged test users may not enforce POSIX mode bits. + await chmod(file, 0o600); + return; + } + const snapshot = tracker.snapshot(); + expect(snapshot.probes).toContainEqual({ op: "read-error", path: file }); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + + await chmod(file, 0o600); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + test("failed resolution candidates invalidate when a file appears", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); scratch.push(dir); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index b056b97f3..d7f3b82fd 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -12,6 +12,7 @@ import { resolve } from "node:path"; */ export type FrontendInputProbe = | { op: "file"; path: string; digest: string } + | { op: "read-error"; path: string } | { op: "kind"; path: string; kind: "file" | "directory" | "other" | "missing" } | { op: "entries"; path: string; files: string[]; directories: string[] } | { op: "realpath"; path: string; target: string | null }; @@ -70,6 +71,16 @@ export class FrontendInputTracker { // reject a perfectly stable input (`kind` vs `file` are two views of // the same current file, not two historical states). this.probes.delete(`kind\0${probe.path}`); + const failedRead = this.probes.get(`read-error\0${probe.path}`); + if (failedRead !== undefined) this.stable = false; + this.probes.delete(`read-error\0${probe.path}`); + } else if (probe.op === "read-error") { + // A path can remain a regular file while its readability changes. + // Keep the failed operation itself, rather than reducing it to a kind + // probe, so a permission/ACL repair invalidates the cached frontend. + const successfulRead = this.probes.get(`file\0${probe.path}`); + if (successfulRead !== undefined) this.stable = false; + this.probes.delete(`file\0${probe.path}`); } } @@ -99,7 +110,7 @@ export function trackedReadFile(path: string): string | null { record({ op: "file", path, digest: digest(text) }); return text; } catch { - record({ op: "kind", path, kind: pathKind(path) }); + record({ op: "read-error", path }); return null; } } @@ -184,6 +195,14 @@ export function frontendInputsStillMatch(snapshot: FrontendInputSnapshot): boole return false; } } + case "read-error": { + try { + readFileSync(probe.path, "utf8"); + return false; + } catch { + return true; + } + } case "kind": return pathKind(probe.path) === probe.kind; case "entries": { @@ -226,6 +245,8 @@ export function validFrontendInputSnapshot(snapshot: unknown): snapshot is Front switch (value.op) { case "file": return typeof value.digest === "string" && /^[0-9a-f]{64}$/.test(value.digest); + case "read-error": + return true; case "kind": return value.kind === "file" || value.kind === "directory" || value.kind === "other" || value.kind === "missing"; diff --git a/packages/compiler/src/frontend/npm-static-rewrite.ts b/packages/compiler/src/frontend/npm-static-rewrite.ts index 5bf6e358d..2321fab69 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite.ts @@ -341,12 +341,8 @@ function requireTargetEsModuleStamped(fromFile: string, spec: string, depth = 0) dir = parent; } } - let src: string; - try { - src = trackedReadFile(file)!; - } catch { - return false; - } + const src = trackedReadFile(file); + if (src === null) return false; try { if (cjsLexedExportsOf(src, file).exports.has("__esModule")) return true; } catch { @@ -365,7 +361,11 @@ function starTargetNames(file: string): Set { try { return cjsLexerVisibleNames( file, - (f) => trackedReadFile(f)!, + (f) => { + const source = trackedReadFile(f); + if (source === null) throw new Error(`cannot read ${f}`); + return source; + }, (from, spec) => resolveRelativeCjs(from, spec), ); } catch { diff --git a/packages/compiler/src/frontend/npm-static.test.ts b/packages/compiler/src/frontend/npm-static.test.ts new file mode 100644 index 000000000..a97b2f41c --- /dev/null +++ b/packages/compiler/src/frontend/npm-static.test.ts @@ -0,0 +1,16 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { npmStaticIneligibleReason } from "./npm-static.js"; + +test("an unreadable runtime entry is an eligibility refusal", async () => { + const dir = await mkdtemp("/tmp/scriptc-npm-static-unreadable-"); + try { + const root = join(dir, "node_modules", "example"); + expect(npmStaticIneligibleReason("example", `${root}/index.d.ts`, `${root}/index.js`)).toBe( + `its runtime entry ${root}/index.js cannot be read`, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 09788d3bd..9392f4da6 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -285,11 +285,14 @@ function packageIsUntyped(path: string): boolean { if (hit !== undefined) return hit; let untyped = true; try { - const pkg = JSON.parse(trackedReadFile(`${pkgDir}/package.json`)!) as Record; - if (pkg["types"] !== undefined || pkg["typings"] !== undefined) untyped = false; - if (untyped && typeof pkg["exports"] === "object" && pkg["exports"] !== null) { - // a "types" condition anywhere inside exports is a claim too - untyped = !JSON.stringify(pkg["exports"]).includes('"types"'); + const pkgText = trackedReadFile(`${pkgDir}/package.json`); + if (pkgText !== null) { + const pkg = JSON.parse(pkgText) as Record; + if (pkg["types"] !== undefined || pkg["typings"] !== undefined) untyped = false; + if (untyped && typeof pkg["exports"] === "object" && pkg["exports"] !== null) { + // a "types" condition anywhere inside exports is a claim too + untyped = !JSON.stringify(pkg["exports"]).includes('"types"'); + } } } catch { /* no package.json — keep probing */ @@ -352,7 +355,8 @@ export function npmStaticFsShadow(): NpmStaticFsShadow | null { if (target.viaTypes) return undefined; if (path.endsWith("/package.json") || path.endsWith("\\package.json")) { try { - return npmStaticTransformPkgJsonText(trackedReadFile(path)!); + const source = trackedReadFile(path); + return source === null ? undefined : npmStaticTransformPkgJsonText(source); } catch { return undefined; } @@ -371,11 +375,14 @@ export function npmStaticFsShadow(): NpmStaticFsShadow | null { if (hit !== undefined) return hit ?? undefined; let rewritten: string | null = null; try { - const answer = rewriteBundlerCjsExports(trackedReadFile(path)!, path); - if (answer !== null && typeof answer === "object") { - reportNpmStaticOffender(target.pkg, answer.degrade); - } else { - rewritten = answer; + const source = trackedReadFile(path); + if (source !== null) { + const answer = rewriteBundlerCjsExports(source, path); + if (answer !== null && typeof answer === "object") { + reportNpmStaticOffender(target.pkg, answer.degrade); + } else { + rewritten = answer; + } } } catch { rewritten = null; // unreadable/unparseable: fall through untouched @@ -442,12 +449,8 @@ export function npmStaticIneligibleReason( return "it ships no own .d.ts declaration surface"; } if (jsEntry === null) return "no runtime JS entry resolves"; - let source: string; - try { - source = trackedReadFile(jsEntry)!; - } catch { - return `its runtime entry ${jsEntry} cannot be read`; - } + const source = trackedReadFile(jsEntry); + if (source === null) return `its runtime entry ${jsEntry} cannot be read`; if (!looksUnminified(source)) return "its shipped JS looks minified"; if (hasTransformMarkers(source)) { return "its shipped JS carries build-transform markers (bundled/transpiled dist)"; diff --git a/packages/compiler/src/library/early-cache.test.ts b/packages/compiler/src/library/early-cache.test.ts index 3d73478d3..608565213 100644 --- a/packages/compiler/src/library/early-cache.test.ts +++ b/packages/compiler/src/library/early-cache.test.ts @@ -86,6 +86,8 @@ test("early library cache restores generated artifacts and metadata", async () = frontend: tracker.snapshot(), }); await Promise.all([rm(f.cPath), rm(f.irPath), rm(f.sidecarPath)]); + const staleCPath = join(f.options.outDir, "entry.lib.c"); + await writeFile(staleCPath, "/* stale c backend */\n"); const hit = await readEarlyLibraryCache(f.root, f.options, null); expect(hit).not.toBeNull(); @@ -93,6 +95,7 @@ test("early library cache restores generated artifacts and metadata", async () = expect(await readFile(f.cPath, "utf8")).toBe("; generated llvm\n"); expect(await readFile(f.irPath, "utf8")).toContain("irVersion"); expect(await readFile(f.sidecarPath, "utf8")).toContain("contract"); + await expect(readFile(staleCPath, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); }); test("early library cache misses on source edits and newly-resolved candidates", async () => { diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts index 46cf03c9e..384cf9132 100644 --- a/packages/compiler/src/library/early-cache.ts +++ b/packages/compiler/src/library/early-cache.ts @@ -150,11 +150,13 @@ function stampIntegrity(stamp: Omit): strin function outputPaths(options: EarlyLibraryCacheOptions, backend: "c" | "llvm"): { cPath: string; + staleCPath: string; irPath: string; } { const stem = basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, ""); return { cPath: join(options.outDir, `${stem}.lib.${backend === "llvm" ? "ll" : "c"}`), + staleCPath: join(options.outDir, `${stem}.lib.${backend === "llvm" ? "c" : "ll"}`), irPath: join(options.outDir, `${stem}.lib.ir.json`), }; } @@ -235,6 +237,7 @@ export async function readEarlyLibraryCache( const paths = outputPaths(options, stamp.native.backend); await installBytes(translationUnit, paths.cPath); + await rm(paths.staleCPath, { force: true }); if (ir !== null) await installBytes(ir, paths.irPath); let sidecarPath: string | undefined; if (sidecar !== null) { From 3c13f4c3b0dc68d6ab25fb4990f01c4732ab25a9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 18 Aug 2026 09:28:11 -0500 Subject: [PATCH 3/5] Fix early cache input race tracking --- .../src/frontend/input-tracker.test.ts | 57 +++++++++++++++++++ .../compiler/src/frontend/input-tracker.ts | 54 ++++++++++-------- 2 files changed, 87 insertions(+), 24 deletions(-) diff --git a/packages/compiler/src/frontend/input-tracker.test.ts b/packages/compiler/src/frontend/input-tracker.test.ts index f4291340b..0680e199a 100644 --- a/packages/compiler/src/frontend/input-tracker.test.ts +++ b/packages/compiler/src/frontend/input-tracker.test.ts @@ -69,6 +69,22 @@ test("failed resolution candidates invalidate when a file appears", async () => expect(frontendInputsStillMatch(snapshot)).toBe(false); }); +test("a candidate appearing during the frontend prevents cache publication", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const candidate = join(dir, "dependency.ts"); + + const tracker = new FrontendInputTracker(); + tracker.run(() => expect(trackedFileExists(candidate)).toBe(false)); + await writeFile(candidate, "export const loaded = true;\n"); + tracker.run(() => expect(trackedReadFile(candidate)).toContain("loaded")); + + const snapshot = tracker.snapshot(); + expect(snapshot.stable).toBe(false); + expect(validFrontendInputSnapshot(snapshot)).toBe(false); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + test("directory enumeration invalidates workspace discovery", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); scratch.push(dir); @@ -81,3 +97,44 @@ test("directory enumeration invalidates workspace discovery", async () => { await mkdir(join(packages, "new-member")); expect(frontendInputsStillMatch(snapshot)).toBe(false); }); + +test("failed directory enumeration invalidates when the operation starts succeeding", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const packages = join(dir, "packages"); + await writeFile(packages, "not a directory\n"); + + const tracker = new FrontendInputTracker(); + tracker.run(() => expect(trackedAccessibleEntries(packages)).toBeNull()); + const snapshot = tracker.snapshot(); + expect(snapshot.probes).toContainEqual({ op: "entries-error", path: packages }); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + + await rm(packages); + await mkdir(packages); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + +test("failed directory enumeration invalidates when access is restored", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const packages = join(dir, "packages"); + await mkdir(packages); + await writeFile(join(packages, "member.ts"), "export const member = true;\n"); + await chmod(packages, 0o000); + + try { + const tracker = new FrontendInputTracker(); + const result = tracker.run(() => trackedAccessibleEntries(packages)); + if (result !== null) return; // Windows and privileged users may ignore POSIX mode bits. + + const snapshot = tracker.snapshot(); + expect(snapshot.probes).toContainEqual({ op: "entries-error", path: packages }); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + + await chmod(packages, 0o700); + expect(frontendInputsStillMatch(snapshot)).toBe(false); + } finally { + await chmod(packages, 0o700); + } +}); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index d7f3b82fd..a639f2a7f 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { AsyncLocalStorage } from "node:async_hooks"; -import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; +import { readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; import { resolve } from "node:path"; /** @@ -15,6 +15,7 @@ export type FrontendInputProbe = | { op: "read-error"; path: string } | { op: "kind"; path: string; kind: "file" | "directory" | "other" | "missing" } | { op: "entries"; path: string; files: string[]; directories: string[] } + | { op: "entries-error"; path: string } | { op: "realpath"; path: string; target: string | null }; export interface FrontendInputSnapshot { @@ -67,9 +68,11 @@ export class FrontendInputTracker { this.probes.set(key, probe); if (probe.op === "file") { // A successful content read supersedes earlier existence probes for the - // same path. Retaining both would make the post-build stability check - // reject a perfectly stable input (`kind` vs `file` are two views of - // the same current file, not two historical states). + // same path only when they also observed a file. A missing/non-file + // answer followed by a successful read means the input changed during + // the frontend and the mixed observation must never be published. + const previousKind = this.probes.get(`kind\0${probe.path}`); + if (previousKind?.op === "kind" && previousKind.kind !== "file") this.stable = false; this.probes.delete(`kind\0${probe.path}`); const failedRead = this.probes.get(`read-error\0${probe.path}`); if (failedRead !== undefined) this.stable = false; @@ -117,33 +120,23 @@ export function trackedReadFile(path: string): string | null { export function trackedFileExists(path: string): boolean { path = resolve(path); - let exists = false; - try { - exists = statSync(path).isFile(); - } catch { - // The exact failed candidate is part of the resolution answer. - } - record({ op: "kind", path, kind: pathKind(path) }); - return exists; + const kind = pathKind(path); + record({ op: "kind", path, kind }); + return kind === "file"; } export function trackedDirectoryExists(path: string): boolean { path = resolve(path); - let exists = false; - try { - exists = statSync(path).isDirectory(); - } catch { - // The exact failed candidate is part of the resolution answer. - } - record({ op: "kind", path, kind: pathKind(path) }); - return exists; + const kind = pathKind(path); + record({ op: "kind", path, kind }); + return kind === "directory"; } export function trackedExists(path: string): boolean { path = resolve(path); - const exists = existsSync(path); - record({ op: "kind", path, kind: pathKind(path) }); - return exists; + const kind = pathKind(path); + record({ op: "kind", path, kind }); + return kind !== "missing"; } export function trackedRealpath(path: string): string | null { @@ -176,7 +169,10 @@ export function trackedAccessibleEntries( record({ op: "entries", path, ...answer }); return answer; } catch { - record({ op: "kind", path, kind: pathKind(path) }); + // Enumeration can fail even while the path remains a directory. Preserve + // the failed operation itself so a permission/ACL repair invalidates the + // cached frontend rather than replaying only the unchanged path kind. + record({ op: "entries-error", path }); return null; } } @@ -222,6 +218,14 @@ export function frontendInputsStillMatch(snapshot: FrontendInputSnapshot): boole return false; } } + case "entries-error": { + try { + readdirSync(probe.path, { withFileTypes: true }); + return false; + } catch { + return true; + } + } case "realpath": { try { return realpathSync(probe.path) === probe.target; @@ -253,6 +257,8 @@ export function validFrontendInputSnapshot(snapshot: unknown): snapshot is Front case "entries": return Array.isArray(value.files) && value.files.every((entry) => typeof entry === "string") && Array.isArray(value.directories) && value.directories.every((entry) => typeof entry === "string"); + case "entries-error": + return true; case "realpath": return value.target === null || typeof value.target === "string"; default: From d99508f74f288c4b3ea3fdd1bb6fbff0952f7b76 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 18 Aug 2026 10:07:51 -0500 Subject: [PATCH 4/5] Fix early frontend cache output tracking --- .../src/frontend/input-tracker.test.ts | 29 ++++++ .../compiler/src/frontend/input-tracker.ts | 90 ++++++++++++++++++- .../compiler/src/library/early-cache.test.ts | 83 ++++++++++++++++- packages/compiler/src/library/early-cache.ts | 59 ++++++++++-- 4 files changed, 249 insertions(+), 12 deletions(-) diff --git a/packages/compiler/src/frontend/input-tracker.test.ts b/packages/compiler/src/frontend/input-tracker.test.ts index 0680e199a..f9057c48f 100644 --- a/packages/compiler/src/frontend/input-tracker.test.ts +++ b/packages/compiler/src/frontend/input-tracker.test.ts @@ -6,6 +6,7 @@ import { FrontendInputTracker, frontendInputsStillMatch, trackedAccessibleEntries, + trackedDirectoryExists, trackedFileExists, trackedReadFile, validFrontendInputSnapshot, @@ -98,6 +99,34 @@ test("directory enumeration invalidates workspace discovery", async () => { expect(frontendInputsStillMatch(snapshot)).toBe(false); }); +test("compiler outputs do not invalidate a fresh output directory", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const generatedRoot = join(dir, "generated"); + const outDir = join(generatedRoot, "nested"); + const generated = join(outDir, "entry.lib.c"); + + const tracker = new FrontendInputTracker(); + tracker.run(() => { + expect(trackedAccessibleEntries(dir)?.directories).toEqual([]); + expect(trackedDirectoryExists(generatedRoot)).toBe(false); + expect(trackedDirectoryExists(outDir)).toBe(false); + expect(trackedAccessibleEntries(outDir)).toBeNull(); + }); + const snapshot = tracker.snapshot(); + const exclusions = { + outputPaths: [generated], + outputDirectories: [dir, generatedRoot, outDir], + }; + + await mkdir(outDir, { recursive: true }); + await writeFile(generated, "/* generated */\n"); + expect(frontendInputsStillMatch(snapshot, exclusions)).toBe(true); + + await writeFile(join(outDir, "new-source.ts"), "export const appeared = true;\n"); + expect(frontendInputsStillMatch(snapshot, exclusions)).toBe(false); +}); + test("failed directory enumeration invalidates when the operation starts succeeding", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); scratch.push(dir); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index a639f2a7f..6398302d8 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { AsyncLocalStorage } from "node:async_hooks"; import { readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; -import { resolve } from "node:path"; +import { basename, dirname, resolve } from "node:path"; /** * Filesystem observations made while constructing and lowering one frontend @@ -24,6 +24,15 @@ export interface FrontendInputSnapshot { stable: boolean; } +/** Compiler-owned paths whose creation/removal must not invalidate the + * frontend that produced them. Directory CONTENT remains tracked: only the + * named artifacts and a generated directory's formerly-missing observation + * are excluded. */ +export interface FrontendInputExclusions { + outputPaths?: Iterable; + outputDirectories?: Iterable; +} + function digest(text: string): string { return createHash("sha256").update(text).digest("hex"); } @@ -178,11 +187,65 @@ export function trackedAccessibleEntries( } /** Re-run every recorded probe against the current filesystem. */ -export function frontendInputsStillMatch(snapshot: FrontendInputSnapshot): boolean { +export function frontendInputsStillMatch( + snapshot: FrontendInputSnapshot, + exclusions: FrontendInputExclusions = {}, +): boolean { if (snapshot.version !== 1 || snapshot.stable !== true || !Array.isArray(snapshot.probes)) { return false; } + const outputPaths = new Set([...(exclusions.outputPaths ?? [])].map((path) => resolve(path))); + const outputDirectories = new Set( + [...(exclusions.outputDirectories ?? [])].map((path) => resolve(path)), + ); + const generatedOnlyDirectory = (directory: string): boolean => { + const allowedFiles = new Set(); + for (const output of outputPaths) { + if (dirname(output) === directory) allowedFiles.add(basename(output)); + } + const allowedDirectories = new Map(); + for (const outputDir of outputDirectories) { + if (outputDir !== directory && dirname(outputDir) === directory) { + allowedDirectories.set(basename(outputDir), outputDir); + } + } + try { + return readdirSync(directory).every((name) => { + if (allowedFiles.has(name)) return true; + const child = allowedDirectories.get(name); + return child !== undefined && generatedOnlyDirectory(child); + }); + } catch { + return false; + } + }; return snapshot.probes.every((probe) => { + if (outputPaths.has(probe.path)) return true; + if (outputDirectories.has(probe.path)) { + // A fresh output directory can be absent while the frontend runs and + // created only when emission starts. Admit it only while its current + // contents are exactly the named compiler artifacts; a user file that + // appears there still invalidates module resolution. + if (probe.op === "kind" && probe.kind === "missing") { + return pathKind(probe.path) === "missing" || generatedOnlyDirectory(probe.path); + } + if (probe.op === "entries-error") { + try { + readdirSync(probe.path); + return generatedOnlyDirectory(probe.path); + } catch { + return true; + } + } + if (probe.op === "realpath" && probe.target === null) { + try { + realpathSync(probe.path); + return generatedOnlyDirectory(probe.path); + } catch { + return true; + } + } + } switch (probe.op) { case "file": { try { @@ -203,6 +266,23 @@ export function frontendInputsStillMatch(snapshot: FrontendInputSnapshot): boole return pathKind(probe.path) === probe.kind; case "entries": { try { + const ignored = new Set(); + for (const output of outputPaths) { + if (dirname(output) === probe.path) ignored.add(basename(output)); + } + for (const outputDir of outputDirectories) { + if (dirname(outputDir) !== probe.path) continue; + const name = basename(outputDir); + // If the directory existed during the frontend, keep tracking its + // presence. Only suppress a directory introduced by this build. + if ( + !probe.files.includes(name) && + !probe.directories.includes(name) && + generatedOnlyDirectory(outputDir) + ) { + ignored.add(name); + } + } const files: string[] = []; const directories: string[] = []; for (const entry of readdirSync(probe.path, { withFileTypes: true })) { @@ -212,8 +292,10 @@ export function frontendInputsStillMatch(snapshot: FrontendInputSnapshot): boole } files.sort(); directories.sort(); - return JSON.stringify(files) === JSON.stringify(probe.files) && - JSON.stringify(directories) === JSON.stringify(probe.directories); + return JSON.stringify(files.filter((name) => !ignored.has(name))) === + JSON.stringify(probe.files.filter((name) => !ignored.has(name))) && + JSON.stringify(directories.filter((name) => !ignored.has(name))) === + JSON.stringify(probe.directories.filter((name) => !ignored.has(name))); } catch { return false; } diff --git a/packages/compiler/src/library/early-cache.test.ts b/packages/compiler/src/library/early-cache.test.ts index 608565213..82656d2ba 100644 --- a/packages/compiler/src/library/early-cache.test.ts +++ b/packages/compiler/src/library/early-cache.test.ts @@ -1,8 +1,8 @@ -import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, readdir, rm, stat, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test } from "vitest"; -import { FrontendInputTracker, trackedFileExists, trackedReadFile } from "../frontend/input-tracker.js"; +import { FrontendInputTracker, trackedAccessibleEntries, trackedDirectoryExists, trackedFileExists, trackedReadFile } from "../frontend/input-tracker.js"; import { publishEarlyLibraryCache, readEarlyLibraryCache, @@ -98,6 +98,85 @@ test("early library cache restores generated artifacts and metadata", async () = await expect(readFile(staleCPath, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); }); +test("early library cache publishes after creating a fresh output directory", async () => { + const f = await fixture(); + const cacheRoot = await mkdtemp(join(tmpdir(), "scriptc-early-cache-root-")); + scratch.push(cacheRoot); + const outDir = join(f.options.outDir, "fresh-out"); + const cPath = join(outDir, "entry.lib.ll"); + const options = { ...f.options, outDir, emitIr: false }; + const tracker = new FrontendInputTracker(); + tracker.run(() => { + trackedAccessibleEntries(f.options.outDir); + trackedDirectoryExists(outDir); + trackedAccessibleEntries(outDir); + trackedReadFile(f.source); + }); + + await mkdir(outDir); + await writeFile(cPath, "; generated llvm in fresh output\n"); + await publishEarlyLibraryCache(cacheRoot, options, { + cPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }); + + await rm(cPath); + expect((await readEarlyLibraryCache(cacheRoot, options, undefined))?.cPath).toBe(cPath); + expect(await readFile(cPath, "utf8")).toContain("fresh output"); +}); + +test("early library cache hits refresh every payload's LRU time", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }); + const earlyRoot = join(f.root, "early-lib"); + const [key] = await readdir(earlyRoot); + const entry = join(earlyRoot, key!); + const cachePaths = [ + join(entry, "stamp.json"), + join(entry, "program.tu"), + join(entry, "program.ir.json"), + join(entry, "contract.json"), + ]; + const old = new Date("2000-01-01T00:00:00.000Z"); + await Promise.all(cachePaths.map((path) => utimes(path, old, old))); + + expect(await readEarlyLibraryCache(f.root, f.options, null)).not.toBeNull(); + for (const path of cachePaths) { + expect((await stat(path)).mtimeMs).toBeGreaterThan(old.getTime()); + } +}); + test("early library cache misses on source edits and newly-resolved candidates", async () => { const f = await fixture(); const tracker = new FrontendInputTracker(); diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts index 384cf9132..16457d87f 100644 --- a/packages/compiler/src/library/early-cache.ts +++ b/packages/compiler/src/library/early-cache.ts @@ -3,7 +3,7 @@ import { chmod, copyFile, mkdir, readFile, readdir, rename, rm, utimes, writeFil import { basename, dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { compilerReleaseVersion } from "./sidecar.js"; -import { frontendInputsStillMatch, validFrontendInputSnapshot, type FrontendInputSnapshot } from "../frontend/input-tracker.js"; +import { frontendInputsStillMatch, validFrontendInputSnapshot, type FrontendInputExclusions, type FrontendInputSnapshot } from "../frontend/input-tracker.js"; interface CachedLibraryFile { name: string; @@ -162,11 +162,41 @@ function outputPaths(options: EarlyLibraryCacheOptions, backend: "c" | "llvm"): } function sidecarOutputPath(options: EarlyLibraryCacheOptions, configured: string | null): string { - const archivePath = options.outPath ?? join( + const archivePath = archiveOutputPath(options); + return configured !== null ? resolve(dirname(archivePath), configured) : `${archivePath}.contract.json`; +} + +function archiveOutputPath(options: EarlyLibraryCacheOptions): string { + return options.outPath ?? join( options.outDir, `${basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, "")}.lib.a`, ); - return configured !== null ? resolve(dirname(archivePath), configured) : `${archivePath}.contract.json`; +} + +function frontendOutputExclusions( + options: EarlyLibraryCacheOptions, + backend: "c" | "llvm", + sidecarPath: string | undefined, +): FrontendInputExclusions { + const paths = outputPaths(options, backend); + const outputArtifacts = [ + paths.cPath, + paths.staleCPath, + ...(options.emitIr ? [paths.irPath] : []), + archiveOutputPath(options), + ...(sidecarPath === undefined ? [] : [sidecarPath]), + ].map((path) => resolve(path)); + const outputDirectories = new Set(); + for (const artifact of outputArtifacts) { + for (let directory = dirname(artifact); ; directory = dirname(directory)) { + outputDirectories.add(directory); + if (dirname(directory) === directory) break; + } + } + return { + outputPaths: outputArtifacts, + outputDirectories, + }; } async function readCachedFile(path: string, expected: string): Promise { @@ -214,7 +244,16 @@ export async function readEarlyLibraryCache( stamp.files.sidecar?.name !== "contract.json" || !/^[0-9a-f]{64}$/.test(stamp.files.sidecar.digest) )) || stampIntegrity(unsigned) !== integrity || - !frontendInputsStillMatch(stamp.frontend) || + !frontendInputsStillMatch( + stamp.frontend, + frontendOutputExclusions( + options, + stamp.native.backend, + stamp.files.sidecar === null + ? undefined + : sidecarOutputPath(options, sidecarConfiguredPath ?? null), + ), + ) || (stamp.files.ir !== null) !== options.emitIr || (stamp.files.sidecar !== null) !== (sidecarConfiguredPath !== undefined) ) return null; @@ -245,7 +284,12 @@ export async function readEarlyLibraryCache( await installBytes(sidecar, sidecarPath); } const now = new Date(); - await utimes(path, now, now).catch(() => undefined); + await Promise.all([ + path, + join(directory, stamp.files.translationUnit.name), + ...(stamp.files.ir === null ? [] : [join(directory, stamp.files.ir.name)]), + ...(stamp.files.sidecar === null ? [] : [join(directory, stamp.files.sidecar.name)]), + ].map((cachePath) => utimes(cachePath, now, now).catch(() => undefined))); return { cPath: paths.cPath, native: stamp.native, @@ -279,7 +323,10 @@ export async function publishEarlyLibraryCache( result.irPath === undefined ? Promise.resolve(null) : publishFile(result.irPath, "program.ir.json"), result.sidecarPath === undefined ? Promise.resolve(null) : publishFile(result.sidecarPath, "contract.json"), ]); - if (!frontendInputsStillMatch(result.frontend)) return; + if (!frontendInputsStillMatch( + result.frontend, + frontendOutputExclusions(options, result.native.backend, result.sidecarPath), + )) return; const unsigned: Omit = { version: 1, key: cacheKey(options), From f4720df9d4f3f9759b4ca09474549396ccf701db Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 18 Aug 2026 10:44:36 -0500 Subject: [PATCH 5/5] Key frontend cache by Node version --- packages/compiler/src/index.ts | 1 + .../compiler/src/library/early-cache.test.ts | 32 +++++++++++++++++++ packages/compiler/src/library/early-cache.ts | 4 +++ 3 files changed, 37 insertions(+) diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 702c84eb9..0ae28dbea 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1700,6 +1700,7 @@ async function compileLibraryTracked( sanitize: opts.sanitize ?? false, target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${process.arch}`, compiler: [process.env["SCRIPTC_CC"] ?? "clang"], + nodeVersion: process.version, implementation: await libraryFrontendImplementationFingerprint(), }; const earlyHit = await readEarlyLibraryCache( diff --git a/packages/compiler/src/library/early-cache.test.ts b/packages/compiler/src/library/early-cache.test.ts index 82656d2ba..366b7b609 100644 --- a/packages/compiler/src/library/early-cache.test.ts +++ b/packages/compiler/src/library/early-cache.test.ts @@ -54,6 +54,7 @@ async function fixture(): Promise<{ sanitize: false, target: "test", compiler: ["clang"], + nodeVersion: "v24-test", implementation: "test-implementation", }, }; @@ -237,6 +238,37 @@ test("early library cache misses on source edits and newly-resolved candidates", expect(await readEarlyLibraryCache(f.root, f.options, null)).toBeNull(); }); +test("early library cache is separated by the host Node version", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }); + + expect(await readEarlyLibraryCache(f.root, f.options, null)).not.toBeNull(); + expect(await readEarlyLibraryCache( + f.root, + { ...f.options, nodeVersion: "v25-test" }, + null, + )).toBeNull(); +}); + test("early library cache rejects corrupted artifacts and metadata", async () => { const f = await fixture(); const tracker = new FrontendInputTracker(); diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts index 16457d87f..eb266af0f 100644 --- a/packages/compiler/src/library/early-cache.ts +++ b/packages/compiler/src/library/early-cache.ts @@ -57,6 +57,9 @@ export interface EarlyLibraryCacheOptions { sanitize: boolean; target: string; compiler: string[]; + /** Host Node runtime whose builtin-module inventory participates in + * frontend classification. */ + nodeVersion: string; implementation: string; } @@ -105,6 +108,7 @@ function cacheKey(options: EarlyLibraryCacheOptions): string { .update(options.sanitize ? "sanitize" : "plain").update("\0") .update(options.target).update("\0") .update(options.compiler.join("\x1f")).update("\0") + .update(options.nodeVersion).update("\0") .update(options.implementation) .digest("hex"); }