diff --git a/packages/cli/README.md b/packages/cli/README.md index 4044bfa7..92b68d80 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -24,7 +24,7 @@ $ npm install -g scriptc Requires clang on the PATH (Xcode Command Line Tools on macOS, `clang` package on Linux). -Builds use a bounded persistent cache by default. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. The compiler remains required so dependency selection is rediscovered on every invocation. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass the cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions. +Builds use a bounded persistent cache by default. Exact unchanged library builds validate their recorded TypeScript/module-resolution inputs and restore the generated C/LLVM unit before starting the frontend; the native cache then applies its independent toolchain checks. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. Experimental provenance-source builds bypass the early frontend tier because their fetched-source registry is process state. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent native artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass every cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions. ## Commands diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 7291b8e6..8af648ae 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -2075,6 +2075,31 @@ export function resolveBuildCacheRoot( : resolve(userHome, ".cache", "scriptc", "build"); } +/** Shared persistent-cache root for compiler-level tiers. The early library + * cache deliberately follows the native cache's activation and hard-disable + * contract, while native compilation retains ownership of toolchain safety. */ +export function buildCacheRoot(): string | null { + return resolveBuildCacheRoot(); +} + +/** Harden/create a compiler-level cache root using the same privacy policy as + * the artifact caches. Failure disables only the optional caller's tier. */ +export async function prepareBuildCacheRoot(root: string | null): Promise { + if (root === null) return null; + try { + await ensurePrivateCacheRoot(root, process.env["SCRIPTC_CACHE_DIR"] === undefined); + return root; + } catch { + return null; + } +} + +/** Register a successful compiler-level cache write with the shared bounded + * LRU policy. */ +export async function pruneBuildCache(root: string | null): Promise { + if (root !== null) await pruneCache(root).catch(() => undefined); +} + function cacheRootDir(): string | null { return resolveBuildCacheRoot(); } diff --git a/packages/compiler/src/frontend/input-tracker.test.ts b/packages/compiler/src/frontend/input-tracker.test.ts new file mode 100644 index 00000000..f9057c48 --- /dev/null +++ b/packages/compiler/src/frontend/input-tracker.test.ts @@ -0,0 +1,169 @@ +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { + FrontendInputTracker, + frontendInputsStillMatch, + trackedAccessibleEntries, + trackedDirectoryExists, + trackedFileExists, + trackedReadFile, + validFrontendInputSnapshot, +} from "./input-tracker.js"; + +const scratch: string[] = []; + +afterEach(async () => { + await Promise.all(scratch.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +test("tracked frontend reads invalidate on byte edits", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const file = join(dir, "entry.ts"); + await writeFile(file, "export const answer = 1;\n"); + + const tracker = new FrontendInputTracker(); + tracker.run(() => expect(trackedReadFile(file)).toContain("answer")); + const snapshot = tracker.snapshot(); + expect(validFrontendInputSnapshot(snapshot)).toBe(true); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + + await writeFile(file, "export const answer = 2;\n"); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + +test("failed frontend reads invalidate when the same file becomes readable", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const file = join(dir, "unreadable.ts"); + await writeFile(file, "export const repaired = true;\n"); + await chmod(file, 0o000); + + const tracker = new FrontendInputTracker(); + const result = tracker.run(() => trackedReadFile(file)); + if (result !== null) { + // Windows and privileged test users may not enforce POSIX mode bits. + await chmod(file, 0o600); + return; + } + const snapshot = tracker.snapshot(); + expect(snapshot.probes).toContainEqual({ op: "read-error", path: file }); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + + await chmod(file, 0o600); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + +test("failed resolution candidates invalidate when a file appears", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const candidate = join(dir, "dependency.ts"); + + const tracker = new FrontendInputTracker(); + tracker.run(() => expect(trackedFileExists(candidate)).toBe(false)); + const snapshot = tracker.snapshot(); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + + await writeFile(candidate, "export const loaded = true;\n"); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + +test("a candidate appearing during the frontend prevents cache publication", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const candidate = join(dir, "dependency.ts"); + + const tracker = new FrontendInputTracker(); + tracker.run(() => expect(trackedFileExists(candidate)).toBe(false)); + await writeFile(candidate, "export const loaded = true;\n"); + tracker.run(() => expect(trackedReadFile(candidate)).toContain("loaded")); + + const snapshot = tracker.snapshot(); + expect(snapshot.stable).toBe(false); + expect(validFrontendInputSnapshot(snapshot)).toBe(false); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + +test("directory enumeration invalidates workspace discovery", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const packages = join(dir, "packages"); + await mkdir(packages); + + const tracker = new FrontendInputTracker(); + tracker.run(() => expect(trackedAccessibleEntries(packages)?.directories).toEqual([])); + const snapshot = tracker.snapshot(); + await mkdir(join(packages, "new-member")); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + +test("compiler outputs do not invalidate a fresh output directory", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const generatedRoot = join(dir, "generated"); + const outDir = join(generatedRoot, "nested"); + const generated = join(outDir, "entry.lib.c"); + + const tracker = new FrontendInputTracker(); + tracker.run(() => { + expect(trackedAccessibleEntries(dir)?.directories).toEqual([]); + expect(trackedDirectoryExists(generatedRoot)).toBe(false); + expect(trackedDirectoryExists(outDir)).toBe(false); + expect(trackedAccessibleEntries(outDir)).toBeNull(); + }); + const snapshot = tracker.snapshot(); + const exclusions = { + outputPaths: [generated], + outputDirectories: [dir, generatedRoot, outDir], + }; + + await mkdir(outDir, { recursive: true }); + await writeFile(generated, "/* generated */\n"); + expect(frontendInputsStillMatch(snapshot, exclusions)).toBe(true); + + await writeFile(join(outDir, "new-source.ts"), "export const appeared = true;\n"); + expect(frontendInputsStillMatch(snapshot, exclusions)).toBe(false); +}); + +test("failed directory enumeration invalidates when the operation starts succeeding", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const packages = join(dir, "packages"); + await writeFile(packages, "not a directory\n"); + + const tracker = new FrontendInputTracker(); + tracker.run(() => expect(trackedAccessibleEntries(packages)).toBeNull()); + const snapshot = tracker.snapshot(); + expect(snapshot.probes).toContainEqual({ op: "entries-error", path: packages }); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + + await rm(packages); + await mkdir(packages); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + +test("failed directory enumeration invalidates when access is restored", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const packages = join(dir, "packages"); + await mkdir(packages); + await writeFile(join(packages, "member.ts"), "export const member = true;\n"); + await chmod(packages, 0o000); + + try { + const tracker = new FrontendInputTracker(); + const result = tracker.run(() => trackedAccessibleEntries(packages)); + if (result !== null) return; // Windows and privileged users may ignore POSIX mode bits. + + const snapshot = tracker.snapshot(); + expect(snapshot.probes).toContainEqual({ op: "entries-error", path: packages }); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + + await chmod(packages, 0o700); + expect(frontendInputsStillMatch(snapshot)).toBe(false); + } finally { + await chmod(packages, 0o700); + } +}); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts new file mode 100644 index 00000000..6398302d --- /dev/null +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -0,0 +1,350 @@ +import { createHash } from "node:crypto"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; +import { basename, dirname, resolve } from "node:path"; + +/** + * Filesystem observations made while constructing and lowering one frontend + * program. The early library cache replays these probes before trusting a + * generated translation unit: successful reads are content-addressed, while + * failed candidate probes are retained so a newly-created module cannot hide + * behind an old resolution answer. + */ +export type FrontendInputProbe = + | { op: "file"; path: string; digest: string } + | { op: "read-error"; path: string } + | { op: "kind"; path: string; kind: "file" | "directory" | "other" | "missing" } + | { op: "entries"; path: string; files: string[]; directories: string[] } + | { op: "entries-error"; path: string } + | { op: "realpath"; path: string; target: string | null }; + +export interface FrontendInputSnapshot { + version: 1; + probes: FrontendInputProbe[]; + stable: boolean; +} + +/** Compiler-owned paths whose creation/removal must not invalidate the + * frontend that produced them. Directory CONTENT remains tracked: only the + * named artifacts and a generated directory's formerly-missing observation + * are excluded. */ +export interface FrontendInputExclusions { + outputPaths?: Iterable; + outputDirectories?: Iterable; +} + +function digest(text: string): string { + return createHash("sha256").update(text).digest("hex"); +} + +function pathKind(path: string): Extract["kind"] { + try { + const info = statSync(path); + return info.isFile() + ? "file" + : info.isDirectory() + ? "directory" + : "other"; + } catch { + return "missing"; + } +} + +const activeTracker = new AsyncLocalStorage(); + +export class FrontendInputTracker { + private readonly probes = new Map(); + private stable = true; + + run(fn: () => T): T { + const parent = activeTracker.getStore(); + if (parent === undefined || parent === this) return activeTracker.run(this, fn); + return activeTracker.run(this, () => { + const result = fn(); + for (const probe of this.probes.values()) parent.record(probe); + return result; + }); + } + + record(probe: FrontendInputProbe): void { + const key = `${probe.op}\0${probe.path}`; + const previous = this.probes.get(key); + if (previous !== undefined && JSON.stringify(previous) !== JSON.stringify(probe)) { + // An input changed while this frontend was running. The build may + // still finish normally, but it is not safe to publish as a cache key. + this.stable = false; + } + this.probes.set(key, probe); + if (probe.op === "file") { + // A successful content read supersedes earlier existence probes for the + // same path only when they also observed a file. A missing/non-file + // answer followed by a successful read means the input changed during + // the frontend and the mixed observation must never be published. + const previousKind = this.probes.get(`kind\0${probe.path}`); + if (previousKind?.op === "kind" && previousKind.kind !== "file") this.stable = false; + this.probes.delete(`kind\0${probe.path}`); + const failedRead = this.probes.get(`read-error\0${probe.path}`); + if (failedRead !== undefined) this.stable = false; + this.probes.delete(`read-error\0${probe.path}`); + } else if (probe.op === "read-error") { + // A path can remain a regular file while its readability changes. + // Keep the failed operation itself, rather than reducing it to a kind + // probe, so a permission/ACL repair invalidates the cached frontend. + const successfulRead = this.probes.get(`file\0${probe.path}`); + if (successfulRead !== undefined) this.stable = false; + this.probes.delete(`file\0${probe.path}`); + } + } + + snapshot(): FrontendInputSnapshot { + return { + version: 1, + probes: [...this.probes.values()].sort((a, b) => + a.path === b.path ? a.op.localeCompare(b.op) : a.path.localeCompare(b.path) + ), + stable: this.stable, + }; + } +} + +export function frontendInputTrackingActive(): boolean { + return activeTracker.getStore() !== undefined; +} + +function record(probe: FrontendInputProbe): void { + activeTracker.getStore()?.record(probe); +} + +export function trackedReadFile(path: string): string | null { + path = resolve(path); + try { + const text = readFileSync(path, "utf8"); + record({ op: "file", path, digest: digest(text) }); + return text; + } catch { + record({ op: "read-error", path }); + return null; + } +} + +export function trackedFileExists(path: string): boolean { + path = resolve(path); + const kind = pathKind(path); + record({ op: "kind", path, kind }); + return kind === "file"; +} + +export function trackedDirectoryExists(path: string): boolean { + path = resolve(path); + const kind = pathKind(path); + record({ op: "kind", path, kind }); + return kind === "directory"; +} + +export function trackedExists(path: string): boolean { + path = resolve(path); + const kind = pathKind(path); + record({ op: "kind", path, kind }); + return kind !== "missing"; +} + +export function trackedRealpath(path: string): string | null { + path = resolve(path); + try { + const target = realpathSync(path); + record({ op: "realpath", path, target }); + return target; + } catch { + record({ op: "realpath", path, target: null }); + return null; + } +} + +export function trackedAccessibleEntries( + path: string, +): { files: string[]; directories: string[] } | null { + path = resolve(path); + try { + const files: string[] = []; + const directories: string[] = []; + for (const entry of readdirSync(path, { withFileTypes: true })) { + const kind = entry.isSymbolicLink() ? pathKind(resolve(path, entry.name)) : null; + if (entry.isFile() || kind === "file") files.push(entry.name); + if (entry.isDirectory() || kind === "directory") directories.push(entry.name); + } + files.sort(); + directories.sort(); + const answer = { files, directories }; + record({ op: "entries", path, ...answer }); + return answer; + } catch { + // Enumeration can fail even while the path remains a directory. Preserve + // the failed operation itself so a permission/ACL repair invalidates the + // cached frontend rather than replaying only the unchanged path kind. + record({ op: "entries-error", path }); + return null; + } +} + +/** Re-run every recorded probe against the current filesystem. */ +export function frontendInputsStillMatch( + snapshot: FrontendInputSnapshot, + exclusions: FrontendInputExclusions = {}, +): boolean { + if (snapshot.version !== 1 || snapshot.stable !== true || !Array.isArray(snapshot.probes)) { + return false; + } + const outputPaths = new Set([...(exclusions.outputPaths ?? [])].map((path) => resolve(path))); + const outputDirectories = new Set( + [...(exclusions.outputDirectories ?? [])].map((path) => resolve(path)), + ); + const generatedOnlyDirectory = (directory: string): boolean => { + const allowedFiles = new Set(); + for (const output of outputPaths) { + if (dirname(output) === directory) allowedFiles.add(basename(output)); + } + const allowedDirectories = new Map(); + for (const outputDir of outputDirectories) { + if (outputDir !== directory && dirname(outputDir) === directory) { + allowedDirectories.set(basename(outputDir), outputDir); + } + } + try { + return readdirSync(directory).every((name) => { + if (allowedFiles.has(name)) return true; + const child = allowedDirectories.get(name); + return child !== undefined && generatedOnlyDirectory(child); + }); + } catch { + return false; + } + }; + return snapshot.probes.every((probe) => { + if (outputPaths.has(probe.path)) return true; + if (outputDirectories.has(probe.path)) { + // A fresh output directory can be absent while the frontend runs and + // created only when emission starts. Admit it only while its current + // contents are exactly the named compiler artifacts; a user file that + // appears there still invalidates module resolution. + if (probe.op === "kind" && probe.kind === "missing") { + return pathKind(probe.path) === "missing" || generatedOnlyDirectory(probe.path); + } + if (probe.op === "entries-error") { + try { + readdirSync(probe.path); + return generatedOnlyDirectory(probe.path); + } catch { + return true; + } + } + if (probe.op === "realpath" && probe.target === null) { + try { + realpathSync(probe.path); + return generatedOnlyDirectory(probe.path); + } catch { + return true; + } + } + } + switch (probe.op) { + case "file": { + try { + return digest(readFileSync(probe.path, "utf8")) === probe.digest; + } catch { + return false; + } + } + case "read-error": { + try { + readFileSync(probe.path, "utf8"); + return false; + } catch { + return true; + } + } + case "kind": + return pathKind(probe.path) === probe.kind; + case "entries": { + try { + const ignored = new Set(); + for (const output of outputPaths) { + if (dirname(output) === probe.path) ignored.add(basename(output)); + } + for (const outputDir of outputDirectories) { + if (dirname(outputDir) !== probe.path) continue; + const name = basename(outputDir); + // If the directory existed during the frontend, keep tracking its + // presence. Only suppress a directory introduced by this build. + if ( + !probe.files.includes(name) && + !probe.directories.includes(name) && + generatedOnlyDirectory(outputDir) + ) { + ignored.add(name); + } + } + const files: string[] = []; + const directories: string[] = []; + for (const entry of readdirSync(probe.path, { withFileTypes: true })) { + const kind = entry.isSymbolicLink() ? pathKind(resolve(probe.path, entry.name)) : null; + if (entry.isFile() || kind === "file") files.push(entry.name); + if (entry.isDirectory() || kind === "directory") directories.push(entry.name); + } + files.sort(); + directories.sort(); + return JSON.stringify(files.filter((name) => !ignored.has(name))) === + JSON.stringify(probe.files.filter((name) => !ignored.has(name))) && + JSON.stringify(directories.filter((name) => !ignored.has(name))) === + JSON.stringify(probe.directories.filter((name) => !ignored.has(name))); + } catch { + return false; + } + } + case "entries-error": { + try { + readdirSync(probe.path, { withFileTypes: true }); + return false; + } catch { + return true; + } + } + case "realpath": { + try { + return realpathSync(probe.path) === probe.target; + } catch { + return probe.target === null; + } + } + } + }); +} + +/** Pure validator used by the persistent-cache reader before any path probes. */ +export function validFrontendInputSnapshot(snapshot: unknown): snapshot is FrontendInputSnapshot { + if (snapshot === null || typeof snapshot !== "object") return false; + const candidate = snapshot as Partial; + if (candidate.version !== 1 || candidate.stable !== true || !Array.isArray(candidate.probes)) return false; + return candidate.probes.every((probe) => { + if (probe === null || typeof probe !== "object") return false; + const value = probe as Partial; + if (typeof value.path !== "string" || typeof value.op !== "string") return false; + switch (value.op) { + case "file": + return typeof value.digest === "string" && /^[0-9a-f]{64}$/.test(value.digest); + case "read-error": + return true; + case "kind": + return value.kind === "file" || value.kind === "directory" || value.kind === "other" || + value.kind === "missing"; + case "entries": + return Array.isArray(value.files) && value.files.every((entry) => typeof entry === "string") && + Array.isArray(value.directories) && value.directories.every((entry) => typeof entry === "string"); + case "entries-error": + return true; + case "realpath": + return value.target === null || typeof value.target === "string"; + default: + return false; + } + }); +} diff --git a/packages/compiler/src/frontend/lowering/lower-builtins.ts b/packages/compiler/src/frontend/lowering/lower-builtins.ts index 908366de..7d1f36f2 100644 --- a/packages/compiler/src/frontend/lowering/lower-builtins.ts +++ b/packages/compiler/src/frontend/lowering/lower-builtins.ts @@ -2,7 +2,6 @@ * crypto, child_process spawn/spawnSync and child/stats/spawn-result * methods), JSON.parse/stringify, process properties/methods and * process.env access, and console.log detection. */ -import { readFileSync } from "node:fs"; import { builtinModules } from "node:module"; import { dirname, resolve } from "node:path"; import * as ts from "../ts7/adapter.js"; @@ -12,6 +11,7 @@ import { canonicalBuiltinModule, isJsSourceFile, locOf, requireSpecOf } from ".. import { isRelativeSpecifier } from "../shared.js"; import { probeNodeRequireRefusal } from "../npm.js"; import { isNpmStaticPackage } from "../npm-static.js"; +import { trackedReadFile } from "../input-tracker.js"; import { invalidJsonModuleDiag, requiresDynamicImportDiag } from "../../diagnostics/diagnostic.js"; import { BuiltinModuleFn, @@ -408,12 +408,7 @@ function lowerBuiltinOptionalDefault( const abs = spec.startsWith("/") ? spec : resolve(dirname(cr.baseFile.fileName), spec); - let text: string | null = null; - try { - text = readFileSync(abs, "utf8"); - } catch { - /* the fence below speaks */ - } + const text = trackedReadFile(abs); if (text === null) { L.noLowering( `createRequire's require of '${spec}' (no file at ${abs})`, diff --git a/packages/compiler/src/frontend/npm-static-rewrite.ts b/packages/compiler/src/frontend/npm-static-rewrite.ts index da1a09e5..2321fab6 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite.ts @@ -73,11 +73,11 @@ * untouched (null): its import-site errors stay, and the frontend's * offender attribution degrades the PACKAGE to the island with a note. */ -import { existsSync, readFileSync, statSync } from "node:fs"; import { dirname, join, resolve as resolvePath } from "node:path"; import ts from "typescript5"; import { cjsLexedExportsOf, cjsLexerVisibleNames } from "./cjs-lexer.js"; import { resolveExports } from "./npm.js"; +import { trackedDirectoryExists, trackedFileExists, trackedReadFile } from "./input-tracker.js"; /** True when `e` is exactly the `exports` identifier. */ function isExportsIdent(e: ts.Expression): boolean { @@ -233,26 +233,23 @@ function definePropertyExportOf( function resolveCjsBase(base: string): string | null { const candidates = [base, `${base}.js`, `${base}.cjs`]; for (const c of candidates) { - try { - if (existsSync(c) && statSync(c).isFile() && /\.(js|cjs)$/.test(c)) return c; - } catch { - /* keep probing */ - } + if (trackedFileExists(c) && /\.(js|cjs)$/.test(c)) return c; } try { - if (existsSync(base) && statSync(base).isDirectory()) { + if (trackedDirectoryExists(base)) { const pkgPath = resolvePath(base, "package.json"); - if (existsSync(pkgPath)) { - const main = (JSON.parse(readFileSync(pkgPath, "utf8")) as { main?: unknown }).main; + const pkgText = trackedReadFile(pkgPath); + if (pkgText !== null) { + const main = (JSON.parse(pkgText) as { main?: unknown }).main; if (typeof main === "string") { const m = resolvePath(base, main); for (const c of [m, `${m}.js`, `${m}.cjs`, resolvePath(m, "index.js")]) { - if (existsSync(c) && statSync(c).isFile() && /\.(js|cjs)$/.test(c)) return c; + if (trackedFileExists(c) && /\.(js|cjs)$/.test(c)) return c; } } } const idx = resolvePath(base, "index.js"); - if (existsSync(idx)) return idx; + if (trackedFileExists(idx)) return idx; } } catch { /* unresolved */ @@ -281,13 +278,14 @@ function resolveBareRequireCjs(fromFile: string, spec: string): string | null { for (let dir = dirname(fromFile); ; ) { const pkgDir = join(dir, "node_modules", name); try { - if (existsSync(pkgDir) && statSync(pkgDir).isDirectory()) { + if (trackedDirectoryExists(pkgDir)) { const pkgPath = join(pkgDir, "package.json"); let exports: unknown; try { - exports = existsSync(pkgPath) - ? (JSON.parse(readFileSync(pkgPath, "utf8")) as { exports?: unknown }).exports - : undefined; + const pkgText = trackedReadFile(pkgPath); + exports = pkgText === null + ? undefined + : (JSON.parse(pkgText) as { exports?: unknown }).exports; } catch { return null; } @@ -329,9 +327,10 @@ function requireTargetEsModuleStamped(fromFile: string, spec: string, depth = 0) // the nearest package.json "type" decides the .js format for (let dir = dirname(file); ; ) { const pkgPath = join(dir, "package.json"); - if (existsSync(pkgPath)) { + const pkgText = trackedReadFile(pkgPath); + if (pkgText !== null) { try { - if ((JSON.parse(readFileSync(pkgPath, "utf8")) as { type?: unknown }).type === "module") return true; + if ((JSON.parse(pkgText) as { type?: unknown }).type === "module") return true; } catch { /* unreadable — treat as CJS */ } @@ -342,12 +341,8 @@ function requireTargetEsModuleStamped(fromFile: string, spec: string, depth = 0) dir = parent; } } - let src: string; - try { - src = readFileSync(file, "utf8"); - } catch { - return false; - } + const src = trackedReadFile(file); + if (src === null) return false; try { if (cjsLexedExportsOf(src, file).exports.has("__esModule")) return true; } catch { @@ -366,7 +361,11 @@ function starTargetNames(file: string): Set { try { return cjsLexerVisibleNames( file, - (f) => readFileSync(f, "utf8"), + (f) => { + const source = trackedReadFile(f); + if (source === null) throw new Error(`cannot read ${f}`); + return source; + }, (from, spec) => resolveRelativeCjs(from, spec), ); } catch { diff --git a/packages/compiler/src/frontend/npm-static.test.ts b/packages/compiler/src/frontend/npm-static.test.ts new file mode 100644 index 00000000..a97b2f41 --- /dev/null +++ b/packages/compiler/src/frontend/npm-static.test.ts @@ -0,0 +1,16 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { npmStaticIneligibleReason } from "./npm-static.js"; + +test("an unreadable runtime entry is an eligibility refusal", async () => { + const dir = await mkdtemp("/tmp/scriptc-npm-static-unreadable-"); + try { + const root = join(dir, "node_modules", "example"); + expect(npmStaticIneligibleReason("example", `${root}/index.d.ts`, `${root}/index.js`)).toBe( + `its runtime entry ${root}/index.js cannot be read`, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 992cd9d7..9392f4da 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -54,10 +54,10 @@ * the state, so a flagless compile after a flagged one sees a clean * slate. */ -import { existsSync, readFileSync, realpathSync } from "node:fs"; import { dirname } from "node:path"; import { rewriteBundlerCjsExports } from "./npm-static-rewrite.js"; import { npmPackageNameOf, registerWorkspacePackage, workspacePackageOfPath } from "./shared.js"; +import { trackedExists, trackedReadFile, trackedRealpath } from "./input-tracker.js"; let activePackages: ReadonlySet = new Set(); @@ -129,11 +129,9 @@ const realpathProbed = new Set(); function registerWorkspaceRealpath(pkg: string, nmDir: string): void { if (realpathProbed.has(nmDir)) return; realpathProbed.add(nmDir); - try { - const real = realpathSync(nmDir).split("\\").join("/"); - if (real !== nmDir && !real.includes("/node_modules/")) registerWorkspacePackage(pkg, real); - } catch { - /* dangling symlink / missing dir — nothing to register */ + const real = trackedRealpath(nmDir)?.split("\\").join("/"); + if (real !== undefined && real !== nmDir && !real.includes("/node_modules/")) { + registerWorkspacePackage(pkg, real); } } @@ -287,22 +285,25 @@ function packageIsUntyped(path: string): boolean { if (hit !== undefined) return hit; let untyped = true; try { - const pkg = JSON.parse(readFileSync(`${pkgDir}/package.json`, "utf8")) as Record; - if (pkg["types"] !== undefined || pkg["typings"] !== undefined) untyped = false; - if (untyped && typeof pkg["exports"] === "object" && pkg["exports"] !== null) { - // a "types" condition anywhere inside exports is a claim too - untyped = !JSON.stringify(pkg["exports"]).includes('"types"'); + const pkgText = trackedReadFile(`${pkgDir}/package.json`); + if (pkgText !== null) { + const pkg = JSON.parse(pkgText) as Record; + if (pkg["types"] !== undefined || pkg["typings"] !== undefined) untyped = false; + if (untyped && typeof pkg["exports"] === "object" && pkg["exports"] !== null) { + // a "types" condition anywhere inside exports is a claim too + untyped = !JSON.stringify(pkg["exports"]).includes('"types"'); + } } } catch { /* no package.json — keep probing */ } - if (untyped && existsSync(`${pkgDir}/index.d.ts`)) untyped = false; + if (untyped && trackedExists(`${pkgDir}/index.d.ts`)) untyped = false; if (untyped) { // the @types twin, hoisted anywhere up the realm chain const mangled = mangledTypesName(dirName); for (let dir = dirname(pkgDir); ; ) { const parent = dirname(dir); - if (existsSync(`${dir}/node_modules/@types/${mangled}/package.json`) || existsSync(`${dir}/@types/${mangled}/package.json`)) { + if (trackedExists(`${dir}/node_modules/@types/${mangled}/package.json`) || trackedExists(`${dir}/@types/${mangled}/package.json`)) { untyped = false; break; } @@ -344,7 +345,7 @@ export function npmStaticFsShadow(): NpmStaticFsShadow | null { if ( (path.endsWith(".js") || path.endsWith(".cjs")) && isNodeModulesPathNorm(path) && - !existsSync(path.replace(/\.(js|cjs)$/, ".d.ts")) && // a sibling .d.ts types this very file + !trackedExists(path.replace(/\.(js|cjs)$/, ".d.ts")) && // a sibling .d.ts types this very file packageIsUntyped(path) ) { return "module.exports = (() => { let u; return u; })();\n"; @@ -354,7 +355,8 @@ export function npmStaticFsShadow(): NpmStaticFsShadow | null { if (target.viaTypes) return undefined; if (path.endsWith("/package.json") || path.endsWith("\\package.json")) { try { - return npmStaticTransformPkgJsonText(readFileSync(path, "utf8")); + const source = trackedReadFile(path); + return source === null ? undefined : npmStaticTransformPkgJsonText(source); } catch { return undefined; } @@ -373,11 +375,14 @@ export function npmStaticFsShadow(): NpmStaticFsShadow | null { if (hit !== undefined) return hit ?? undefined; let rewritten: string | null = null; try { - const answer = rewriteBundlerCjsExports(readFileSync(path, "utf8"), path); - if (answer !== null && typeof answer === "object") { - reportNpmStaticOffender(target.pkg, answer.degrade); - } else { - rewritten = answer; + const source = trackedReadFile(path); + if (source !== null) { + const answer = rewriteBundlerCjsExports(source, path); + if (answer !== null && typeof answer === "object") { + reportNpmStaticOffender(target.pkg, answer.degrade); + } else { + rewritten = answer; + } } } catch { rewritten = null; // unreadable/unparseable: fall through untouched @@ -444,12 +449,8 @@ export function npmStaticIneligibleReason( return "it ships no own .d.ts declaration surface"; } if (jsEntry === null) return "no runtime JS entry resolves"; - let source: string; - try { - source = readFileSync(jsEntry, "utf8"); - } catch { - return `its runtime entry ${jsEntry} cannot be read`; - } + const source = trackedReadFile(jsEntry); + if (source === null) return `its runtime entry ${jsEntry} cannot be read`; if (!looksUnminified(source)) return "its shipped JS looks minified"; if (hasTransformMarkers(source)) { return "its shipped JS carries build-transform markers (bundled/transpiled dist)"; diff --git a/packages/compiler/src/frontend/npm.ts b/packages/compiler/src/frontend/npm.ts index 8b919cd0..142e699d 100644 --- a/packages/compiler/src/frontend/npm.ts +++ b/packages/compiler/src/frontend/npm.ts @@ -85,10 +85,10 @@ * lives under its key (requireHelperOriginOf follows the import and * re-export hops). */ -import { readFileSync, realpathSync } from "node:fs"; import { dirname, extname, join, resolve } from "node:path"; import ts from "typescript5"; import { cjsLexedExportsOf } from "./cjs-lexer.js"; +import { trackedDirectoryExists, trackedFileExists, trackedReadFile, trackedRealpath } from "./input-tracker.js"; export type EmbeddedFormat = "esm" | "cjs" | "json"; @@ -732,21 +732,11 @@ interface Host { const realHost: Host = { readFile: (path) => { - try { - return readFileSync(path, "utf8"); - } catch { - return null; - } - }, - isFile: (path) => ts.sys.fileExists(path), - isDirectory: (path) => ts.sys.directoryExists(path), - realpath: (path) => { - try { - return realpathSync(path); - } catch { - return path; - } + return trackedReadFile(path); }, + isFile: (path) => trackedFileExists(path), + isDirectory: (path) => trackedDirectoryExists(path), + realpath: (path) => trackedRealpath(path) ?? path, }; interface PkgJson { diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 89e0e58d..8a4c41e1 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -40,7 +40,6 @@ * that path (no snapshot pins it). */ import { builtinModules } from "node:module"; -import { statSync } from "node:fs"; import { dirname, resolve } from "node:path"; import * as ts from "./ts7/adapter.js"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; @@ -76,6 +75,7 @@ import { unsupportedModuleFeatureOf, workspacePackageOfPath, } from "./shared.js"; +import { trackedFileExists } from "./input-tracker.js"; const BASE_OPTIONS: ts.Ts7CompilerOptions = { strict: true, @@ -401,9 +401,7 @@ export function loadProgram( ); } const declarationPath = resolve(file); - try { - if (!statSync(declarationPath).isFile()) throw new Error("not a file"); - } catch { + if (!trackedFileExists(declarationPath)) { throw new TypeError( `external type declaration for ${JSON.stringify(specifier)} does not name a readable file: ${declarationPath}`, ); diff --git a/packages/compiler/src/frontend/resolve.ts b/packages/compiler/src/frontend/resolve.ts index bdf2f0c4..68958a95 100644 --- a/packages/compiler/src/frontend/resolve.ts +++ b/packages/compiler/src/frontend/resolve.ts @@ -14,33 +14,21 @@ * implementations and requires identical answers. Change 5.9.3's options * and these tables are wrong — that is what the suite is for. */ -import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; import { dirname, isAbsolute, join, resolve } from "node:path"; import { isNpmStaticPackage, npmStaticPackageOfPath, npmStaticTransformPkgJson } from "./npm-static.js"; import { provenanceEntryFor } from "./provenance-registry.js"; +import { trackedAccessibleEntries, trackedDirectoryExists, trackedExists, trackedFileExists, trackedReadFile, trackedRealpath } from "./input-tracker.js"; function isFile(path: string): boolean { - try { - return statSync(path).isFile(); - } catch { - return false; - } + return trackedFileExists(path); } function isDirectory(path: string): boolean { - try { - return statSync(path).isDirectory(); - } catch { - return false; - } + return trackedDirectoryExists(path); } function realpathOr(path: string): string { - try { - return realpathSync(path); - } catch { - return path; - } + return trackedRealpath(path) ?? path; } interface PkgJson { @@ -63,9 +51,9 @@ function pkgJsonOf(dir: string): PkgJson | null { if (cached === undefined) { const path = join(dir, "package.json"); let parsed: PkgJson | null = null; - if (existsSync(path)) { + if (trackedExists(path)) { try { - parsed = JSON.parse(readFileSync(path, "utf8")) as PkgJson; + parsed = JSON.parse(trackedReadFile(path)!) as PkgJson; } catch { parsed = null; } @@ -114,12 +102,7 @@ function expandWorkspacePattern(root: string, pattern: string): string[] { const next: string[] = []; for (const dir of dirs) { if (seg === "*") { - let entries: string[]; - try { - entries = readdirSync(dir); - } catch { - continue; - } + const entries = trackedAccessibleEntries(dir)?.directories ?? []; for (const e of entries) { if (e === "node_modules" || e.startsWith(".")) continue; const child = join(dir, e); @@ -463,7 +446,7 @@ function nearestPkgDir(dir: string): string | null { function nearestPackageConfigDir(dir: string): string | null { let d = dir; for (;;) { - if (existsSync(join(d, "package.json"))) return d; + if (trackedExists(join(d, "package.json"))) return d; const parent = dirname(d); if (parent === d) return null; d = parent; diff --git a/packages/compiler/src/frontend/ts7/program.ts b/packages/compiler/src/frontend/ts7/program.ts index 74c91b0d..f319016f 100644 --- a/packages/compiler/src/frontend/ts7/program.ts +++ b/packages/compiler/src/frontend/ts7/program.ts @@ -15,7 +15,7 @@ * 387 cold). createProgram() takes an optional shared host — the default * spawns a private one that dispose() closes. */ -import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { API } from "typescript/unstable/sync"; import type { @@ -28,6 +28,7 @@ import type { SourceFile } from "typescript/unstable/ast"; import { CheckerFacade } from "./checker.js"; import { enumKeyOf, ModuleDetectionKind, ModuleKind, ModuleResolutionKind, ScriptTarget } from "./enums.js"; import { tsgoPath } from "../shared.js"; +import { trackedAccessibleEntries, trackedDirectoryExists, trackedFileExists, trackedReadFile, trackedRealpath } from "../input-tracker.js"; /** The compiler options our createProgram accepts: TS7's CompilerOptions * shape (numeric enums for target/module/moduleResolution — the enums module @@ -116,18 +117,22 @@ export class Ts7Host { readFile: (fileName) => { const virtual = virtualFiles.get(tsgoPath(fileName)); if (virtual !== undefined) return virtual; - if (shadow === null) return undefined; - if (shadow.hideFile(fileName)) return null; - return shadow.readFile(fileName); + if (shadow !== null) { + if (shadow.hideFile(fileName)) return null; + const replacement = shadow.readFile(fileName); + if (replacement !== undefined) return replacement; + } + return trackedReadFile(fileName); }, fileExists: (fileName) => { if (virtualFiles.has(tsgoPath(fileName))) return true; if (shadow !== null && shadow.hideFile(fileName)) return false; - return undefined; + return trackedFileExists(fileName); }, - directoryExists: () => undefined, - realpath: (path) => (virtualFiles.has(tsgoPath(path)) ? path : undefined), - getAccessibleEntries: () => undefined, + directoryExists: (path) => trackedDirectoryExists(path), + realpath: (path) => + virtualFiles.has(tsgoPath(path)) ? path : (trackedRealpath(path) ?? path), + getAccessibleEntries: (path) => trackedAccessibleEntries(path) ?? { files: [], directories: [] }, }, }); } @@ -309,20 +314,16 @@ export function findConfigFile( * are the direct implementations. */ export const sys = { fileExists(path: string): boolean { - return existsSync(path) && statSync(path).isFile(); + return trackedFileExists(path); }, readFile(path: string): string | undefined { - try { - return readFileSync(path, "utf8"); - } catch { - return undefined; - } + return trackedReadFile(path) ?? undefined; }, writeFile(path: string, data: string): void { writeFileSync(path, data); }, directoryExists(path: string): boolean { - return existsSync(path) && statSync(path).isDirectory(); + return trackedDirectoryExists(path); }, getCurrentDirectory(): string { return process.cwd(); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 948d7a4f..0ae28dbe 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,6 +1,6 @@ import { mkdir, rm, writeFile } from "node:fs/promises"; import { basename, dirname, join, resolve } from "node:path"; -import { CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, resolveCc, targetPlatform } from "./backend/cc.js"; +import { buildCacheRoot, CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform } from "./backend/cc.js"; import { emitModule } from "./backend/emission/emitter.js"; import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js"; import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; @@ -27,12 +27,14 @@ import { validateModule } from "./ir/validate.js"; import { canonicalBuiltinModule, checkPreflight, isNodeTypesPath, loadProgram, locOf, requiresOf, resolveNpmImport, type LoadResult } from "./frontend/program.js"; import { npmStaticIneligibleReason, npmStaticOffenders, npmStaticPackageOfPath } from "./frontend/npm-static.js"; import { provenanceSources } from "./frontend/provenance-registry.js"; -import { resolveBareModule } from "./frontend/resolve.js"; +import { clearResolveCaches, resolveBareModule } from "./frontend/resolve.js"; import { isJsSourceFileName, isRelativeSpecifier } from "./frontend/shared.js"; import { lowerToIr, type LowerOptions, type LowerResult } from "./frontend/lowering/lowerer.js"; import type { CoverageInput, NpmStaticStatus } from "./coverage/report.js"; import { loadFfiProfile, type FfiProfile } from "./ffi/profile.js"; import { hasForeignFfiCallback } from "./backend/ffi-callbacks.js"; +import { FrontendInputTracker, trackedReadFile } from "./frontend/input-tracker.js"; +import { libraryFrontendImplementationFingerprint, publishEarlyLibraryCache, readEarlyLibraryCache, type EarlyLibraryCacheOptions, type EarlyLibraryCachePublish, type EarlyLibraryNativeFeatures } from "./library/early-cache.js"; export const VERSION = "0.0.1"; @@ -517,6 +519,11 @@ function runFrontend( npmStatic?: readonly string[] | "auto" | "lib", externalTypes?: Readonly>, ): Frontend { + // Resolver package/workspace metadata is intentionally shared across the + // several load attempts of ONE auto-detection fixpoint, but never across + // separate compiles in a long-lived process. A cache miss must observe + // package.json edits before it can publish a new early-library entry. + clearResolveCaches(); const statuses: NpmStaticStatus[] = []; const npmSites = new Map(); const judged = new Set(); @@ -1499,7 +1506,77 @@ function mergeSidecarIntSlots( return { ok: true, config: cfg }; } +function libraryNativeFeatures( + mod: IrModule, + backend: "c" | "llvm", +): EarlyLibraryNativeFeatures { + return { + backend, + regex: moduleUsesRegex(mod), + assert: moduleUsesAssert(mod), + inspect: moduleUsesInspect(mod), + symbol: moduleUsesSymbol(mod), + searchParams: moduleUsesSearchParams(mod), + emitter: moduleUsesEmitter(mod), + zlib: moduleUsesZlib(mod), + copying: moduleUsesCopying(mod), + textDecoderLegacy: moduleUsesLegacyTextDecoder(mod), + }; +} + +function libraryLocalizeSymbols(profile: LibraryProfile): string[] | undefined { + return profile.localizeRuntime + ? [ + profile.initSymbol, + profile.sinkRegisterSymbol, + ...(profile.collectSymbol !== null ? [profile.collectSymbol] : []), + ...(profile.resultResetSymbol !== null ? [profile.resultResetSymbol] : []), + ...(profile.callbackRegisterSymbol !== null ? [profile.callbackRegisterSymbol] : []), + ...(profile.sidecar !== null + ? [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol] + : []), + ...profile.exports.map((entry) => entry.symbol), + ] + : undefined; +} + +async function compileLibraryNative( + profile: LibraryProfile, + cPath: string, + archivePath: string, + sanitize: boolean, + features: EarlyLibraryNativeFeatures, +): Promise { + const localizeSymbols = libraryLocalizeSymbols(profile); + await compileLibArchive({ + cPath, + outPath: archivePath, + cacheIdentity: "scriptc-generated-library-v1", + sanitize, + optimization: profile.optimization, + ...(localizeSymbols !== undefined ? { localizeSymbols } : {}), + ...(profile.instancePerThread ? { threadInstances: true } : {}), + regex: features.regex, + assert: features.assert, + inspect: features.inspect, + symbol: features.symbol, + searchParams: features.searchParams, + emitter: features.emitter, + zlib: features.zlib, + copying: features.copying, + textDecoderLegacy: features.textDecoderLegacy, + }); +} + export async function compileLibrary(opts: CompileLibraryOptions): Promise { + const frontendInputs = new FrontendInputTracker(); + return frontendInputs.run(() => compileLibraryTracked(opts, frontendInputs)); +} + +async function compileLibraryTracked( + opts: CompileLibraryOptions, + frontendInputs: FrontendInputTracker, +): Promise { const timingOn = process.env["SCRIPTC_TIMING"] === "1"; const timingStart = performance.now(); let timingLast = timingStart; @@ -1525,6 +1602,23 @@ export async function compileLibrary(opts: CompileLibraryOptions): Promise { + const path = join(directory, name); + trackedReadFile(path); + }); + } + if (directory === profileDir || dirname(directory) === directory) break; + } + const archivePath = opts.outPath ?? join( + opts.outDir, + `${basename(entryPath).replace(/\.(ts|js|mjs|cjs)$/, "")}.lib.a`, + ); // Mobile-target admission first — a pure env/host check, so a refused // pairing never reaches toolchain discovery. iOS targets (device and @@ -1593,6 +1687,49 @@ export async function compileLibrary(opts: CompileLibraryOptions): Promise e.symbol), - ] - : undefined; - await compileLibArchive({ + const nativeFeatures = libraryNativeFeatures(mod, profile.emission); + await compileLibraryNative( + profile, cPath, - outPath: archivePath, - cacheIdentity: "scriptc-generated-library-v1", - sanitize: opts.sanitize ?? false, - optimization: profile.optimization, - ...(localizeSymbols !== undefined ? { localizeSymbols } : {}), - ...(profile.instancePerThread ? { threadInstances: true } : {}), - regex: moduleUsesRegex(mod), - assert: moduleUsesAssert(mod), - inspect: moduleUsesInspect(mod), - symbol: moduleUsesSymbol(mod), - searchParams: moduleUsesSearchParams(mod), - emitter: moduleUsesEmitter(mod), - zlib: moduleUsesZlib(mod), - copying: moduleUsesCopying(mod), - textDecoderLegacy: moduleUsesLegacyTextDecoder(mod), - }); + archivePath, + opts.sanitize ?? false, + nativeFeatures, + ); timing("native-archive"); // The sidecar lands beside the compiled object, written by the same @@ -1895,6 +2005,16 @@ export async function compileLibrary(opts: CompileLibraryOptions): Promise undefined); + await pruneBuildCache(cacheRoot); + timing("early-cache-publish"); timing("complete"); return { ok: true, diff --git a/packages/compiler/src/library/early-cache.test.ts b/packages/compiler/src/library/early-cache.test.ts new file mode 100644 index 00000000..366b7b60 --- /dev/null +++ b/packages/compiler/src/library/early-cache.test.ts @@ -0,0 +1,329 @@ +import { mkdir, mkdtemp, readFile, readdir, rm, stat, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { FrontendInputTracker, trackedAccessibleEntries, trackedDirectoryExists, trackedFileExists, trackedReadFile } from "../frontend/input-tracker.js"; +import { + publishEarlyLibraryCache, + readEarlyLibraryCache, + type EarlyLibraryCacheOptions, +} from "./early-cache.js"; + +const scratch: string[] = []; + +afterEach(async () => { + await Promise.all(scratch.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +async function fixture(): Promise<{ + root: string; + options: EarlyLibraryCacheOptions; + source: string; + missing: string; + cPath: string; + irPath: string; + sidecarPath: string; +}> { + const dir = await mkdtemp(join(tmpdir(), "scriptc-early-cache-")); + scratch.push(dir); + const source = join(dir, "entry.ts"); + const profilePath = join(dir, "profile.json"); + const cPath = join(dir, "entry.lib.ll"); + const irPath = join(dir, "entry.lib.ir.json"); + const sidecarPath = join(dir, "entry.lib.a.contract.json"); + await Promise.all([ + writeFile(source, "export function value(): number { return 1; }\n"), + writeFile(profilePath, "{}\n"), + writeFile(cPath, "; generated llvm\n"), + writeFile(irPath, "{\"irVersion\":6}\n"), + writeFile(sidecarPath, "{\"contract\":true}\n"), + ]); + return { + root: join(dir, "cache"), + source, + missing: join(dir, "missing.ts"), + cPath, + irPath, + sidecarPath, + options: { + profilePath, + profileBytes: new TextEncoder().encode("{}\n"), + entryPath: source, + outDir: dir, + emitIr: true, + sanitize: false, + target: "test", + compiler: ["clang"], + nodeVersion: "v24-test", + implementation: "test-implementation", + }, + }; +} + +test("early library cache restores generated artifacts and metadata", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => { + trackedReadFile(f.source); + trackedFileExists(f.missing); + }); + const native = { + backend: "llvm" as const, + regex: false, + assert: true, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }; + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native, + frontend: tracker.snapshot(), + }); + await Promise.all([rm(f.cPath), rm(f.irPath), rm(f.sidecarPath)]); + const staleCPath = join(f.options.outDir, "entry.lib.c"); + await writeFile(staleCPath, "/* stale c backend */\n"); + + const hit = await readEarlyLibraryCache(f.root, f.options, null); + expect(hit).not.toBeNull(); + expect(hit?.native).toEqual(native); + expect(await readFile(f.cPath, "utf8")).toBe("; generated llvm\n"); + expect(await readFile(f.irPath, "utf8")).toContain("irVersion"); + expect(await readFile(f.sidecarPath, "utf8")).toContain("contract"); + await expect(readFile(staleCPath, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); +}); + +test("early library cache publishes after creating a fresh output directory", async () => { + const f = await fixture(); + const cacheRoot = await mkdtemp(join(tmpdir(), "scriptc-early-cache-root-")); + scratch.push(cacheRoot); + const outDir = join(f.options.outDir, "fresh-out"); + const cPath = join(outDir, "entry.lib.ll"); + const options = { ...f.options, outDir, emitIr: false }; + const tracker = new FrontendInputTracker(); + tracker.run(() => { + trackedAccessibleEntries(f.options.outDir); + trackedDirectoryExists(outDir); + trackedAccessibleEntries(outDir); + trackedReadFile(f.source); + }); + + await mkdir(outDir); + await writeFile(cPath, "; generated llvm in fresh output\n"); + await publishEarlyLibraryCache(cacheRoot, options, { + cPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }); + + await rm(cPath); + expect((await readEarlyLibraryCache(cacheRoot, options, undefined))?.cPath).toBe(cPath); + expect(await readFile(cPath, "utf8")).toContain("fresh output"); +}); + +test("early library cache hits refresh every payload's LRU time", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }); + const earlyRoot = join(f.root, "early-lib"); + const [key] = await readdir(earlyRoot); + const entry = join(earlyRoot, key!); + const cachePaths = [ + join(entry, "stamp.json"), + join(entry, "program.tu"), + join(entry, "program.ir.json"), + join(entry, "contract.json"), + ]; + const old = new Date("2000-01-01T00:00:00.000Z"); + await Promise.all(cachePaths.map((path) => utimes(path, old, old))); + + expect(await readEarlyLibraryCache(f.root, f.options, null)).not.toBeNull(); + for (const path of cachePaths) { + expect((await stat(path)).mtimeMs).toBeGreaterThan(old.getTime()); + } +}); + +test("early library cache misses on source edits and newly-resolved candidates", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => { + trackedReadFile(f.source); + trackedFileExists(f.missing); + }); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }); + + await writeFile(f.source, "export function value(): number { return 2; }\n"); + expect(await readEarlyLibraryCache(f.root, f.options, null)).toBeNull(); + await writeFile(f.cPath, "; generated llvm v2\n"); + const editedTracker = new FrontendInputTracker(); + editedTracker.run(() => { + trackedReadFile(f.source); + trackedFileExists(f.missing); + }); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: editedTracker.snapshot(), + }); + expect((await readEarlyLibraryCache(f.root, f.options, null))?.cPath).toBe(f.cPath); + expect(await readFile(f.cPath, "utf8")).toBe("; generated llvm v2\n"); + + await writeFile(f.source, "export function value(): number { return 1; }\n"); + await writeFile(f.missing, "export const appeared = true;\n"); + expect(await readEarlyLibraryCache(f.root, f.options, null)).toBeNull(); +}); + +test("early library cache is separated by the host Node version", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }); + + expect(await readEarlyLibraryCache(f.root, f.options, null)).not.toBeNull(); + expect(await readEarlyLibraryCache( + f.root, + { ...f.options, nodeVersion: "v25-test" }, + null, + )).toBeNull(); +}); + +test("early library cache rejects corrupted artifacts and metadata", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }); + const earlyRoot = join(f.root, "early-lib"); + const [key] = await readdir(earlyRoot); + await writeFile(join(earlyRoot, key!, "program.tu"), "corrupt\n"); + expect(await readEarlyLibraryCache(f.root, f.options, null)).toBeNull(); + + const stamp = join(earlyRoot, key!, "stamp.json"); + await writeFile(stamp, "{\"version\":1}\n"); + expect(await readEarlyLibraryCache(f.root, f.options, null)).toBeNull(); +}); + +test("disabled early library cache performs no reads or writes", async () => { + const f = await fixture(); + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + const publish = { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "llvm" as const, + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + }; + await publishEarlyLibraryCache(null, f.options, publish); + await expect(readdir(f.root)).rejects.toThrow(); + expect(await readEarlyLibraryCache(null, f.options, null)).toBeNull(); +}); diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts new file mode 100644 index 00000000..eb266af0 --- /dev/null +++ b/packages/compiler/src/library/early-cache.ts @@ -0,0 +1,362 @@ +import { createHash } from "node:crypto"; +import { chmod, copyFile, mkdir, readFile, readdir, rename, rm, utimes, writeFile } from "node:fs/promises"; +import { basename, dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { compilerReleaseVersion } from "./sidecar.js"; +import { frontendInputsStillMatch, validFrontendInputSnapshot, type FrontendInputExclusions, type FrontendInputSnapshot } from "../frontend/input-tracker.js"; + +interface CachedLibraryFile { + name: string; + digest: string; +} + +interface EarlyLibraryCacheStamp { + version: 1; + key: string; + frontend: FrontendInputSnapshot; + files: { + translationUnit: CachedLibraryFile; + ir: CachedLibraryFile | null; + sidecar: CachedLibraryFile | null; + }; + native: { + backend: "c" | "llvm"; + regex: boolean; + assert: boolean; + inspect: boolean; + symbol: boolean; + searchParams: boolean; + emitter: boolean; + zlib: boolean; + copying: boolean; + textDecoderLegacy: boolean; + }; + integrity: string; +} + +export interface EarlyLibraryNativeFeatures { + backend: "c" | "llvm"; + regex: boolean; + assert: boolean; + inspect: boolean; + symbol: boolean; + searchParams: boolean; + emitter: boolean; + zlib: boolean; + copying: boolean; + textDecoderLegacy: boolean; +} + +export interface EarlyLibraryCacheOptions { + profilePath: string; + profileBytes: Uint8Array; + entryPath: string; + outDir: string; + outPath?: string; + emitIr: boolean; + sanitize: boolean; + target: string; + compiler: string[]; + /** Host Node runtime whose builtin-module inventory participates in + * frontend classification. */ + nodeVersion: string; + implementation: string; +} + +export interface EarlyLibraryCacheHit { + cPath: string; + irPath?: string; + sidecarPath?: string; + native: EarlyLibraryNativeFeatures; +} + +export interface EarlyLibraryCachePublish extends EarlyLibraryCacheHit { + frontend: FrontendInputSnapshot; +} + +function validNativeFeatures(value: unknown): value is EarlyLibraryNativeFeatures { + if (value === null || typeof value !== "object") return false; + const native = value as Partial; + return (native.backend === "c" || native.backend === "llvm") && + [ + native.regex, + native.assert, + native.inspect, + native.symbol, + native.searchParams, + native.emitter, + native.zlib, + native.copying, + native.textDecoderLegacy, + ].every((flag) => typeof flag === "boolean"); +} + +function digest(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function cacheKey(options: EarlyLibraryCacheOptions): string { + return createHash("sha256") + .update("early-library-v1\0") + .update(compilerReleaseVersion()).update("\0") + .update(resolve(options.profilePath)).update("\0") + .update(options.profileBytes).update("\0") + .update(resolve(options.entryPath)).update("\0") + .update(resolve(options.outDir)).update("\0") + .update(options.outPath === undefined ? "" : resolve(options.outPath)).update("\0") + .update(options.emitIr ? "emit-ir" : "no-ir").update("\0") + .update(options.sanitize ? "sanitize" : "plain").update("\0") + .update(options.target).update("\0") + .update(options.compiler.join("\x1f")).update("\0") + .update(options.nodeVersion).update("\0") + .update(options.implementation) + .digest("hex"); +} + +/** + * Release versions separate published compilers. A source checkout keeps the + * package version unchanged while a branch is under test, so the built + * implementation directory also joins the key. Published packages execute + * the same `dist` tree and therefore get the same protection from partial or + * mixed installs without hashing the TypeScript source tree beside it. + */ +export async function libraryFrontendImplementationFingerprint(): Promise { + const moduleDir = dirname(fileURLToPath(import.meta.url)); + const implementationRoot = resolve(moduleDir, "..", ".."); + const files: string[] = []; + const walk = async (directory: string): Promise => { + const entries = await readdir(directory, { withFileTypes: true }); + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + const path = join(directory, entry.name); + if (entry.isDirectory()) await walk(path); + else if (entry.isFile()) files.push(path); + } + }; + await walk(implementationRoot); + const hash = createHash("sha256").update("scriptc-frontend-implementation-v1\0"); + for (const file of files) { + hash.update(relative(implementationRoot, file)).update("\0").update(await readFile(file)).update("\0"); + } + return hash.digest("hex"); +} + +function stampPath(root: string, options: EarlyLibraryCacheOptions): string { + return join(root, "early-lib", cacheKey(options), "stamp.json"); +} + +function stampIntegrity(stamp: Omit): string { + return createHash("sha256") + .update("early-library-stamp-v1\0") + .update(JSON.stringify(stamp)) + .digest("hex"); +} + +function outputPaths(options: EarlyLibraryCacheOptions, backend: "c" | "llvm"): { + cPath: string; + staleCPath: string; + irPath: string; +} { + const stem = basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, ""); + return { + cPath: join(options.outDir, `${stem}.lib.${backend === "llvm" ? "ll" : "c"}`), + staleCPath: join(options.outDir, `${stem}.lib.${backend === "llvm" ? "c" : "ll"}`), + irPath: join(options.outDir, `${stem}.lib.ir.json`), + }; +} + +function sidecarOutputPath(options: EarlyLibraryCacheOptions, configured: string | null): string { + const archivePath = archiveOutputPath(options); + return configured !== null ? resolve(dirname(archivePath), configured) : `${archivePath}.contract.json`; +} + +function archiveOutputPath(options: EarlyLibraryCacheOptions): string { + return options.outPath ?? join( + options.outDir, + `${basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, "")}.lib.a`, + ); +} + +function frontendOutputExclusions( + options: EarlyLibraryCacheOptions, + backend: "c" | "llvm", + sidecarPath: string | undefined, +): FrontendInputExclusions { + const paths = outputPaths(options, backend); + const outputArtifacts = [ + paths.cPath, + paths.staleCPath, + ...(options.emitIr ? [paths.irPath] : []), + archiveOutputPath(options), + ...(sidecarPath === undefined ? [] : [sidecarPath]), + ].map((path) => resolve(path)); + const outputDirectories = new Set(); + for (const artifact of outputArtifacts) { + for (let directory = dirname(artifact); ; directory = dirname(directory)) { + outputDirectories.add(directory); + if (dirname(directory) === directory) break; + } + } + return { + outputPaths: outputArtifacts, + outputDirectories, + }; +} + +async function readCachedFile(path: string, expected: string): Promise { + try { + const bytes = await readFile(path); + return digest(bytes) === expected ? bytes : null; + } catch { + return null; + } +} + +async function installBytes(bytes: Uint8Array, destination: string): Promise { + await mkdir(dirname(destination), { recursive: true }); + const tmp = join(dirname(destination), `.scriptc-early-hit-${process.pid}-${Math.random().toString(36).slice(2)}`); + try { + await writeFile(tmp, bytes, { mode: 0o600 }); + await chmod(tmp, 0o666 & ~process.umask()); + await rename(tmp, destination); + } finally { + await rm(tmp, { force: true }).catch(() => undefined); + } +} + +export async function readEarlyLibraryCache( + root: string | null, + options: EarlyLibraryCacheOptions, + sidecarConfiguredPath: string | null | undefined, +): Promise { + if (root === null) return null; + const path = stampPath(root, options); + try { + const stamp = JSON.parse(await readFile(path, "utf8")) as EarlyLibraryCacheStamp; + const { integrity, ...unsigned } = stamp; + if ( + stamp.version !== 1 || + stamp.key !== cacheKey(options) || + !validFrontendInputSnapshot(stamp.frontend) || + !validNativeFeatures(stamp.native) || + stamp.files?.translationUnit?.name !== "program.tu" || + !/^[0-9a-f]{64}$/.test(stamp.files.translationUnit.digest) || + (stamp.files.ir !== null && ( + stamp.files.ir?.name !== "program.ir.json" || !/^[0-9a-f]{64}$/.test(stamp.files.ir.digest) + )) || + (stamp.files.sidecar !== null && ( + stamp.files.sidecar?.name !== "contract.json" || !/^[0-9a-f]{64}$/.test(stamp.files.sidecar.digest) + )) || + stampIntegrity(unsigned) !== integrity || + !frontendInputsStillMatch( + stamp.frontend, + frontendOutputExclusions( + options, + stamp.native.backend, + stamp.files.sidecar === null + ? undefined + : sidecarOutputPath(options, sidecarConfiguredPath ?? null), + ), + ) || + (stamp.files.ir !== null) !== options.emitIr || + (stamp.files.sidecar !== null) !== (sidecarConfiguredPath !== undefined) + ) return null; + + const directory = dirname(path); + const [translationUnit, ir, sidecar] = await Promise.all([ + readCachedFile(join(directory, stamp.files.translationUnit.name), stamp.files.translationUnit.digest), + stamp.files.ir === null + ? Promise.resolve(null) + : readCachedFile(join(directory, stamp.files.ir.name), stamp.files.ir.digest), + stamp.files.sidecar === null + ? Promise.resolve(null) + : readCachedFile(join(directory, stamp.files.sidecar.name), stamp.files.sidecar.digest), + ]); + if ( + translationUnit === null || + stamp.files.ir !== null && ir === null || + stamp.files.sidecar !== null && sidecar === null + ) return null; + + const paths = outputPaths(options, stamp.native.backend); + await installBytes(translationUnit, paths.cPath); + await rm(paths.staleCPath, { force: true }); + if (ir !== null) await installBytes(ir, paths.irPath); + let sidecarPath: string | undefined; + if (sidecar !== null) { + sidecarPath = sidecarOutputPath(options, sidecarConfiguredPath ?? null); + await installBytes(sidecar, sidecarPath); + } + const now = new Date(); + await Promise.all([ + path, + join(directory, stamp.files.translationUnit.name), + ...(stamp.files.ir === null ? [] : [join(directory, stamp.files.ir.name)]), + ...(stamp.files.sidecar === null ? [] : [join(directory, stamp.files.sidecar.name)]), + ].map((cachePath) => utimes(cachePath, now, now).catch(() => undefined))); + return { + cPath: paths.cPath, + native: stamp.native, + ...(ir !== null ? { irPath: paths.irPath } : {}), + ...(sidecarPath !== undefined ? { sidecarPath } : {}), + }; + } catch { + return null; + } +} + +export async function publishEarlyLibraryCache( + root: string | null, + options: EarlyLibraryCacheOptions, + result: EarlyLibraryCachePublish, +): Promise { + if (root === null || !result.frontend.stable) return; + const destination = dirname(stampPath(root, options)); + const parent = dirname(destination); + const stage = join(parent, `.tmp-${basename(destination).slice(0, 12)}-${process.pid}-${Math.random().toString(36).slice(2)}`); + try { + await mkdir(stage, { recursive: true, mode: 0o700 }); + const publishFile = async (source: string, name: string): Promise => { + const target = join(stage, name); + await copyFile(source, target); + await chmod(target, 0o600); + return { name, digest: digest(await readFile(target)) }; + }; + const [translationUnit, ir, sidecar] = await Promise.all([ + publishFile(result.cPath, "program.tu"), + result.irPath === undefined ? Promise.resolve(null) : publishFile(result.irPath, "program.ir.json"), + result.sidecarPath === undefined ? Promise.resolve(null) : publishFile(result.sidecarPath, "contract.json"), + ]); + if (!frontendInputsStillMatch( + result.frontend, + frontendOutputExclusions(options, result.native.backend, result.sidecarPath), + )) return; + const unsigned: Omit = { + version: 1, + key: cacheKey(options), + frontend: result.frontend, + files: { translationUnit, ir, sidecar }, + native: result.native, + }; + const stamp: EarlyLibraryCacheStamp = { ...unsigned, integrity: stampIntegrity(unsigned) }; + await writeFile(join(stage, "stamp.json"), `${JSON.stringify(stamp)}\n`, { mode: 0o600 }); + await mkdir(destination, { recursive: true, mode: 0o700 }); + const install = async (name: string): Promise => { + const source = join(stage, name); + const target = join(destination, name); + await rename(source, target).catch(async () => { + // Windows does not replace an existing destination through rename. + // A racing reader sees either the old file or a miss; the stamp lands + // last, so no mixed artifact set can validate as a hit. + await rm(target, { force: true }); + await rename(source, target); + }); + }; + await install(translationUnit.name); + if (ir !== null) await install(ir.name); + if (sidecar !== null) await install(sidecar.name); + await install("stamp.json"); + } finally { + await rm(stage, { recursive: true, force: true }).catch(() => undefined); + } +} diff --git a/tests/harness/README.md b/tests/harness/README.md index 965913d7..a7781b15 100644 --- a/tests/harness/README.md +++ b/tests/harness/README.md @@ -73,6 +73,7 @@ Test runs are dominated by clang (~275 corpus programs × two lanes at -O2/-O1+A - **binaries** (`bin/`, cc.ts): key = resolved clang identity/version + target/compiler environment + implicit system-header dependency bytes + linker/assembler identities + runtime fingerprint (every runtime .c/.h + the vendor pin) + the full normalized command line + the emitted C bytes (byte-stable by project invariant). A hit skips native code generation and linking; the binary still RUNS live, so no comparison or sanitizer coverage is ever skipped. Each hit is checksum-verified. The sanitized lane's flags land in naturally distinct keys. FFI archive/object inputs and ambient system libraries always relink because their named files can hide mutable transitive dependencies. - **library archives** (`lib/`, cc.ts): key = resolved clang and archiver identities/versions + target/compiler environment and implicit dependencies + runtime fingerprint + target/flags + gated runtime-source set + emitted program-TU bytes. A checksum-verified hit skips native code generation and `ar`. +- **early library frontend** (`early-lib/`, library/early-cache.ts): exact library repeats validate content hashes for every file the TypeScript frontend read plus recorded failed-resolution and directory-enumeration probes, then restore the generated C/LLVM unit, optional IR, sidecar, and native feature gates without spawning TypeScript or lowering again. Source/config/package edits and newly appearing resolution candidates miss; the native archive tier still performs its own toolchain/runtime checks. - **runtime objects** (`obj/`, cc.ts): per-flavor .o for the runtime sources, including a distinct `-DSCR_LIB` flavor, so an edited executable or library recompiles only the program's own translation unit before linking or archiving. Each object carries a verified digest; a damaged entry is rebuilt before it reaches the linker or archiver. Publication rechecks the runtime and implicit-toolchain fingerprints after compilation so a concurrent source/header edit cannot place new bytes under an old key. Compiles route through ccache when installed, silently falling back when not. - **oracle results** (`oracle/`, differential.test.ts): Node's stdout/exit per program, keyed by program bytes + the spawned node's version + shim contents + invocation shape. Only the spawn is skipped; the comparison never changes. Real-time programs (setTimeout/setInterval/Promise.race — 18 of 298) are excluded and always spawn Node live: their stdout is a timer interleave that Node and the native binary only agree on under the same instantaneous load, so a cached verdict from one run must never meet a live native run from another.