Onboard the lib task to code generation caching and include it in generate

CONTRIBUTING.md

@@ -104,6 +104,7 @@ npx hereby test          # Run compiler and language-service Go tests
 npx hereby test:all      # Also run benchmarks, tools, and API tests
 npx hereby lint          # Run custom golangci-lint for both Go modules
 npx hereby generate:go   # Run the generators also exposed through go generate
+npx hereby generate:libs # Incrementally copy declaration libraries to built/local
 npx hereby generate      # Regenerate all sources, assets, localization, and vendored files
 npx hereby format        # Format Go, TypeScript, JSON, and YAML
 npx hereby check:format  # Check formatting without changing files
@@ -117,6 +118,11 @@ to bypass incremental caches. `generate` includes every generator and fetches
 the pinned LSP model when its local cache is missing or stale. Use
 `generate:go` for the narrower scope of `go generate ./...` in the `tsc` module.
 
+`generate` also includes `generate:libs`, which caches declaration libraries as
+one set. When sources or copies change, it removes the destination's `lib*.d.ts`
+files and recopies the current set, preserving non-library files. The `lib` task
+remains an alias for `generate:libs`.
+
 Package-specific commands:
 
 ```bash

Herebyfile.mjs

@@ -223,19 +223,33 @@ const libsRegexp = /(?:^|[\\/])internal[\\/]bundled[\\/]libs[\\/]/;
  * @param {string} out
  */
 async function generateLibs(out) {
+    const { GeneratedDirectory } = await import("./tools/scripts/gen/generatedFile.mts");
+    out = path.resolve(__dirname, out);
+    const sourceDir = path.resolve(__dirname, libsDir);
+    const libs = await fs.promises.readdir(sourceDir);
+    const output = new GeneratedDirectory(out, [__filename, ...libs.map(lib => path.join(sourceDir, lib))], "lib*.d.ts");
+    if (output.isCurrent(!!options.force)) {
+        console.log(`Library files in ${out} are up to date.`);
+        return;
+    }
+    output.invalidate();
     await fs.promises.mkdir(out, { recursive: true });
-
-    const libs = await fs.promises.readdir(libsDir);
-
-    await Promise.all(libs.map(async lib => {
-        fs.promises.copyFile(path.join(libsDir, lib), path.join(out, lib));
-    }));
+    await Promise.all(fs.globSync("lib*.d.ts", { cwd: out }).map(lib => fs.promises.unlink(path.join(out, lib))));
+    await Promise.all(libs.map(lib => fs.promises.copyFile(path.join(sourceDir, lib), path.join(out, lib))));
+    output.markCurrent();
+    console.log(`Copied ${libs.length} library files to ${out}.`);
 }
 
+export const generateLib = task({
+    name: "generate:libs",
+    description: "Copies the libs to built/local. Pass --force to recopy unchanged files.",
+    run: () => generateLibs(builtLocal),
+});
+
 export const lib = task({
     name: "lib",
-    description: "Copies the libs to built/local.",
-    run: () => generateLibs(builtLocal),
+    description: "Alias for generate:libs.",
+    dependencies: [generateLib],
 });
 
 /**
@@ -712,8 +726,8 @@ const generateCompiler = task({
 
 export const generate = task({
     name: "generate",
-    description: "Runs all code generation, including AST, LSP, APIs, extension localization, and vendored dependencies.",
-    dependencies: [generateCompiler, generateSync, generateExtensionTest, generateVendor],
+    description: "Runs all code generation, including AST, LSP, APIs, library copies, extension localization, and vendored dependencies.",
+    dependencies: [generateCompiler, generateSync, generateExtensionTest, generateVendor, generateLib],
 });
 
 const coverageDir = path.join(__dirname, "coverage");

tools/scripts/gen/generatedFile.mts

@@ -59,9 +59,13 @@ export class GeneratedFile {
         return JSON.stringify([this.inputHash, digest(output)]);
     }
 
+    protected readOutput(): Buffer | undefined {
+        return readIfExists(this.fileName);
+    }
+
     isCurrent(force = false): boolean {
         if (force) return false;
-        const output = readIfExists(this.fileName);
+        const output = this.readOutput();
         return output !== undefined && readIfExists(this.cacheFile)?.toString() === this.state(output);
     }
 
@@ -79,8 +83,29 @@ export class GeneratedFile {
     markCurrent(): void {
         invalidateFileFingerprints(this.fileName);
         if (this.inputHash !== this.hashInputs(true)) return;
-        const state = this.state(fs.readFileSync(this.fileName));
+        const output = this.readOutput();
+        if (output === undefined) throw new Error(`Missing generated output: ${this.fileName}`);
+        const state = this.state(output);
         fs.mkdirSync(path.dirname(this.cacheFile), { recursive: true });
         fs.writeFileSync(this.cacheFile, state);
     }
 }
+
+export class GeneratedDirectory extends GeneratedFile {
+    private readonly pattern: string;
+
+    constructor(directory: string, inputs: readonly string[], pattern: string) {
+        super(directory, inputs, undefined, pattern);
+        this.pattern = pattern;
+    }
+
+    protected override readOutput(): Buffer | undefined {
+        if (!fs.existsSync(this.fileName)) return undefined;
+        return Buffer.from(JSON.stringify(
+            fs.globSync(this.pattern, { cwd: this.fileName }).sort().map(file => [
+                file,
+                digest(fs.readFileSync(path.join(this.fileName, file))),
+            ]),
+        ));
+    }
+}

tools/scripts/gen/generatedFile.test.mts

@@ -12,6 +12,7 @@ import ts from "typescript";
 import cache from "./cache.mts";
 import {
     defaultCacheDirectory,
+    GeneratedDirectory,
     GeneratedFile,
 } from "./generatedFile.mts";
 import {
@@ -133,6 +134,86 @@ test("generated files are current only while their inputs and formatted output m
     assert.equal(generated().isCurrent(), false);
 });
 
+test("generated directories track matching file names and contents", context => {
+    const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-directory-"));
+    const generated = () => new GeneratedDirectory(directory, [], "lib*.d.ts");
+    const output = generated();
+    context.after(() => {
+        output.invalidate();
+        fs.rmSync(directory, { recursive: true, force: true });
+    });
+    const first = path.join(directory, "lib.first.d.ts");
+    const second = path.join(directory, "lib.second.d.ts");
+    const unrelated = path.join(directory, "notes.txt");
+    fs.writeFileSync(first, "first");
+    fs.writeFileSync(unrelated, "unrelated");
+    assert.equal(output.isCurrent(), false);
+    output.markCurrent();
+    assert.equal(generated().isCurrent(), true);
+    fs.utimesSync(first, new Date(0), new Date(0));
+    fs.writeFileSync(unrelated, "changed but ignored");
+    assert.equal(generated().isCurrent(), true);
+    fs.rmSync(unrelated);
+    assert.equal(generated().isCurrent(), true);
+    fs.writeFileSync(first, "edited");
+    assert.equal(generated().isCurrent(), false);
+    fs.writeFileSync(first, "first");
+    assert.equal(generated().isCurrent(), true);
+    fs.renameSync(first, second);
+    assert.equal(generated().isCurrent(), false);
+    fs.writeFileSync(first, "first");
+    assert.equal(generated().isCurrent(), false);
+    fs.rmSync(second);
+    assert.equal(generated().isCurrent(), true);
+    fs.rmSync(first);
+    assert.equal(generated().isCurrent(), false);
+});
+
+test("generated directories distinguish missing and empty outputs and support invalidation", context => {
+    const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-directory-"));
+    const destination = path.join(directory, "output");
+    const generated = () => new GeneratedDirectory(destination, [], "*.d.ts");
+    const output = generated();
+    context.after(() => {
+        output.invalidate();
+        fs.rmSync(directory, { recursive: true, force: true });
+    });
+    assert.equal(output.isCurrent(), false);
+    assert.throws(() => output.markCurrent(), /Missing generated output/);
+    fs.mkdirSync(destination);
+    output.markCurrent();
+    assert.equal(generated().isCurrent(), true);
+    assert.equal(generated().isCurrent(true), false);
+    assert.equal(new GeneratedDirectory(destination, [], "*.txt").isCurrent(), false);
+    output.invalidate();
+    assert.equal(generated().isCurrent(), false);
+    output.markCurrent();
+    assert.equal(generated().isCurrent(), true);
+    fs.rmdirSync(destination);
+    assert.equal(generated().isCurrent(), false);
+});
+
+test("generated directories reject stale inputs when marking current", context => {
+    const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-directory-"));
+    const input = path.join(directory, "input.txt");
+    fs.writeFileSync(input, "initial");
+    fs.writeFileSync(path.join(directory, "lib.d.ts"), "generated");
+    const generated = () => new GeneratedDirectory(directory, [input], "*.d.ts");
+    const output = generated();
+    context.after(() => {
+        output.invalidate();
+        fs.rmSync(directory, { recursive: true, force: true });
+    });
+    output.markCurrent();
+    assert.equal(generated().isCurrent(), true);
+    fs.writeFileSync(input, "changed during generation");
+    assert.equal(generated().isCurrent(), false);
+    output.markCurrent();
+    assert.equal(generated().isCurrent(), false);
+    generated().markCurrent();
+    assert.equal(generated().isCurrent(), true);
+});
+
 test("generated files share input fingerprints and refresh changed inputs", context => {
     const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-fingerprint-"));
     context.after(() => fs.rmSync(directory, { recursive: true, force: true }));