From a442d3afe0d124a6006f13fbc1483eb0f9ac0549 Mon Sep 17 00:00:00 2001 From: ls02155419 Date: Mon, 9 Nov 2026 21:25:31 -0800 Subject: [PATCH] Build a project's map in one process at a time Each MCP server analyzed a project on its own, so sessions that opened the same repository at once ran the same analysis side by side. A map build now takes an exclusive transaction on build.lock beside the map; a process that finds it held waits, then finds the version built there, or builds from it. The system releases the lock when its process stops, however it stops. A capture whose version already exists takes no lock, so queries do not wait. Co-Authored-By: Claude Opus 5.5 --- src/code-index.ts | 17 +++++++- src/layermap.ts | 36 ++++++++++++++++- test/store.test.ts | 99 ++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 147 insertions(+), 5 deletions(-) diff --git a/src/code-index.ts b/src/code-index.ts index 87ac63d..d95d2ba 100644 --- a/src/code-index.ts +++ b/src/code-index.ts @@ -35,6 +35,12 @@ export class CodeIndex { readonly store: CodeIndexStore, private readonly clock: () => string = () => new Date().toISOString(), private readonly analyzer: ProjectMapAnalyzer = new ProjectMapAnalyzer(), + /** + * Holds the store's map builds to one at a time across the processes that share it, until the + * returned release; a store only this index writes needs none. + */ + private readonly lockBuilds: (signal: AbortSignal) => Promise<() => void> = async () => + () => {}, ) { this.query = new CodeIndexQuery(store, analyzer.id, (signal, read) => this.read(signal, read)); this.capture = new CodeIndexCapture(analyzer.id); @@ -150,7 +156,16 @@ export class CodeIndex { this.store.discardMapStage(stage); } }; - const completion = run(this.mapParent(result, startFromProjectRef)).finally(() => { + // A build that waited for another process's finds the version built there, or builds from what + // that one built. + const completion = (async () => { + const release = await this.lockBuilds(controller.signal); + try { + if (!this.store.getVersion(ref)) await run(this.mapParent(result, startFromProjectRef)); + } finally { + release(); + } + })().finally(() => { if (this.mapBuilds.get(ref)?.completion === completion) this.mapBuilds.delete(ref); }); const build = { controller, completion, waiters: new Set() }; diff --git a/src/layermap.ts b/src/layermap.ts index 427f688..18cb73b 100644 --- a/src/layermap.ts +++ b/src/layermap.ts @@ -2,6 +2,7 @@ import { createHash, randomUUID } from "node:crypto"; import { mkdirSync, statSync } from "node:fs"; import { lstat, realpath } from "node:fs/promises"; import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; import { DatabaseSync, type DatabaseSyncInstance } from "@photostructure/sqlite"; import { CodeIndex } from "./code-index"; import { @@ -57,6 +58,10 @@ const BUSY_TIMEOUT = "PRAGMA busy_timeout = 30000;"; const JOURNAL_LIMIT = 32 * 1024 * 1024; /** Captures of a directory whose files keep changing while they are read, before giving up. */ const CAPTURE_ATTEMPTS = 3; +/** How often a build waiting for another process's checks whether that one has finished. */ +const BUILD_WAIT_MS = 500; +/** SQLite's code for a database another connection has locked. */ +const SQLITE_BUSY = 5; // What a capture meets when files change while it reads or analyzes them, or when another // process's cleanup removed the version it found before it held it. const CAPTURE_AGAIN = new Set([ @@ -135,6 +140,33 @@ export function openMapDatabase(file: string): DatabaseSyncInstance { } } +/** + * Holds a project's map builds to one at a time across processes, until the returned release: the + * lock is an exclusive transaction on a file beside the map, which the system releases when its + * process stops, however it stops. + */ +async function lockMapBuilds(file: string, signal: AbortSignal): Promise<() => void> { + const lock = new DatabaseSync(file); + try { + lock.exec("PRAGMA busy_timeout = 0;"); + for (;;) { + try { + lock.exec("BEGIN EXCLUSIVE"); + // Closing ends the transaction. + return () => lock.close(); + } catch (error) { + if ((error as { errcode?: number }).errcode !== SQLITE_BUSY) throw error; + } + await delay(BUILD_WAIT_MS, undefined, { signal }).catch(() => { + throw new ProjectEvidenceError("PROJECT_READ_CANCELLED", true); + }); + } + } catch (error) { + lock.close(); + throw error; + } +} + /** Where a project's map lives: one database per canonical project directory. */ export const mapDatabasePath = (cacheDirectory: string, canonicalProject: string) => path.join( @@ -202,7 +234,9 @@ export class LayerMap { project, source, store, - new CodeIndex(store, undefined, analyzer), + new CodeIndex(store, undefined, analyzer, (signal) => + lockMapBuilds(path.join(path.dirname(file), "build.lock"), signal), + ), analyzer.id, { maxBytes: options.maxOutputBytes ?? 96 * 1024, envelopeBytes: 0 }, `${file}-wal`, diff --git a/test/store.test.ts b/test/store.test.ts index 3c7589c..a0c0964 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -1,15 +1,18 @@ import assert from "node:assert/strict"; import { execFile, spawn } from "node:child_process"; +import { once } from "node:events"; +import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; -import { test } from "node:test"; +import { type TestContext, test } from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { DatabaseSync } from "@photostructure/sqlite"; import { CodeCommitReader } from "../src/code-commit-reader"; import type { CodeIndexStore } from "../src/code-index-store"; import { GitRunner } from "../src/git-runner"; -import { mapDatabasePath } from "../src/layermap"; -import { FILES, openMap, project, sandbox, signal } from "./support"; +import { LayerMap, mapDatabasePath } from "../src/layermap"; +import { analyzers, FILES, openMap, project, sandbox, signal } from "./support"; const run = promisify(execFile); const target = (factor: number) => @@ -48,6 +51,96 @@ test("processes opening a new map at once all open it", async () => { ); }); +// The analyzers, with a TypeScript worker that notes each start before it runs the real one. +async function countedAnalyzers() { + const resources = await analyzers(); + const directory = await sandbox(); + const log = path.join(directory, "starts"); + const worker = path.join(directory, "worker.mjs"); + await writeFile( + worker, + `import { appendFileSync } from "node:fs";\nappendFileSync(${JSON.stringify(log)}, "start\\n");\nawait import(${JSON.stringify(pathToFileURL(resources.typescript.workerPath).href)});\n`, + ); + return { + analyzers: { ...resources, typescript: { ...resources.typescript, workerPath: worker } }, + starts: async () => (await readFile(log, "utf8").catch(() => "")).split("\n").length - 1, + }; +} + +// A session's map of the project, closed when the test ends. +async function openCounted( + t: TestContext, + root: string, + cacheDirectory: string, + counted: Awaited>, +) { + const map = await LayerMap.open({ project: root, cacheDirectory, analyzers: counted.analyzers }); + t.after(() => map.close()); + return map; +} + +test("sessions that open a project at once build its map once, and each finds it", async (t) => { + const { root } = await project(FILES); + // One session alone, for how many analyses a build of the project takes. + const alone = await countedAnalyzers(); + await (await openCounted(t, root, path.join(await sandbox(), "cache"), alone)).refresh(signal()); + const build = await alone.starts(); + assert.ok(build > 0); + + const shared = await countedAnalyzers(); + const cache = path.join(await sandbox(), "cache"); + const sessions = [ + await openCounted(t, root, cache, shared), + await openCounted(t, root, cache, shared), + ]; + const [first, second] = await Promise.all(sessions.map((map) => map.refresh(signal()))); + assert.equal(first, second); + assert.equal(await shared.starts(), build); +}); + +// Bounded: a build that went on waiting would otherwise hold the run open. +test("a build waits while another process builds the project's map, and goes on once that one is killed", { + timeout: 60_000, +}, async (t) => { + const { root, write } = await project(FILES); + const cache = path.join(await sandbox(), "cache"); + const counted = await countedAnalyzers(); + const map = await openCounted(t, root, cache, counted); + const version = await map.refresh(signal()); + // A process stopped part way through a build, holding the project's lock until it is killed. + const lock = path.join(path.dirname(mapDatabasePath(cache, map.project)), "build.lock"); + const holder = spawn( + process.execPath, + [ + "--input-type=module", + "-e", + // Kept referenced: a collected connection closes, ending its transaction. + `import { DatabaseSync } from "@photostructure/sqlite"; globalThis.lock = new DatabaseSync(${JSON.stringify(lock)}); globalThis.lock.exec("BEGIN EXCLUSIVE"); process.stdout.write("held\\n"); setInterval(() => {}, 1000);`, + ], + { cwd: packageRoot, stdio: ["ignore", "pipe", "inherit"] }, + ); + t.after(() => holder.kill("SIGKILL")); + await once(holder.stdout, "data"); + // A directory whose map is built meanwhile needs no build, and does not wait. + assert.equal(await map.refresh(signal()), version); + + await write("src/target.ts", target(3)); + const built = await counted.starts(); + const refreshed = map.refresh(signal()); + // A build still waiting stops when its caller cancels it. + const other = await openCounted(t, root, cache, counted); + const cancel = new AbortController(); + const cancelled = other.refresh(cancel.signal); + await delay(1_500); + // Neither has started analyzing while the other process builds. + assert.equal(await counted.starts(), built); + cancel.abort(); + await assert.rejects(cancelled, { code: "PROJECT_READ_CANCELLED" }); + + holder.kill("SIGKILL"); + assert.notEqual(await refreshed, version); +}); + test("a version another process holds outlasts this one's cleanup, until the hold expires", async (t) => { const { root, write } = await project(FILES); const cache = path.join(await sandbox(), "cache");