Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/code-index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ export class CodeIndex {
readonly store: CodeIndexStore,
private readonly clock: () => string = () => new Date().toISOString(),
private readonly analyzer: ProjectMapAnalyzer = new ProjectMapAnalyzer(),
/**
* Holds the store's map builds to one at a time across the processes that share it, until the
* returned release; a store only this index writes needs none.
*/
private readonly lockBuilds: (signal: AbortSignal) => Promise<() => void> = async () =>
() => {},
) {
this.query = new CodeIndexQuery(store, analyzer.id, (signal, read) => this.read(signal, read));
this.capture = new CodeIndexCapture(analyzer.id);
Expand Down Expand Up @@ -150,7 +156,16 @@ export class CodeIndex {
this.store.discardMapStage(stage);
}
};
const completion = run(this.mapParent(result, startFromProjectRef)).finally(() => {
// A build that waited for another process's finds the version built there, or builds from what
// that one built.
const completion = (async () => {
const release = await this.lockBuilds(controller.signal);
try {
if (!this.store.getVersion(ref)) await run(this.mapParent(result, startFromProjectRef));
} finally {
release();
}
})().finally(() => {
if (this.mapBuilds.get(ref)?.completion === completion) this.mapBuilds.delete(ref);
});
const build = { controller, completion, waiters: new Set<symbol>() };
Expand Down
36 changes: 35 additions & 1 deletion src/layermap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { createHash, randomUUID } from "node:crypto";
import { mkdirSync, statSync } from "node:fs";
import { lstat, realpath } from "node:fs/promises";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { DatabaseSync, type DatabaseSyncInstance } from "@photostructure/sqlite";
import { CodeIndex } from "./code-index";
import {
Expand Down Expand Up @@ -57,6 +58,10 @@ const BUSY_TIMEOUT = "PRAGMA busy_timeout = 30000;";
const JOURNAL_LIMIT = 32 * 1024 * 1024;
/** Captures of a directory whose files keep changing while they are read, before giving up. */
const CAPTURE_ATTEMPTS = 3;
/** How often a build waiting for another process's checks whether that one has finished. */
const BUILD_WAIT_MS = 500;
/** SQLite's code for a database another connection has locked. */
const SQLITE_BUSY = 5;
// What a capture meets when files change while it reads or analyzes them, or when another
// process's cleanup removed the version it found before it held it.
const CAPTURE_AGAIN = new Set([
Expand Down Expand Up @@ -135,6 +140,33 @@ export function openMapDatabase(file: string): DatabaseSyncInstance {
}
}

/**
* Holds a project's map builds to one at a time across processes, until the returned release: the
* lock is an exclusive transaction on a file beside the map, which the system releases when its
* process stops, however it stops.
*/
async function lockMapBuilds(file: string, signal: AbortSignal): Promise<() => void> {
const lock = new DatabaseSync(file);
try {
lock.exec("PRAGMA busy_timeout = 0;");
for (;;) {
try {
lock.exec("BEGIN EXCLUSIVE");
// Closing ends the transaction.
return () => lock.close();
} catch (error) {
if ((error as { errcode?: number }).errcode !== SQLITE_BUSY) throw error;
}
await delay(BUILD_WAIT_MS, undefined, { signal }).catch(() => {
throw new ProjectEvidenceError("PROJECT_READ_CANCELLED", true);
});
}
} catch (error) {
lock.close();
throw error;
}
}

/** Where a project's map lives: one database per canonical project directory. */
export const mapDatabasePath = (cacheDirectory: string, canonicalProject: string) =>
path.join(
Expand Down Expand Up @@ -202,7 +234,9 @@ export class LayerMap {
project,
source,
store,
new CodeIndex(store, undefined, analyzer),
new CodeIndex(store, undefined, analyzer, (signal) =>
lockMapBuilds(path.join(path.dirname(file), "build.lock"), signal),
),
analyzer.id,
{ maxBytes: options.maxOutputBytes ?? 96 * 1024, envelopeBytes: 0 },
`${file}-wal`,
Expand Down
99 changes: 96 additions & 3 deletions test/store.test.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
import assert from "node:assert/strict";
import { execFile, spawn } from "node:child_process";
import { once } from "node:events";
import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { test } from "node:test";
import { type TestContext, test } from "node:test";
import { setTimeout as delay } from "node:timers/promises";
import { fileURLToPath, pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { DatabaseSync } from "@photostructure/sqlite";
import { CodeCommitReader } from "../src/code-commit-reader";
import type { CodeIndexStore } from "../src/code-index-store";
import { GitRunner } from "../src/git-runner";
import { mapDatabasePath } from "../src/layermap";
import { FILES, openMap, project, sandbox, signal } from "./support";
import { LayerMap, mapDatabasePath } from "../src/layermap";
import { analyzers, FILES, openMap, project, sandbox, signal } from "./support";

const run = promisify(execFile);
const target = (factor: number) =>
Expand Down Expand Up @@ -48,6 +51,96 @@ test("processes opening a new map at once all open it", async () => {
);
});

// The analyzers, with a TypeScript worker that notes each start before it runs the real one.
async function countedAnalyzers() {
const resources = await analyzers();
const directory = await sandbox();
const log = path.join(directory, "starts");
const worker = path.join(directory, "worker.mjs");
await writeFile(
worker,
`import { appendFileSync } from "node:fs";\nappendFileSync(${JSON.stringify(log)}, "start\\n");\nawait import(${JSON.stringify(pathToFileURL(resources.typescript.workerPath).href)});\n`,
);
return {
analyzers: { ...resources, typescript: { ...resources.typescript, workerPath: worker } },
starts: async () => (await readFile(log, "utf8").catch(() => "")).split("\n").length - 1,
};
}

// A session's map of the project, closed when the test ends.
async function openCounted(
t: TestContext,
root: string,
cacheDirectory: string,
counted: Awaited<ReturnType<typeof countedAnalyzers>>,
) {
const map = await LayerMap.open({ project: root, cacheDirectory, analyzers: counted.analyzers });
t.after(() => map.close());
return map;
}

test("sessions that open a project at once build its map once, and each finds it", async (t) => {
const { root } = await project(FILES);
// One session alone, for how many analyses a build of the project takes.
const alone = await countedAnalyzers();
await (await openCounted(t, root, path.join(await sandbox(), "cache"), alone)).refresh(signal());
const build = await alone.starts();
assert.ok(build > 0);

const shared = await countedAnalyzers();
const cache = path.join(await sandbox(), "cache");
const sessions = [
await openCounted(t, root, cache, shared),
await openCounted(t, root, cache, shared),
];
const [first, second] = await Promise.all(sessions.map((map) => map.refresh(signal())));
assert.equal(first, second);
assert.equal(await shared.starts(), build);
});

// Bounded: a build that went on waiting would otherwise hold the run open.
test("a build waits while another process builds the project's map, and goes on once that one is killed", {
timeout: 60_000,
}, async (t) => {
const { root, write } = await project(FILES);
const cache = path.join(await sandbox(), "cache");
const counted = await countedAnalyzers();
const map = await openCounted(t, root, cache, counted);
const version = await map.refresh(signal());
// A process stopped part way through a build, holding the project's lock until it is killed.
const lock = path.join(path.dirname(mapDatabasePath(cache, map.project)), "build.lock");
const holder = spawn(
process.execPath,
[
"--input-type=module",
"-e",
// Kept referenced: a collected connection closes, ending its transaction.
`import { DatabaseSync } from "@photostructure/sqlite"; globalThis.lock = new DatabaseSync(${JSON.stringify(lock)}); globalThis.lock.exec("BEGIN EXCLUSIVE"); process.stdout.write("held\\n"); setInterval(() => {}, 1000);`,
],
{ cwd: packageRoot, stdio: ["ignore", "pipe", "inherit"] },
);
t.after(() => holder.kill("SIGKILL"));
await once(holder.stdout, "data");
// A directory whose map is built meanwhile needs no build, and does not wait.
assert.equal(await map.refresh(signal()), version);

await write("src/target.ts", target(3));
const built = await counted.starts();
const refreshed = map.refresh(signal());
// A build still waiting stops when its caller cancels it.
const other = await openCounted(t, root, cache, counted);
const cancel = new AbortController();
const cancelled = other.refresh(cancel.signal);
await delay(1_500);
// Neither has started analyzing while the other process builds.
assert.equal(await counted.starts(), built);
cancel.abort();
await assert.rejects(cancelled, { code: "PROJECT_READ_CANCELLED" });

holder.kill("SIGKILL");
assert.notEqual(await refreshed, version);
});

test("a version another process holds outlasts this one's cleanup, until the hold expires", async (t) => {
const { root, write } = await project(FILES);
const cache = path.join(await sandbox(), "cache");
Expand Down
Loading