diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 1954274fe..94a44084c 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -4,12 +4,40 @@ import fs from "node:fs"; import net from "node:net"; import path from "node:path"; import process from "node:process"; +import { fileURLToPath } from "node:url"; import { parseArgs } from "./lib/args.mjs"; -import { BROKER_BUSY_RPC_CODE, CodexAppServerClient } from "./lib/app-server.mjs"; +import { BROKER_BUSY_RPC_CODE, BROKER_OWNERSHIP_RPC_CODE, BROKER_STREAM_COMPLETED_METHOD, CodexAppServerClient } from "./lib/app-server.mjs"; +import { + loadBrokerRegistration, + publishBrokerChild, + publishBrokerChildObservation, + releaseBrokerChild +} from "./lib/broker-ownership.mjs"; import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs"; +import { getLiveProcessPids, getProcessIdentity } from "./lib/process.mjs"; +const DEFAULT_CHILD_IDLE_MS = 5 * 60 * 1000; const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]); +const TEST_BROKER_TTL_ENV = "CODEX_COMPANION_TEST_BROKER_TTL_MS"; +const BROKER_CLEANUP_UNVERIFIED_RPC_CODE = -32002; +const BROKER_SHUTDOWN_RPC_CODE = -32003; +const BROKER_NOT_ACTIVATED_RPC_CODE = -32004; +const BROKER_ACTIVATION_ACK = "activated"; +const CHILD_OBSERVATION_RETRY_DELAYS_MS = [10, 25, 50, 100]; + +export function isBrokerRequestAllowedDuringShutdown(shuttingDown, message) { + return !shuttingDown || message?.method === "broker/shutdown"; +} + +function resolveChildIdleMs(env = process.env) { + const raw = env.CODEX_COMPANION_BROKER_CHILD_IDLE_MS; + if (raw == null || raw === "") { + return DEFAULT_CHILD_IDLE_MS; + } + const value = Number(raw); + return Number.isFinite(value) && value >= 0 ? value : DEFAULT_CHILD_IDLE_MS; +} function buildStreamThreadIds(method, params, result) { const threadIds = new Set(); @@ -33,10 +61,35 @@ function send(socket, message) { socket.write(`${JSON.stringify(message)}\n`); } +function flushSocket(socket) { + if (socket.destroyed) { + return Promise.resolve(false); + } + return new Promise((resolve) => { + socket.write("", () => resolve(true)); + }); +} + function isInterruptRequest(message) { return message?.method === "turn/interrupt"; } +function isRegistryContention(reason) { + return reason === "registry-busy" || reason === "registry-lock-contention"; +} + +async function publishChildObservationWithRetry(registration, options) { + let observation = publishBrokerChildObservation(registration, options); + for (const delayMs of CHILD_OBSERVATION_RETRY_DELAYS_MS) { + if (observation.observed === true || !isRegistryContention(observation.reason)) { + return observation; + } + await new Promise((resolve) => setTimeout(resolve, delayMs)); + observation = publishBrokerChildObservation(registration, options); + } + return observation; +} + function writePidFile(pidFile) { if (!pidFile) { return; @@ -45,6 +98,18 @@ function writePidFile(pidFile) { fs.writeFileSync(pidFile, `${process.pid}\n`, "utf8"); } +function resolveTestBrokerTtlMs() { + const raw = process.env[TEST_BROKER_TTL_ENV]; + if (raw == null || raw === "") { + return null; + } + const value = Number(raw); + if (!Number.isInteger(value) || value < 100 || value > 10 * 60 * 1000) { + throw new Error(`${TEST_BROKER_TTL_ENV} must be an integer from 100 to 600000 milliseconds.`); + } + return value; +} + async function main() { const [subcommand, ...argv] = process.argv.slice(2); if (subcommand !== "serve") { @@ -52,7 +117,8 @@ async function main() { } const { options } = parseArgs(argv, { - valueOptions: ["cwd", "pid-file", "endpoint"] + valueOptions: ["cwd", "pid-file", "endpoint"], + booleanOptions: ["require-activation-stdin"] }); if (!options.endpoint) { @@ -63,64 +129,429 @@ async function main() { const endpoint = String(options.endpoint); const listenTarget = parseBrokerEndpoint(endpoint); const pidFile = options["pid-file"] ? path.resolve(options["pid-file"]) : null; + const activationRequired = options["require-activation-stdin"] === true; + const testBrokerTtlMs = resolveTestBrokerTtlMs(); writePidFile(pidFile); - const appClient = await CodexAppServerClient.connect(cwd, { disableBroker: true }); + const childIdleMs = resolveChildIdleMs(); + let appClient = null; + let appClientRegistryChild = null; + let appClientStartPromise = null; + let appClientClosePromise = null; + let childIdleTimer = null; + let inFlightRequests = 0; + let shutdownPromise = null; + let shuttingDown = false; + let activated = !activationRequired; + let activationAbortStarted = false; let activeRequestSocket = null; let activeStreamSocket = null; let activeStreamThreadIds = null; + let activeStreamRunning = false; + // Identifies the turn that owns the current stream. The owning socket may + // disconnect while the turn keeps running, so socket identity cannot be used + // to decide whether a late result still belongs to the active stream. + let activeStreamTurn = 0; + let streamTurnCounter = 0; + let blockedCleanup = null; + let brokerRegistration = null; + let testBrokerTtlTimer = null; const sockets = new Set(); + function getBrokerRegistration() { + try { + const brokerIdentity = getProcessIdentity(process.pid); + if (brokerIdentity) { + const candidate = loadBrokerRegistration({ endpoint, brokerIdentity }); + if (candidate.registered === true) { + brokerRegistration = candidate; + return brokerRegistration; + } + } + } catch { + // Missing registry evidence keeps the child outside automated cleanup. + } + brokerRegistration = null; + return null; + } + + function cancelChildIdleClose() { + if (childIdleTimer) { + clearTimeout(childIdleTimer); + childIdleTimer = null; + } + } + + function hasActiveWork() { + return ( + sockets.size > 0 || + inFlightRequests > 0 || + activeRequestSocket !== null || + activeStreamRunning + ); + } + + function clearStreamState() { + activeStreamRunning = false; + activeStreamSocket = null; + activeStreamThreadIds = null; + activeStreamTurn = 0; + } + + function recordUnverifiedCleanup(client) { + const outcome = client.cleanupOutcome; + if (!outcome || outcome.verified !== false) { + return; + } + const survivors = outcome.survivors ?? []; + blockedCleanup = { + degraded: Boolean(outcome.degraded) || survivors.length === 0, + survivors, + survivorIdentities: outcome.survivorIdentities ?? [] + }; + process.stderr.write( + `Warning: shared Codex broker will not spawn a replacement after unverified cleanup; surviving PIDs: ${survivors.join(", ") || "none known"}.\n` + ); + } + + function recordVerifiedChildRelease(client, child) { + const registration = getBrokerRegistration(); + if (registration?.registered !== true || !child || client.cleanupOutcome?.verified !== true) { + return; + } + try { + const released = releaseBrokerChild(registration, { + child, + cleanupOutcome: client.cleanupOutcome + }); + if (released.released !== true) { + process.stderr.write(`Warning: unable to release shared Codex app-server ownership (${released.reason ?? "unknown"}).\n`); + } + } catch (error) { + process.stderr.write(`Warning: unable to release shared Codex app-server ownership: ${error.message}.\n`); + } + } + + async function closeAppClient() { + cancelChildIdleClose(); + clearStreamState(); + if (appClientStartPromise) { + await appClientStartPromise.catch(() => {}); + } + if (appClientClosePromise) { + await appClientClosePromise; + return; + } + const client = appClient; + const registryChild = appClientRegistryChild; + appClient = null; + appClientRegistryChild = null; + if (!client) { + return; + } + appClientClosePromise = client + .close() + .then(() => { + recordUnverifiedCleanup(client); + recordVerifiedChildRelease(client, registryChild); + }) + .catch((error) => { + blockedCleanup = { degraded: true, survivors: [] }; + process.stderr.write(`Warning: shared Codex broker cleanup failed: ${error.message}. No replacement child will be spawned.\n`); + }) + .finally(() => { + appClientClosePromise = null; + }); + await appClientClosePromise; + } + + async function prepareRequestErrorForReply(error) { + if (error?.requiresChildTeardown !== true) { + return error; + } + await closeAppClient(); + if (!blockedCleanup) { + return error; + } + const cleanupError = new Error("Shared Codex app-server cleanup is unverified after ownership publication failed."); + cleanupError.rpcCode = BROKER_CLEANUP_UNVERIFIED_RPC_CODE; + return cleanupError; + } + + function scheduleChildIdleClose() { + cancelChildIdleClose(); + if (!appClient || hasActiveWork()) { + return; + } + childIdleTimer = setTimeout(() => { + childIdleTimer = null; + if (hasActiveWork()) { + return; + } + void closeAppClient(); + }, childIdleMs); + childIdleTimer.unref?.(); + } + function clearSocketOwnership(socket) { if (activeRequestSocket === socket) { activeRequestSocket = null; } if (activeStreamSocket === socket) { + // Detach only the notification socket. The turn may still be running in + // the app-server; stream state is cleared on turn/completed, on a + // streaming request error, on child exit, or when the child is released. activeStreamSocket = null; - activeStreamThreadIds = null; } } function routeNotification(message) { const target = activeRequestSocket ?? activeStreamSocket; - if (!target) { - return; + if (target) { + send(target, message); } - send(target, message); - if (message.method === "turn/completed" && activeStreamSocket === target) { + if (message.method === "turn/completed" && activeStreamRunning) { const threadId = message.params?.threadId ?? null; if (!threadId || !activeStreamThreadIds || activeStreamThreadIds.has(threadId)) { + const streamSocket = activeStreamSocket; + activeStreamRunning = false; activeStreamSocket = null; activeStreamThreadIds = null; - if (activeRequestSocket === target) { + if (activeRequestSocket === streamSocket) { activeRequestSocket = null; } + scheduleChildIdleClose(); } } } - async function shutdown(server) { - for (const socket of sockets) { - socket.end(); + function ensureCleanupSafeToSpawn() { + if (blockedCleanup) { + if (blockedCleanup.survivors.length > 0) { + const liveSurvivors = getLiveProcessPids(blockedCleanup.survivors, { + identities: blockedCleanup.survivorIdentities + }); + if (liveSurvivors.length > 0) { + const error = new Error(`Shared Codex broker cleanup is unverified; surviving PIDs: ${liveSurvivors.join(", ")}.`); + error.rpcCode = BROKER_CLEANUP_UNVERIFIED_RPC_CODE; + throw error; + } + blockedCleanup = null; + } else if (blockedCleanup.degraded) { + const error = new Error("Shared Codex broker cleanup is unverified; refusing to spawn a replacement child."); + error.rpcCode = BROKER_CLEANUP_UNVERIFIED_RPC_CODE; + throw error; + } else { + blockedCleanup = null; + } } - await appClient.close().catch(() => {}); - await new Promise((resolve) => server.close(resolve)); - if (listenTarget.kind === "unix" && fs.existsSync(listenTarget.path)) { - fs.unlinkSync(listenTarget.path); + } + + async function getAppClient() { + if (shuttingDown) { + const error = new Error("Shared Codex broker is shutting down."); + error.rpcCode = BROKER_SHUTDOWN_RPC_CODE; + throw error; } - if (pidFile && fs.existsSync(pidFile)) { - fs.unlinkSync(pidFile); + cancelChildIdleClose(); + ensureCleanupSafeToSpawn(); + if (appClient) { + return appClient; } + if (appClientClosePromise) { + await appClientClosePromise; + if (shuttingDown) { + const error = new Error("Shared Codex broker is shutting down."); + error.rpcCode = BROKER_SHUTDOWN_RPC_CODE; + throw error; + } + ensureCleanupSafeToSpawn(); + } + if (!appClientStartPromise) { + let childRegistration = null; + let registration = null; + appClientStartPromise = CodexAppServerClient.connect(cwd, { + disableBroker: true, + gatedBrokerChild: true, + async beforeAppServerActivation(ownershipSnapshot) { + registration = getBrokerRegistration(); + if (registration?.registered !== true) { + const error = new Error("Shared Codex broker ownership registration is unavailable."); + error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + throw error; + } + childRegistration = publishBrokerChild(registration, { ownershipSnapshot }); + if (childRegistration.registered !== true) { + const error = new Error(`Unable to register shared Codex app-server ownership (${childRegistration.reason ?? "unknown"}).`); + error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + throw error; + } + }, + async afterAppServerOwnershipRefresh(ownershipSnapshot) { + if (registration?.registered !== true || childRegistration?.registered !== true) { + const error = new Error("Shared Codex app-server ownership registration was lost before helper observation."); + error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + throw error; + } + const observation = await publishChildObservationWithRetry(registration, { + child: childRegistration.child, + ownershipSnapshot + }); + if (observation.observed !== true) { + const error = new Error(`Unable to publish shared Codex app-server helper ownership (${observation.reason ?? "unknown"}).`); + error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + error.requiresChildTeardown = true; + throw error; + } + childRegistration = { + ...childRegistration, + child: observation.child + }; + } + }) + .then(async (client) => { + if (registration?.registered !== true || childRegistration?.registered !== true) { + await client.close().catch(() => {}); + const error = new Error("Shared Codex app-server activation lost its durable ownership registration."); + error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + throw error; + } + let registryChild = null; + appClient = client; + registryChild = childRegistration.child; + appClientRegistryChild = registryChild; + client.setNotificationHandler(routeNotification); + const childPid = client.proc?.pid ?? null; + client.setExitHandler(() => { + clearStreamState(); + if (appClient === client) { + appClient = null; + appClientRegistryChild = null; + } + if (!client.closed && childPid != null && process.platform !== "win32") { + // The child is a detached process-group leader; on an unexpected + // exit its surviving helpers reparent away from the broker, so + // wait for the client's shared direct/broker crash cleanup before + // allowing a replacement to spawn. + appClientClosePromise = client + .waitForUnexpectedExitCleanup() + ?.then(() => { + recordUnverifiedCleanup(client); + recordVerifiedChildRelease(client, registryChild); + }) + .catch((error) => { + blockedCleanup = { degraded: true, survivors: [] }; + process.stderr.write(`Warning: shared Codex broker could not reclaim exited app-server group: ${error.message}. No replacement child will be spawned.\n`); + }) + .finally(() => { + appClientClosePromise = null; + }) ?? null; + } + scheduleChildIdleClose(); + }); + return client; + }) + .catch((error) => { + if (error.cleanupOutcome) { + recordUnverifiedCleanup({ cleanupOutcome: error.cleanupOutcome }); + } + throw error; + }) + .finally(() => { + appClientStartPromise = null; + }); + } + return appClientStartPromise; } - appClient.setNotificationHandler(routeNotification); + async function shutdown(server) { + if (shutdownPromise) { + return shutdownPromise; + } + shuttingDown = true; + const serverClosePromise = new Promise((resolve) => { + server.close(resolve); + }); + shutdownPromise = (async () => { + if (testBrokerTtlTimer) { + clearTimeout(testBrokerTtlTimer); + testBrokerTtlTimer = null; + } + cancelChildIdleClose(); + for (const socket of sockets) { + socket.destroy(); + } + await closeAppClient(); + await serverClosePromise; + if (listenTarget.kind === "unix" && fs.existsSync(listenTarget.path)) { + fs.unlinkSync(listenTarget.path); + } + if (pidFile && fs.existsSync(pidFile)) { + fs.unlinkSync(pidFile); + } + })(); + return shutdownPromise; + } + + function setupActivationGate(server) { + let buffer = ""; + const abort = () => { + if (activated || activationAbortStarted) { + return; + } + activationAbortStarted = true; + void shutdown(server) + .catch(() => {}) + .finally(() => process.exit(1)); + }; + + process.stdin.setEncoding("utf8"); + process.stdin.on("data", (chunk) => { + if (activated || activationAbortStarted) { + return; + } + buffer += chunk; + const newlineIndex = buffer.indexOf("\n"); + if (newlineIndex === -1) { + return; + } + const command = buffer.slice(0, newlineIndex).trim(); + if (command !== "activate") { + abort(); + return; + } + activated = true; + try { + fs.writeSync(3, `${BROKER_ACTIVATION_ACK}\n`); + fs.closeSync(3); + } catch { + activated = false; + abort(); + } + }); + process.stdin.on("end", abort); + process.stdin.on("error", abort); + process.stdin.resume(); + if (process.stdin.readableEnded) { + abort(); + } + } const server = net.createServer((socket) => { + if (shuttingDown) { + socket.destroy(); + return; + } + cancelChildIdleClose(); sockets.add(socket); socket.setEncoding("utf8"); let buffer = ""; socket.on("data", async (chunk) => { + if (shuttingDown) { + socket.destroy(); + return; + } buffer += chunk; let newlineIndex = buffer.indexOf("\n"); while (newlineIndex !== -1) { @@ -143,6 +574,17 @@ async function main() { continue; } + if (activationRequired && !activated && message.method !== "broker/shutdown") { + if (message.id !== undefined) { + send(socket, { + id: message.id, + error: buildJsonRpcError(BROKER_NOT_ACTIVATED_RPC_CODE, "Shared Codex broker is not activated.") + }); + } + socket.end(); + continue; + } + if (message.id !== undefined && message.method === "initialize") { send(socket, { id: message.id, @@ -157,21 +599,48 @@ async function main() { continue; } + if (message.method === BROKER_STREAM_COMPLETED_METHOD && message.id === undefined) { + const threadId = message.params?.threadId ?? null; + if ( + activeStreamRunning && + activeStreamSocket === socket && + (!threadId || !activeStreamThreadIds || activeStreamThreadIds.has(threadId)) + ) { + clearStreamState(); + scheduleChildIdleClose(); + } + continue; + } + if (message.id !== undefined && message.method === "broker/shutdown") { + shuttingDown = true; send(socket, { id: message.id, result: {} }); + await flushSocket(socket); await shutdown(server); process.exit(0); } + if (!isBrokerRequestAllowedDuringShutdown(shuttingDown, message)) { + if (message.id !== undefined) { + send(socket, { + id: message.id, + error: buildJsonRpcError(BROKER_SHUTDOWN_RPC_CODE, "Shared Codex broker is shutting down.") + }); + } + socket.destroy(); + continue; + } + if (message.id === undefined) { continue; } const allowInterruptDuringActiveStream = - isInterruptRequest(message) && activeStreamSocket && activeStreamSocket !== socket && !activeRequestSocket; + isInterruptRequest(message) && activeStreamRunning && activeStreamSocket !== socket && !activeRequestSocket; if ( - ((activeRequestSocket && activeRequestSocket !== socket) || (activeStreamSocket && activeStreamSocket !== socket)) && + ((activeRequestSocket && activeRequestSocket !== socket) || + (activeStreamRunning && activeStreamSocket !== socket)) && !allowInterruptDuringActiveStream ) { send(socket, { @@ -182,42 +651,65 @@ async function main() { } if (allowInterruptDuringActiveStream) { + inFlightRequests += 1; try { - const result = await appClient.request(message.method, message.params ?? {}); + const client = await getAppClient(); + const result = await client.request(message.method, message.params ?? {}); send(socket, { id: message.id, result }); } catch (error) { + const replyError = await prepareRequestErrorForReply(error); send(socket, { id: message.id, - error: buildJsonRpcError(error.rpcCode ?? -32000, error.message) + error: buildJsonRpcError(replyError.rpcCode ?? -32000, replyError.message) }); + } finally { + inFlightRequests -= 1; + scheduleChildIdleClose(); } continue; } const isStreaming = STREAMING_METHODS.has(message.method); activeRequestSocket = socket; + let streamTurn = 0; + if (isStreaming) { + activeStreamRunning = true; + activeStreamSocket = socket; + activeStreamThreadIds = buildStreamThreadIds(message.method, message.params ?? {}, null); + streamTurn = ++streamTurnCounter; + activeStreamTurn = streamTurn; + } + inFlightRequests += 1; try { - const result = await appClient.request(message.method, message.params ?? {}); + const client = await getAppClient(); + const result = await client.request(message.method, message.params ?? {}); send(socket, { id: message.id, result }); - if (isStreaming) { - activeStreamSocket = socket; + // Gate on the turn, not the socket: a client that disconnects mid-turn + // clears activeStreamSocket while the turn keeps running, and without + // the result-derived thread ids the completion notification can never + // match, leaving the broker busy forever. + if (isStreaming && activeStreamRunning && activeStreamTurn === streamTurn) { activeStreamThreadIds = buildStreamThreadIds(message.method, message.params ?? {}, result); } if (activeRequestSocket === socket) { activeRequestSocket = null; } } catch (error) { + const replyError = await prepareRequestErrorForReply(error); send(socket, { id: message.id, - error: buildJsonRpcError(error.rpcCode ?? -32000, error.message) + error: buildJsonRpcError(replyError.rpcCode ?? -32000, replyError.message) }); if (activeRequestSocket === socket) { activeRequestSocket = null; } - if (activeStreamSocket === socket && !isStreaming) { - activeStreamSocket = null; + if (isStreaming) { + clearStreamState(); } + } finally { + inFlightRequests -= 1; + scheduleChildIdleClose(); } } }); @@ -225,11 +717,13 @@ async function main() { socket.on("close", () => { sockets.delete(socket); clearSocketOwnership(socket); + scheduleChildIdleClose(); }); socket.on("error", () => { sockets.delete(socket); clearSocketOwnership(socket); + scheduleChildIdleClose(); }); }); @@ -243,10 +737,28 @@ async function main() { process.exit(0); }); - server.listen(listenTarget.path); + server.listen(listenTarget.path, () => { + if (activationRequired) { + setupActivationGate(server); + } + if (testBrokerTtlMs != null) { + // Test suites intentionally exercise real detached broker behavior. A + // bounded test-only lifetime prevents an interrupted runner from leaving + // its fixture broker behind after normal teardown becomes impossible. + testBrokerTtlTimer = setTimeout(() => { + testBrokerTtlTimer = null; + void shutdown(server) + .catch(() => {}) + .finally(() => process.exit(0)); + }, testBrokerTtlMs); + testBrokerTtlTimer.unref?.(); + } + }); } -main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - process.exit(1); -}); +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/plugins/codex/scripts/app-server-child.mjs b/plugins/codex/scripts/app-server-child.mjs new file mode 100644 index 000000000..650c9c416 --- /dev/null +++ b/plugins/codex/scripts/app-server-child.mjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import process from "node:process"; +import { spawn } from "node:child_process"; + +const activation = fs.createReadStream(null, { fd: 3, autoClose: true }); +let activated = false; +let activationBuffer = ""; +let child = null; + +function exitWithoutChild(code = 1) { + if (child || process.exitCode != null) { + return; + } + process.exitCode = code; + process.stdin.resume(); +} + +function startAppServer() { + if (activated || child) { + return; + } + activated = true; + child = spawn("codex", ["app-server"], { + cwd: process.cwd(), + env: process.env, + detached: false, + stdio: ["pipe", "pipe", "pipe"], + shell: process.platform === "win32" ? (process.env.SHELL || true) : false, + windowsHide: true + }); + child.stdout.pipe(process.stdout); + child.stderr.pipe(process.stderr); + process.stdin.pipe(child.stdin); + child.on("error", (error) => { + process.stderr.write(`Unable to start codex app-server: ${error.message}\n`); + process.exit(1); + }); + // `close` follows both process exit and stdio closure, so the wrapper does + // not truncate the JSONL stream while forwarding the child's final bytes. + child.on("close", (code, signal) => { + process.exit(Number.isInteger(code) ? code : signal ? 1 : 0); + }); +} + +activation.setEncoding("utf8"); +activation.on("data", (chunk) => { + if (activated) { + return; + } + activationBuffer += chunk; + const newlineIndex = activationBuffer.indexOf("\n"); + if (newlineIndex === -1) { + return; + } + if (activationBuffer.slice(0, newlineIndex).trim() !== "activate") { + exitWithoutChild(1); + return; + } + startAppServer(); +}); +activation.on("end", () => { + if (!activated) { + exitWithoutChild(1); + } +}); +activation.on("error", () => { + if (!activated) { + exitWithoutChild(1); + } +}); + +process.stdin.on("end", () => { + if (!activated) { + exitWithoutChild(1); + return; + } + child?.stdin.end(); +}); +process.stdin.on("error", () => { + child?.stdin.destroy(); +}); +// Keep protocol bytes buffered until activation has created the real app +// server and its stdin pipe. Flowing stdin here can discard an initialize +// request that races the activation-control pipe. +process.stdin.pause(); diff --git a/plugins/codex/scripts/codex-companion.mjs b/plugins/codex/scripts/codex-companion.mjs index 83df468ad..8a331b460 100644 --- a/plugins/codex/scripts/codex-companion.mjs +++ b/plugins/codex/scripts/codex-companion.mjs @@ -4,7 +4,7 @@ import { spawn } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import process from "node:process"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs, splitRawArgumentString } from "./lib/args.mjs"; import { @@ -24,7 +24,7 @@ import { import { resolveClaudeSessionPath } from "./lib/claude-session-transfer.mjs"; import { readStdinIfPiped } from "./lib/fs.mjs"; import { collectReviewContext, ensureGitRepository, resolveReviewTarget } from "./lib/git.mjs"; -import { binaryAvailable, terminateProcessTree } from "./lib/process.mjs"; +import { binaryAvailable, getProcessIdentity, terminateProcessTree } from "./lib/process.mjs"; import { loadPromptTemplate, interpolateTemplate } from "./lib/prompts.mjs"; import { generateJobId, @@ -32,6 +32,7 @@ import { listJobs, setConfig, upsertJob, + writeCancelFlag, writeJobFile } from "./lib/state.mjs"; import { @@ -681,22 +682,61 @@ function spawnDetachedTaskWorker(cwd, jobId) { return child; } -function enqueueBackgroundTask(cwd, job, request) { +function recordTaskWorkerSpawnFailure(workspaceRoot, jobId, error) { + const storedJob = readStoredJob(workspaceRoot, jobId); + if (!storedJob || storedJob.status !== "queued" || storedJob.pid != null) { + return; + } + + const errorMessage = error instanceof Error ? error.message : String(error); + const completedAt = nowIso(); + const failedRecord = { + ...storedJob, + status: "failed", + phase: "failed", + pid: null, + errorMessage, + completedAt + }; + writeJobFile(workspaceRoot, jobId, failedRecord); + upsertJob(workspaceRoot, { + id: jobId, + status: "failed", + phase: "failed", + pid: null, + errorMessage, + completedAt + }); + appendLogLine(storedJob.logFile, `Worker spawn failed: ${errorMessage}`); +} + +export function enqueueBackgroundTask(cwd, job, request, dependencies = {}) { const { logFile } = createTrackedProgress(job); appendLogLine(logFile, "Queued for background execution."); - const child = spawnDetachedTaskWorker(cwd, job.id); const queuedRecord = { ...job, status: "queued", phase: "queued", - pid: child.pid ?? null, + pid: null, logFile, request }; writeJobFile(job.workspaceRoot, job.id, queuedRecord); upsertJob(job.workspaceRoot, queuedRecord); + const spawnWorker = dependencies.spawnDetachedTaskWorkerImpl ?? spawnDetachedTaskWorker; + let worker; + try { + worker = spawnWorker(cwd, job.id); + } catch (error) { + recordTaskWorkerSpawnFailure(job.workspaceRoot, job.id, error); + throw error; + } + worker?.once?.("error", (error) => { + recordTaskWorkerSpawnFailure(job.workspaceRoot, job.id, error); + }); + return { payload: { jobId: job.id, @@ -835,7 +875,7 @@ async function handleTransfer(argv) { outputCommandResult(payload, rendered, options.json); } -async function handleTaskWorker(argv) { +export async function handleTaskWorker(argv, dependencies = {}) { const { options } = parseCommandInput(argv, { valueOptions: ["cwd", "job-id"] }); @@ -851,6 +891,20 @@ async function handleTaskWorker(argv) { throw new Error(`No stored job found for ${options["job-id"]}.`); } + const { + processIdentity: _storedProcessIdentity, + ownershipSnapshot: _storedOwnershipSnapshot, + ownershipCaptureFailed: _storedOwnershipCaptureFailed, + ...storedTask + } = storedJob; + let workerOwnership; + try { + const processIdentity = (dependencies.getProcessIdentityImpl ?? getProcessIdentity)(process.pid); + workerOwnership = processIdentity ? { processIdentity } : { ownershipCaptureFailed: true }; + } catch { + workerOwnership = { ownershipCaptureFailed: true }; + } + const request = storedJob.request; if (!request || typeof request !== "object") { throw new Error(`Stored job ${options["job-id"]} is missing its task request payload.`); @@ -858,18 +912,19 @@ async function handleTaskWorker(argv) { const { logFile, progress } = createTrackedProgress( { - ...storedJob, + ...storedTask, workspaceRoot }, { logFile: storedJob.logFile ?? null } ); - await runTrackedJob( + await (dependencies.runTrackedJobImpl ?? runTrackedJob)( { - ...storedJob, + ...storedTask, workspaceRoot, - logFile + logFile, + ...workerOwnership }, () => executeTaskRun({ @@ -960,35 +1015,12 @@ function handleTaskResumeCandidate(argv) { outputCommandResult(payload, rendered, options.json); } -async function handleCancel(argv) { - const { options, positionals } = parseCommandInput(argv, { - valueOptions: ["cwd"], - booleanOptions: ["json"] - }); - - const cwd = resolveCommandCwd(options); - const reference = positionals[0] ?? ""; - const { workspaceRoot, job } = resolveCancelableJob(cwd, reference, { env: process.env }); - const existing = readStoredJob(workspaceRoot, job.id) ?? {}; - const threadId = existing.threadId ?? job.threadId ?? null; - const turnId = existing.turnId ?? job.turnId ?? null; - - const interrupt = await interruptAppServerTurn(cwd, { threadId, turnId }); - if (interrupt.attempted) { - appendLogLine( - job.logFile, - interrupt.interrupted - ? `Requested Codex turn interrupt for ${turnId} on ${threadId}.` - : `Codex turn interrupt failed${interrupt.detail ? `: ${interrupt.detail}` : "."}` - ); - } - - terminateProcessTree(job.pid ?? Number.NaN); - appendLogLine(job.logFile, "Cancelled by user."); +function finishCancelledJob(workspaceRoot, record, interrupt, options) { + appendLogLine(record.logFile, "Cancelled by user."); const completedAt = nowIso(); const nextJob = { - ...job, + ...record, status: "cancelled", phase: "cancelled", pid: null, @@ -996,13 +1028,12 @@ async function handleCancel(argv) { errorMessage: "Cancelled by user." }; - writeJobFile(workspaceRoot, job.id, { - ...existing, + writeJobFile(workspaceRoot, record.id, { ...nextJob, cancelledAt: completedAt }); upsertJob(workspaceRoot, { - id: job.id, + id: record.id, status: "cancelled", phase: "cancelled", pid: null, @@ -1011,9 +1042,9 @@ async function handleCancel(argv) { }); const payload = { - jobId: job.id, + jobId: record.id, status: "cancelled", - title: job.title, + title: record.title, turnInterruptAttempted: interrupt.attempted, turnInterrupted: interrupt.interrupted }; @@ -1021,6 +1052,83 @@ async function handleCancel(argv) { outputCommandResult(payload, renderCancelReport(nextJob), options.json); } +export async function handleCancel(argv, dependencies = {}) { + const { options, positionals } = parseCommandInput(argv, { + valueOptions: ["cwd"], + booleanOptions: ["json"] + }); + + const cwd = resolveCommandCwd(options); + const reference = positionals[0] ?? ""; + const { workspaceRoot, job } = resolveCancelableJob(cwd, reference, { env: process.env }); + let existing = readStoredJob(workspaceRoot, job.id) ?? {}; + let record = { ...job, ...existing }; + + if (!Number.isFinite(record.pid)) { + writeCancelFlag(workspaceRoot, job.id); + existing = readStoredJob(workspaceRoot, job.id) ?? existing; + record = { ...job, ...existing }; + if (!Number.isFinite(record.pid)) { + finishCancelledJob( + workspaceRoot, + record, + { attempted: false, interrupted: false }, + options + ); + return; + } + } + + const threadId = record.threadId ?? null; + const turnId = record.turnId ?? null; + + const interrupt = await (dependencies.interruptAppServerTurnImpl ?? interruptAppServerTurn)(cwd, { threadId, turnId }); + if (interrupt.attempted) { + appendLogLine( + record.logFile, + interrupt.interrupted + ? `Requested Codex turn interrupt for ${turnId} on ${threadId}.` + : `Codex turn interrupt failed${interrupt.detail ? `: ${interrupt.detail}` : "."}` + ); + } + + const expectedRootIdentity = existing.processIdentity ?? null; + const ownershipCaptureFailed = existing.ownershipCaptureFailed === true; + const cleanupOutcome = await (dependencies.terminateProcessTreeImpl ?? terminateProcessTree)(record.pid, { + expectedRootIdentity, + ownershipSnapshot: null, + requireVerifiedOwnership: ownershipCaptureFailed, + priorCleanupDegraded: existing.cleanupOutcome?.degraded === true + }); + if (cleanupOutcome?.verified !== true) { + const failureMessage = + ownershipCaptureFailed && !expectedRootIdentity + ? `Job ${job.id} could not be verified as owned and was left alone.` + : `Unable to verify cleanup for ${job.id}; ownership records were preserved for retry.`; + appendLogLine(record.logFile, failureMessage); + const recoveryRecord = { + ...record, + status: record.status, + phase: "cleanup-pending", + pid: record.pid, + cleanupOutcome, + cleanupFailure: failureMessage + }; + writeJobFile(workspaceRoot, job.id, recoveryRecord); + upsertJob(workspaceRoot, { + id: job.id, + status: record.status, + phase: "cleanup-pending", + pid: record.pid, + cleanupOutcome, + cleanupFailure: failureMessage + }); + throw new Error(failureMessage); + } + + finishCancelledJob(workspaceRoot, record, interrupt, options); +} + async function main() { const [subcommand, ...argv] = process.argv.slice(2); if (!subcommand || subcommand === "help" || subcommand === "--help") { @@ -1066,8 +1174,10 @@ async function main() { } } -main().catch((error) => { - const message = error instanceof Error ? error.message : String(error); - process.stderr.write(`${message}\n`); - process.exitCode = 1; -}); +if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) { + main().catch((error) => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${message}\n`); + process.exitCode = 1; + }); +} diff --git a/plugins/codex/scripts/lib/app-server-protocol.d.ts b/plugins/codex/scripts/lib/app-server-protocol.d.ts index f61a4588e..0e61c92e7 100644 --- a/plugins/codex/scripts/lib/app-server-protocol.d.ts +++ b/plugins/codex/scripts/lib/app-server-protocol.d.ts @@ -54,6 +54,9 @@ export interface CodexAppServerClientOptions { brokerEndpoint?: string; disableBroker?: boolean; reuseExistingBroker?: boolean; + gatedBrokerChild?: boolean; + beforeAppServerActivation?: (ownershipSnapshot: unknown) => void | Promise; + afterAppServerOwnershipRefresh?: (ownershipSnapshot: unknown) => void | Promise; } export interface AppServerMethodMap { @@ -66,6 +69,7 @@ export interface AppServerMethodMap { "review/start": { params: ReviewStartParams; result: ReviewStartResponse }; "turn/start": { params: TurnStartParams; result: TurnStartResponse }; "turn/interrupt": { params: TurnInterruptParams; result: TurnInterruptResponse }; + "broker/stream-completed": { params: { threadId: string; turnId: string | null }; result: Record }; } export type AppServerMethod = keyof AppServerMethodMap; diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index 72b30a764..330e3ad02 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -12,15 +12,20 @@ import net from "node:net"; import process from "node:process"; import { spawn } from "node:child_process"; import readline from "node:readline"; +import { fileURLToPath } from "node:url"; import { parseBrokerEndpoint } from "./broker-endpoint.mjs"; -import { ensureBrokerSession, loadBrokerSession } from "./broker-lifecycle.mjs"; -import { terminateProcessTree } from "./process.mjs"; +import { ensureBrokerSession, loadReusableBrokerSession } from "./broker-lifecycle.mjs"; +import { captureProcessOwnership, normalizeProcessCleanupOutcome, terminateProcessGroup, terminateProcessTree } from "./process.mjs"; const PLUGIN_MANIFEST_URL = new URL("../../.claude-plugin/plugin.json", import.meta.url); const PLUGIN_MANIFEST = JSON.parse(fs.readFileSync(PLUGIN_MANIFEST_URL, "utf8")); +const DEFAULT_CLOSE_WAIT_MS = 2000; +const GATED_APP_SERVER_CHILD = fileURLToPath(new URL("../app-server-child.mjs", import.meta.url)); export const BROKER_ENDPOINT_ENV = "CODEX_COMPANION_APP_SERVER_ENDPOINT"; export const BROKER_BUSY_RPC_CODE = -32001; +export const BROKER_OWNERSHIP_RPC_CODE = -32005; +export const BROKER_STREAM_COMPLETED_METHOD = "broker/stream-completed"; /** @type {ClientInfo} */ const DEFAULT_CLIENT_INFO = { @@ -63,6 +68,7 @@ class AppServerClientBase { this.stderr = ""; this.closed = false; this.exitError = null; + this.exitHandler = null; /** @type {AppServerNotificationHandler | null} */ this.notificationHandler = null; this.lineBuffer = ""; @@ -77,6 +83,10 @@ class AppServerClientBase { this.notificationHandler = handler; } + setExitHandler(handler) { + this.exitHandler = handler; + } + /** * @template {AppServerMethod} M * @param {M} method @@ -149,10 +159,14 @@ class AppServerClientBase { } if (message.method && this.notificationHandler) { - this.notificationHandler(/** @type {AppServerNotification} */ (message)); + this.handleNotification(/** @type {AppServerNotification} */ (message)); } } + handleNotification(message) { + this.notificationHandler?.(message); + } + handleServerRequest(message) { this.sendMessage({ id: message.id, @@ -173,6 +187,7 @@ class AppServerClientBase { } this.pending.clear(); this.resolveExit(undefined); + this.exitHandler?.(this.exitError); } sendMessage(_message) { @@ -184,17 +199,19 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { constructor(cwd, options = {}) { super(cwd, options); this.transport = "direct"; + this.notificationRefreshPromise = Promise.resolve(); } async initialize() { - this.proc = spawn("codex", ["app-server"], { + const gated = this.options.gatedBrokerChild === true && process.platform !== "win32"; + this.proc = spawn(gated ? process.execPath : "codex", gated ? [GATED_APP_SERVER_CHILD] : ["app-server"], { cwd: this.cwd, env: this.options.env ?? process.env, - stdio: ["pipe", "pipe", "pipe"], + detached: process.platform !== "win32", + stdio: gated ? ["pipe", "pipe", "pipe", "pipe"] : ["pipe", "pipe", "pipe"], shell: process.platform === "win32" ? (process.env.SHELL || true) : false, windowsHide: true }); - this.proc.stdout.setEncoding("utf8"); this.proc.stderr.setEncoding("utf8"); @@ -214,6 +231,9 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { : createProtocolError( `codex app-server exited unexpectedly (${signal ? `signal ${signal}` : `exit ${code}`}).${stderr ? `\n${stderr}` : ""}` ); + if (!this.closed) { + this.startUnexpectedExitCleanup(this.proc.pid); + } this.handleExit(detail); }); @@ -222,6 +242,32 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { this.handleLine(line); }); + try { + const captureOwnership = this.options.captureProcessOwnershipImpl ?? captureProcessOwnership; + this.ownershipSnapshot = + process.platform === "win32" + ? null + : captureOwnership(this.proc.pid, { + cwd: this.cwd, + env: this.options.env ?? process.env + }); + this.procIdentity = this.ownershipSnapshot?.rootIdentity ?? null; + } catch (error) { + this.identityCaptureFailed = true; + throw error; + } + if (process.platform !== "win32" && !this.procIdentity) { + this.identityCaptureFailed = true; + throw new Error("Unable to capture codex app-server process identity."); + } + if (gated) { + await this.options.beforeAppServerActivation?.(this.ownershipSnapshot); + const activationControl = this.proc.stdio?.[3]; + if (!activationControl) { + throw new Error("Codex app-server activation control is unavailable."); + } + activationControl.end("activate\n"); + } await this.request("initialize", { clientInfo: this.options.clientInfo ?? DEFAULT_CLIENT_INFO, capabilities: this.options.capabilities ?? DEFAULT_CAPABILITIES @@ -229,40 +275,213 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { this.notify("initialized", {}); } + mergeOwnershipSnapshot(observation) { + if ( + !observation?.rootIdentity || + observation.rootIdentity !== this.procIdentity || + observation.rootPid !== this.ownershipSnapshot?.rootPid + ) { + throw new Error("Codex app-server ownership identity changed while refreshing its helper tree."); + } + const members = new Map(); + for (const snapshot of [this.ownershipSnapshot, observation]) { + for (const member of snapshot?.members ?? []) { + members.set(member.identity, member); + } + } + this.ownershipSnapshot = { + ...observation, + members: [...members.values()] + }; + return this.ownershipSnapshot; + } + + async refreshOwnership() { + if (process.platform === "win32" || !this.procIdentity || !this.proc?.pid) { + return this.ownershipSnapshot ?? null; + } + const captureOwnership = this.options.captureProcessOwnershipImpl ?? captureProcessOwnership; + const observation = captureOwnership(this.proc.pid, { + cwd: this.cwd, + env: this.options.env ?? process.env + }); + const ownershipSnapshot = this.mergeOwnershipSnapshot(observation); + await this.options.afterAppServerOwnershipRefresh?.(ownershipSnapshot); + return ownershipSnapshot; + } + + handleNotification(message) { + this.notificationRefreshPromise = this.notificationRefreshPromise.then(async () => { + if (this.exitResolved) { + return; + } + try { + // Streaming requests return before their turn is complete. Publish a + // fresh helper observation before forwarding each later notification + // so independently grouped helpers are durable before a crash can + // strand them outside both local and registered cleanup. + await this.refreshOwnership(); + } catch (error) { + this.startUnexpectedExitCleanup(this.proc?.pid); + this.handleExit(error); + return; + } + if (!this.closed) { + this.notificationHandler?.(message); + } + }); + return this.notificationRefreshPromise; + } + + async request(method, params) { + let result; + let requestError; + try { + result = await super.request(method, params); + } catch (error) { + requestError = error; + } + // Helpers may be created only after the gated wrapper is activated. Take + // and durably publish a fresh identity snapshot at every request boundary + // before the broker exposes the response or error to its caller. + await this.refreshOwnership(); + if (requestError) { + throw requestError; + } + return result; + } + + startUnexpectedExitCleanup(pid) { + if ( + this.unexpectedExitCleanupPromise || + process.platform === "win32" || + !Number.isFinite(pid) || + !this.ownershipSnapshot?.rootIdentity + ) { + return this.unexpectedExitCleanupPromise ?? null; + } + + const observationBarrier = this.notificationRefreshPromise ?? Promise.resolve(); + this.unexpectedExitCleanupPromise = observationBarrier + .catch(() => {}) + .then(() => terminateProcessGroup(pid, { + ownershipSnapshot: this.ownershipSnapshot, + cwd: this.cwd, + env: this.options.env ?? process.env, + warnImpl: () => {} + })) + .then((outcome) => { + this.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); + if (!outcome.verified) { + process.stderr.write( + `Warning: unable to verify crashed codex app-server group cleanup; surviving PIDs: ${outcome.survivors?.join(", ") || "none known"}.\n` + ); + } + return this.cleanupOutcome; + }) + .catch((error) => { + this.cleanupOutcome = normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified: false, + degraded: true, + method: "process-group", + survivors: [], + survivorIdentities: [] + }); + process.stderr.write(`Warning: crashed codex app-server group cleanup failed: ${error.message}.\n`); + return this.cleanupOutcome; + }); + return this.unexpectedExitCleanupPromise; + } + + waitForUnexpectedExitCleanup() { + return this.unexpectedExitCleanupPromise ?? null; + } + async close() { if (this.closed) { - await this.exitPromise; + await this.waitForExit(); return; } this.closed = true; + await this.notificationRefreshPromise?.catch(() => {}); + if (this.readline) { this.readline.close(); } - if (this.proc && !this.proc.killed) { - this.proc.stdin.end(); - setTimeout(() => { - if (this.proc && !this.proc.killed && this.proc.exitCode === null) { - // On Windows with shell: true, the direct child is cmd.exe. - // Use terminateProcessTree to kill the entire tree including - // the grandchild node process. - if (process.platform === "win32") { - try { - terminateProcessTree(this.proc.pid); - } catch { - // Best-effort cleanup inside an unref'd timer — swallow errors - // to avoid crashing the host process during shutdown. - } - } else { - this.proc.kill("SIGTERM"); + if (this.proc) { + this.proc.stdio?.[3]?.end?.(); + try { + this.proc.stdin.end(); + } catch { + // The child may have closed its input before cleanup began. + } + if (process.platform === "win32") { + setTimeout(() => { + if (this.proc && !this.proc.killed && this.proc.exitCode === null) { + void terminateProcessTree(this.proc.pid, { + expectedRootIdentity: this.procIdentity, + ownershipSnapshot: this.ownershipSnapshot, + requireVerifiedOwnership: this.identityCaptureFailed, + ownerHoldsLiveHandle: true + }) + .then((outcome) => { + this.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); + }) + .catch(() => { + // Best-effort cleanup inside an unref'd timer — swallow errors + // to avoid crashing the host process during shutdown. + }); + } + }, 50).unref?.(); + } else { + // The app-server is its own process-group leader on Unix. Terminate + // the group so MCP helpers cannot outlive the app-server parent. + if (this.unexpectedExitCleanupPromise) { + await this.unexpectedExitCleanupPromise; + } else { + const outcome = await terminateProcessTree(this.proc.pid, { + expectedRootIdentity: this.procIdentity, + ownershipSnapshot: this.ownershipSnapshot, + requireVerifiedOwnership: this.identityCaptureFailed, + ownerHoldsLiveHandle: true, + directKillImpl: (signal) => this.proc.kill(signal), + warnImpl: () => {} + }); + this.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); + if (!outcome.verified) { + process.stderr.write( + `Warning: unable to verify codex app-server cleanup; surviving PIDs: ${outcome.survivors?.join(", ") || "none known"}.\n` + ); } } - }, 50).unref?.(); + } } - await this.exitPromise; + const exited = await this.waitForExit(); + if (!exited) { + this.cleanupOutcome = normalizeProcessCleanupOutcome({ + ...(this.cleanupOutcome ?? {}), + attempted: true, + verified: false, + degraded: true + }); + } + } + + async waitForExit() { + const timeoutMs = Number.isFinite(this.options.closeWaitMs) ? this.options.closeWaitMs : DEFAULT_CLOSE_WAIT_MS; + let timeout; + const timedOut = new Promise((resolve) => { + timeout = setTimeout(() => resolve(false), timeoutMs); + }); + const exited = await Promise.race([this.exitPromise.then(() => true), timedOut]); + clearTimeout(timeout); + return exited; } sendMessage(message) { @@ -338,7 +557,7 @@ export class CodexAppServerClient { if (!options.disableBroker) { brokerEndpoint = options.brokerEndpoint ?? options.env?.[BROKER_ENDPOINT_ENV] ?? process.env[BROKER_ENDPOINT_ENV] ?? null; if (!brokerEndpoint && options.reuseExistingBroker) { - brokerEndpoint = loadBrokerSession(cwd)?.endpoint ?? null; + brokerEndpoint = loadReusableBrokerSession(cwd, options.env ?? process.env)?.endpoint ?? null; } if (!brokerEndpoint && !options.reuseExistingBroker) { const brokerSession = await ensureBrokerSession(cwd, { env: options.env }); @@ -348,7 +567,23 @@ export class CodexAppServerClient { const client = brokerEndpoint ? new BrokerCodexAppServerClient(cwd, { ...options, brokerEndpoint }) : new SpawnedCodexAppServerClient(cwd, options); - await client.initialize(); - return client; + try { + await client.initialize(); + return client; + } catch (error) { + try { + await client.close(); + } catch { + client.cleanupOutcome = { + verified: false, + survivors: [], + degraded: true + }; + } + if (client.cleanupOutcome?.verified === false) { + error.cleanupOutcome = client.cleanupOutcome; + } + throw error; + } } } diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index ef763819c..8e5b168d5 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -4,13 +4,126 @@ import os from "node:os"; import path from "node:path"; import process from "node:process"; import { spawn } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; import { fileURLToPath } from "node:url"; +import { + acquireBrokerRegistryLock, + assessBrokerOwners, + hasLiveBrokerOwnerIdentity, + loadBrokerChildren, + loadBrokerRegistration, + publishRegisteredBroker, + registerBrokerOwner, + releaseBrokerChild, + releaseBrokerOwner, + releaseBrokerRegistryLock, + resolveBrokerOwnershipRoot +} from "./broker-ownership.mjs"; import { createBrokerEndpoint, parseBrokerEndpoint } from "./broker-endpoint.mjs"; +import { captureProcessOwnership, getProcessIdentity, terminateProcessTree } from "./process.mjs"; import { resolveStateDir } from "./state.mjs"; export const PID_FILE_ENV = "CODEX_COMPANION_APP_SERVER_PID_FILE"; export const LOG_FILE_ENV = "CODEX_COMPANION_APP_SERVER_LOG_FILE"; const BROKER_STATE_FILE = "broker.json"; +const BROKER_ACTIVATION_ACK = "activated"; +const BROKER_LAUNCH_LOCK_HOST = "127.0.0.1"; +const BROKER_LAUNCH_LOCK_MIN_PORT = 49152; +const BROKER_LAUNCH_LOCK_PORT_COUNT = 16384; + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function brokerLaunchLockIdentity(cwd) { + return createHash("sha256").update(resolveBrokerStateFile(cwd)).digest("hex"); +} + +export function brokerLaunchLockPort(cwd) { + const identity = brokerLaunchLockIdentity(cwd); + const digest = Buffer.from(identity, "hex"); + return BROKER_LAUNCH_LOCK_MIN_PORT + (digest.readUInt16BE(0) % BROKER_LAUNCH_LOCK_PORT_COUNT); +} + +function tryListenForBrokerLaunch(port, greeting) { + return new Promise((resolve, reject) => { + const server = net.createServer((socket) => socket.end(`${greeting}\n`)); + const onError = (error) => { + if (error?.code === "EADDRINUSE") { + resolve(null); + return; + } + reject(error); + }; + server.once("error", onError); + server.listen({ host: BROKER_LAUNCH_LOCK_HOST, port, exclusive: true }, () => { + server.removeListener("error", onError); + resolve(server); + }); + }); +} + +function probeBrokerLaunchPort(port, expectedGreeting, timeoutMs = 150) { + return new Promise((resolve) => { + const socket = net.createConnection({ host: BROKER_LAUNCH_LOCK_HOST, port }); + let connected = false; + let buffer = ""; + let settled = false; + const finish = (state) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + socket.destroy(); + resolve(state); + }; + const timer = setTimeout(() => finish(connected ? "foreign" : "transient"), timeoutMs); + socket.setEncoding("utf8"); + socket.on("connect", () => { + connected = true; + }); + socket.on("data", (chunk) => { + buffer += chunk; + if (buffer.includes("\n")) { + finish(buffer.trim() === expectedGreeting ? "matching" : "foreign"); + } + }); + socket.on("end", () => finish(buffer.trim() === expectedGreeting ? "matching" : "foreign")); + socket.on("error", () => finish("transient")); + }); +} + +export async function acquireBrokerLaunchLock(cwd, options = {}) { + const identity = brokerLaunchLockIdentity(cwd); + const greeting = `codex-broker-launch-v1:${identity}`; + const port = brokerLaunchLockPort(cwd); + const timeoutMs = options.timeoutMs ?? 10_000; + const deadline = Date.now() + timeoutMs; + + while (Date.now() < deadline) { + const server = await tryListenForBrokerLaunch(port, greeting); + if (server) { + return { + port, + async release() { + await new Promise((resolve) => server.close(resolve)); + } + }; + } + const observed = await probeBrokerLaunchPort(port, greeting); + if (observed === "foreign") { + const error = new Error("The deterministic Codex broker launch lock is occupied by another local service."); + error.code = "BROKER_LAUNCH_LOCK_UNAVAILABLE"; + throw error; + } + await delay(25); + } + + const error = new Error("Timed out waiting for another Codex broker launch to finish."); + error.code = "BROKER_LAUNCH_LOCK_TIMEOUT"; + throw error; +} export function createBrokerSessionDir(prefix = "cxc-") { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); @@ -58,15 +171,119 @@ export async function sendBrokerShutdown(endpoint) { export function spawnBrokerProcess({ scriptPath, cwd, endpoint, pidFile, logFile, env = process.env }) { const logFd = fs.openSync(logFile, "a"); - const child = spawn(process.execPath, [scriptPath, "serve", "--endpoint", endpoint, "--cwd", cwd, "--pid-file", pidFile], { - cwd, - env, - detached: true, - stdio: ["ignore", logFd, logFd] + try { + const child = spawn( + process.execPath, + [scriptPath, "serve", "--endpoint", endpoint, "--cwd", cwd, "--pid-file", pidFile, "--require-activation-stdin"], + { + cwd, + env, + detached: true, + stdio: ["pipe", logFd, logFd, "pipe"] + } + ); + child.on("error", () => {}); + return child; + } finally { + fs.closeSync(logFd); + } +} + +export function activateBrokerProcess(child, timeoutMs = 2000) { + return new Promise((resolve, reject) => { + const status = child?.stdio?.[3]; + if (!child?.stdin || !status) { + const error = new Error("Broker activation channels are unavailable."); + error.code = "BROKER_ACTIVATION_FAILED"; + reject(error); + return; + } + + let buffer = ""; + let settled = false; + const ignoreStdinError = () => {}; + const cleanup = () => { + clearTimeout(timer); + status.removeListener("data", onData); + status.removeListener("error", onStatusError); + child.removeListener("error", onChildError); + child.removeListener("exit", onChildExit); + child.stdin.removeListener("error", onStdinError); + }; + const finish = (error = null) => { + if (settled) { + return; + } + settled = true; + cleanup(); + status.destroy(); + child.stdin.on("error", ignoreStdinError); + if (error) { + reject(error); + return; + } + child.unref(); + resolve(); + }; + const activationError = (message) => { + const error = new Error(message); + error.code = "BROKER_ACTIVATION_FAILED"; + return error; + }; + const onData = (chunk) => { + buffer += chunk; + if (buffer.includes("\n")) { + finish(buffer.trim() === BROKER_ACTIVATION_ACK + ? null + : activationError(`Unexpected broker activation response: ${buffer.trim() || "empty"}.`)); + } + }; + const onStatusError = (error) => finish(activationError(`Broker activation response failed: ${error.message}`)); + const onChildError = (error) => finish(activationError(`Broker activation process failed: ${error.message}`)); + const onChildExit = () => finish(activationError("Broker exited before activation completed.")); + const onStdinError = (error) => finish(activationError(`Broker activation request failed: ${error.message}`)); + const timer = setTimeout( + () => finish(activationError("Timed out waiting for broker activation acknowledgement.")), + timeoutMs + ); + + status.setEncoding("utf8"); + status.on("data", onData); + status.once("error", onStatusError); + child.once("error", onChildError); + child.once("exit", onChildExit); + child.stdin.once("error", onStdinError); + child.stdin.end("activate\n"); + }); +} + +function closeBrokerActivationChannels(child) { + if (child?.stdin && !child.stdin.destroyed) { + child.stdin.once("error", () => {}); + child.stdin.end(); + } + child?.stdio?.[3]?.destroy(); +} + +async function waitForBrokerProcessExit(child, timeoutMs = 1000) { + if (!child || child.exitCode !== null || child.signalCode !== null) { + return true; + } + return new Promise((resolve) => { + let settled = false; + const finish = (exited) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + child.removeListener("exit", onExit); + resolve(exited); + }; + const onExit = () => finish(true); + const timer = setTimeout(() => finish(false), timeoutMs); + child.once("exit", onExit); }); - child.unref(); - fs.closeSync(logFd); - return child; } function resolveBrokerStateFile(cwd) { @@ -89,7 +306,20 @@ export function loadBrokerSession(cwd) { export function saveBrokerSession(cwd, session) { const stateDir = resolveStateDir(cwd); fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync(resolveBrokerStateFile(cwd), `${JSON.stringify(session, null, 2)}\n`, "utf8"); + const stateFile = resolveBrokerStateFile(cwd); + const temporaryStateFile = `${stateFile}.${process.pid}.${randomUUID()}.tmp`; + try { + fs.writeFileSync(temporaryStateFile, `${JSON.stringify(session, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600 + }); + fs.renameSync(temporaryStateFile, stateFile); + } finally { + if (fs.existsSync(temporaryStateFile)) { + fs.unlinkSync(temporaryStateFile); + } + } } export function clearBrokerSession(cwd) { @@ -99,6 +329,49 @@ export function clearBrokerSession(cwd) { } } +export function loadReusableBrokerSession(cwd, env = process.env) { + if (!resolveBrokerOwnershipRoot(env) || !hasLiveBrokerOwnerIdentity(env)) { + return null; + } + const session = loadBrokerSession(cwd); + if ( + session?.activationPending === true || + session?.activationFailed === true || + session?.registry?.registered !== true || + typeof session.pidIdentity !== "string" + ) { + return null; + } + try { + if (getProcessIdentity(session.pid, { cwd, env }) !== session.pidIdentity) { + return null; + } + } catch { + return null; + } + const registration = loadBrokerRegistration({ + endpoint: session.endpoint, + brokerIdentity: session.pidIdentity, + env + }); + if ( + registration.registered !== true || + registration.brokerKey !== session.registry.brokerKey || + registration.registryDir !== session.registry.registryDir + ) { + return null; + } + try { + const target = parseBrokerEndpoint(session.endpoint); + if (target.kind !== "unix" || !fs.lstatSync(target.path).isSocket()) { + return null; + } + } catch { + return null; + } + return { ...session, registry: registration }; +} + async function isBrokerEndpointReady(endpoint) { if (!endpoint) { return false; @@ -110,22 +383,242 @@ async function isBrokerEndpointReady(endpoint) { } } -export async function ensureBrokerSession(cwd, options = {}) { - const existing = loadBrokerSession(cwd); - if (existing && (await isBrokerEndpointReady(existing.endpoint))) { - return existing; +async function rollbackNewBrokerSession(cwd, child, session, options) { + closeBrokerActivationChannels(child); + const exited = await waitForBrokerProcessExit(child, options.rollbackExitTimeoutMs ?? 1000); + let cleanup; + try { + cleanup = await teardownBrokerSession({ + endpoint: session.endpoint, + pidFile: session.pidFile, + logFile: session.logFile, + sessionDir: session.sessionDir, + pid: session.pid, + pidIdentity: session.pidIdentity, + ownershipSnapshot: session.ownershipSnapshot, + requireVerifiedOwnership: session.ownershipCaptureFailed === true, + killProcess: exited ? null : (options.killProcess ?? terminateProcessTree) + }); + } finally { + child?.unref?.(); } - if (existing) { - teardownBrokerSession({ + if (cleanup?.verified === true) { + clearBrokerSession(cwd); + return cleanup; + } + + try { + saveBrokerSession(cwd, { ...session, activationFailed: true }); + } catch { + // Preserve the cleanup outcome as the primary failure if even the + // report-only recovery record cannot be written. + } + return cleanup; +} + +async function cleanupExistingBrokerSession(cwd, existing, options) { + const env = options.env ?? process.env; + const acquireRegistryLock = options.acquireBrokerRegistryLockImpl ?? acquireBrokerRegistryLock; + const releaseRegistryLock = options.releaseBrokerRegistryLockImpl ?? releaseBrokerRegistryLock; + const registryLock = acquireRegistryLock(existing.registry); + if (registryLock?.acquired !== true) { + const error = new Error(`Broker cleanup eligibility could not be locked (${registryLock?.reason ?? "unknown"}).`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + + try { + const loadRegistration = options.loadBrokerRegistrationImpl ?? loadBrokerRegistration; + const registration = loadRegistration({ + endpoint: existing.endpoint, + brokerIdentity: existing.pidIdentity, + env + }); + if ( + registration.registered !== true || + registration.brokerKey !== existing.registry.brokerKey || + registration.registryDir !== existing.registry.registryDir + ) { + const error = new Error("The existing Codex broker registration is invalid; cleanup remains report-only."); + error.code = "BROKER_REGISTRATION_REQUIRED"; + throw error; + } + + const readProcessIdentity = options.getProcessIdentityImpl ?? getProcessIdentity; + let currentIdentity; + try { + currentIdentity = readProcessIdentity(existing.pid, { cwd, env }); + } catch (cause) { + const error = new Error("Broker liveness could not be verified; cleanup remains report-only."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + error.cause = cause; + throw error; + } + if (currentIdentity && currentIdentity !== existing.pidIdentity) { + const error = new Error("The saved broker PID has been reused; refusing to signal it or start a replacement."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + if (currentIdentity === existing.pidIdentity) { + const assessOwners = options.assessBrokerOwnersImpl ?? assessBrokerOwners; + const assessment = assessOwners(registration); + if (assessment?.safeToShutdown !== true) { + if (assessment?.reason === "live-owner") { + return { cleaned: false, blockedByLiveOwner: true }; + } + const error = new Error(`Broker ownership is ambiguous (${assessment?.reason ?? "unknown"}); cleanup remains report-only.`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + } + + if (!currentIdentity) { + const loadChildren = options.loadBrokerChildrenImpl ?? loadBrokerChildren; + const children = loadChildren(registration); + if (children?.valid !== true) { + const error = new Error(`Registered broker children are invalid (${children?.reason ?? "unknown"}); cleanup remains report-only.`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + const terminateChild = options.terminateBrokerChildImpl ?? terminateProcessTree; + const releaseChild = options.releaseBrokerChildImpl ?? releaseBrokerChild; + for (const child of children.children) { + const outcome = await terminateChild(child.pid, { + expectedRootIdentity: child.pidIdentity, + ownershipSnapshot: child.ownershipSnapshot, + cwd, + env + }); + if (outcome?.verified !== true) { + const error = new Error("Registered broker child cleanup is unverified; refusing to start a replacement broker."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + const released = releaseChild(registration, { + child, + cleanupOutcome: outcome, + registryLock + }); + if (released?.released !== true) { + const error = new Error(`Registered broker child release failed (${released?.reason ?? "unknown"}); refusing to start a replacement broker.`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + } + } + + const teardownBroker = options.teardownBrokerSessionImpl ?? teardownBrokerSession; + const cleanup = await teardownBroker({ endpoint: existing.endpoint ?? null, pidFile: existing.pidFile ?? null, logFile: existing.logFile ?? null, sessionDir: existing.sessionDir ?? null, pid: existing.pid ?? null, - killProcess: options.killProcess ?? null + pidIdentity: existing.pidIdentity ?? null, + ownershipSnapshot: existing.ownershipSnapshot ?? null, + requireVerifiedOwnership: existing.ownershipCaptureFailed === true, + killProcess: options.killProcess ?? terminateProcessTree }); + if (cleanup?.verified !== true) { + const error = new Error("Broker cleanup is unverified; refusing to start another broker session."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } clearBrokerSession(cwd); + return { cleaned: true, cleanup }; + } finally { + const released = releaseRegistryLock(existing.registry, registryLock); + if (released?.released !== true) { + const error = new Error(`Broker registry lock release failed (${released?.reason ?? "unknown"}).`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + } +} + +async function ensureBrokerSessionLocked(cwd, options = {}) { + const env = options.env ?? process.env; + // Automatic brokers are detached and outlive the process that spawned them. + // Until Windows has a durable, reuse-resistant process identity for the + // registry, use the attached direct app-server lifecycle instead of + // publishing an unregistered broker that SessionEnd must refuse to signal. + const stateFileExists = fs.existsSync(resolveBrokerStateFile(cwd)); + const existing = loadBrokerSession(cwd); + if (stateFileExists && !existing) { + return null; + } + if (existing && existing.registry?.registered !== true) { + return null; + } + let existingProcessIdentity = null; + if (existing && existing.activationFailed !== true) { + try { + const readProcessIdentity = options.getProcessIdentityImpl ?? getProcessIdentity; + existingProcessIdentity = readProcessIdentity(existing.pid, { cwd, env }); + } catch (cause) { + const error = new Error("Existing broker liveness could not be verified; refusing reuse or replacement."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + error.cause = cause; + throw error; + } + } + if ( + existing && + existing.activationPending !== true && + existing.activationFailed !== true && + existingProcessIdentity === existing.pidIdentity && + (await isBrokerEndpointReady(existing.endpoint)) + ) { + const acquireRegistryLock = options.acquireBrokerRegistryLockImpl ?? acquireBrokerRegistryLock; + const releaseRegistryLock = options.releaseBrokerRegistryLockImpl ?? releaseBrokerRegistryLock; + const registryLock = acquireRegistryLock(existing.registry); + if (registryLock?.acquired !== true) { + return null; + } + try { + const loadRegistration = options.loadBrokerRegistrationImpl ?? loadBrokerRegistration; + const registration = loadRegistration({ + endpoint: existing.endpoint, + brokerIdentity: existing.pidIdentity, + env + }); + if ( + registration.registered !== true || + registration.brokerKey !== existing.registry.brokerKey || + registration.registryDir !== existing.registry.registryDir || + !hasLiveBrokerOwnerIdentity(env) + ) { + return null; + } + const readProcessIdentity = options.getProcessIdentityImpl ?? getProcessIdentity; + if ( + readProcessIdentity(existing.pid, { cwd, env }) !== existing.pidIdentity || + !(await isBrokerEndpointReady(existing.endpoint)) + ) { + return null; + } + const registerOwner = options.registerBrokerOwnerImpl ?? registerBrokerOwner; + const owner = registerOwner(registration, { env, registryLock }); + if (owner.registered !== true) { + return null; + } + return { ...existing, registry: registration }; + } finally { + const released = releaseRegistryLock(existing.registry, registryLock); + if (released?.released !== true) { + const error = new Error(`Broker registry lock release failed (${released?.reason ?? "unknown"}).`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + } + } + + if (existing) { + const recovery = await cleanupExistingBrokerSession(cwd, existing, options); + if (recovery.blockedByLiveOwner) { + return null; + } } const sessionDir = createBrokerSessionDir(); @@ -137,7 +630,8 @@ export async function ensureBrokerSession(cwd, options = {}) { options.scriptPath ?? fileURLToPath(new URL("../app-server-broker.mjs", import.meta.url)); - const child = spawnBrokerProcess({ + const spawnBroker = options.spawnBrokerProcessImpl ?? spawnBrokerProcess; + const child = spawnBroker({ scriptPath, cwd, endpoint, @@ -145,18 +639,20 @@ export async function ensureBrokerSession(cwd, options = {}) { logFile, env: options.env ?? process.env }); - - const ready = await waitForBrokerEndpoint(endpoint, options.timeoutMs ?? 2000); - if (!ready) { - teardownBrokerSession({ - endpoint, - pidFile, - logFile, - sessionDir, - pid: child.pid ?? null, - killProcess: options.killProcess ?? null - }); - return null; + const captureOwnership = options.captureProcessOwnershipImpl ?? captureProcessOwnership; + let ownershipSnapshot = null; + let ownershipCaptureFailed = false; + if ((options.platform ?? process.platform) !== "win32") { + try { + ownershipSnapshot = captureOwnership(child.pid ?? Number.NaN, { + cwd, + env: options.env ?? process.env, + platform: options.platform + }); + ownershipCaptureFailed = !ownershipSnapshot?.rootIdentity; + } catch { + ownershipCaptureFailed = true; + } } const session = { @@ -164,18 +660,171 @@ export async function ensureBrokerSession(cwd, options = {}) { pidFile, logFile, sessionDir, - pid: child.pid ?? null + pid: child.pid ?? null, + pidIdentity: ownershipSnapshot?.rootIdentity ?? null, + ownershipSnapshot, + ownershipCaptureFailed, + registry: null }; - saveBrokerSession(cwd, session); - return session; + const ready = await waitForBrokerEndpoint(endpoint, options.timeoutMs ?? 2000); + if (!ready) { + const cleanup = await rollbackNewBrokerSession(cwd, child, session, options); + if (cleanup?.verified !== true) { + const error = new Error("Failed broker startup cleanup is unverified; refusing to create or hide another process."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + return null; + } + + let fallbackToDirect = false; + let ownerRegistered = false; + let transactionRegistryLock = null; + try { + const publishRegistration = options.publishRegisteredBrokerImpl ?? publishRegisteredBroker; + const candidate = publishRegistration({ + cwd, + endpoint, + pid: child.pid ?? null, + ownershipSnapshot, + env, + retainRegistryLock: true + }); + if (candidate.registered !== true) { + fallbackToDirect = [ + "broker-identity-unavailable", + "broker-not-live", + "plugin-data-unavailable", + "session-owner-not-live", + "session-owner-unavailable" + ].includes(candidate.reason); + throw new Error(`Broker registration is unavailable (${candidate.reason ?? "unknown"}).`); + } + if (candidate.registryLock?.acquired !== true) { + throw new Error("Broker registration did not retain the launch transaction lock."); + } + transactionRegistryLock = candidate.registryLock; + const { registryLock: _registryLock, ...registration } = candidate; + session.registry = registration; + ownerRegistered = true; + const saveSession = options.saveBrokerSessionImpl ?? saveBrokerSession; + session.activationPending = true; + saveSession(cwd, session); + + const activateBroker = options.activateBrokerProcessImpl ?? activateBrokerProcess; + await activateBroker(child, options.activationTimeoutMs ?? 2000); + session.activationPending = false; + saveSession(cwd, session); + return session; + } catch (error) { + // Keep the initial owner and the publication-time registry lock intact + // until exact rollback has converged. Otherwise a reaper can classify the + // half-launched broker as abandoned and race the launcher's own teardown. + const cleanup = await rollbackNewBrokerSession(cwd, child, session, options); + if (ownerRegistered && cleanup?.verified === true) { + const releaseOwner = options.releaseBrokerOwnerImpl ?? releaseBrokerOwner; + try { + const released = releaseOwner(session.registry, { + env, + ...(transactionRegistryLock ? { registryLock: transactionRegistryLock } : {}) + }); + if (released.released !== true) { + process.stderr.write(`Warning: unable to release rolled-back Codex broker owner (${released.reason ?? "unknown"}).\n`); + } + } catch (releaseError) { + process.stderr.write(`Warning: unable to release rolled-back Codex broker owner: ${releaseError.message}.\n`); + } + } + if (cleanup?.verified !== true) { + const cleanupError = new Error(`Broker launch failed and exact rollback is unverified: ${error.message}`); + cleanupError.code = "BROKER_CLEANUP_UNVERIFIED"; + cleanupError.cause = error; + throw cleanupError; + } + if (fallbackToDirect) { + return null; + } + const transactionError = new Error(`Broker launch transaction failed: ${error.message}`); + transactionError.code = "BROKER_REGISTRATION_FAILED"; + transactionError.cause = error; + throw transactionError; + } finally { + if (transactionRegistryLock && session.registry) { + const releaseRegistryLock = options.releaseBrokerRegistryLockImpl ?? releaseBrokerRegistryLock; + const released = releaseRegistryLock(session.registry, transactionRegistryLock); + if (released?.released !== true) { + const lockError = new Error(`Broker launch transaction lock release failed (${released?.reason ?? "unknown"}).`); + lockError.code = "BROKER_CLEANUP_UNVERIFIED"; + throw lockError; + } + } + } } -export function teardownBrokerSession({ endpoint = null, pidFile, logFile, sessionDir = null, pid = null, killProcess = null }) { +export async function ensureBrokerSession(cwd, options = {}) { + if ((options.platform ?? process.platform) === "win32") { + return null; + } + const env = options.env ?? process.env; + if (!resolveBrokerOwnershipRoot(env) || !hasLiveBrokerOwnerIdentity(env)) { + return null; + } + const acquireLaunchLock = options.acquireBrokerLaunchLockImpl ?? acquireBrokerLaunchLock; + let launchLock; + try { + launchLock = await acquireLaunchLock(cwd, { timeoutMs: options.launchLockTimeoutMs ?? 10_000 }); + } catch { + return null; + } + try { + return await ensureBrokerSessionLocked(cwd, options); + } finally { + await launchLock.release(); + } +} + +export async function teardownBrokerSession({ + endpoint = null, + pidFile, + logFile, + sessionDir = null, + pid = null, + pidIdentity = null, + ownershipSnapshot = null, + requireVerifiedOwnership = false, + killProcess = null +}) { + let cleanupOutcome = { + attempted: false, + delivered: false, + verified: true, + degraded: false, + method: null, + targets: [], + targetIdentities: [], + survivors: [], + survivorIdentities: [] + }; if (Number.isFinite(pid) && killProcess) { try { - killProcess(pid); - } catch { - // Ignore missing or already-exited broker processes. + const outcome = await killProcess(pid, { + expectedRootIdentity: pidIdentity, + ownershipSnapshot, + requireVerifiedOwnership + }); + cleanupOutcome = outcome ?? { + ...cleanupOutcome, + attempted: true, + verified: false, + degraded: true + }; + } catch (error) { + if (error?.code !== "ESRCH" && error?.code !== "ENOENT") { + throw error; + } + } + if (cleanupOutcome.verified !== true) { + return cleanupOutcome; } } @@ -206,4 +855,5 @@ export function teardownBrokerSession({ endpoint = null, pidFile, logFile, sessi // Ignore non-empty or missing directories. } } + return cleanupOutcome; } diff --git a/plugins/codex/scripts/lib/broker-ownership.mjs b/plugins/codex/scripts/lib/broker-ownership.mjs new file mode 100644 index 000000000..d76686eb5 --- /dev/null +++ b/plugins/codex/scripts/lib/broker-ownership.mjs @@ -0,0 +1,1273 @@ +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; + +import { getLiveProcessPids, getProcessIdentity, hasLiveProcessIdentity } from "./process.mjs"; + +export const BROKER_OWNERSHIP_VERSION = 1; +export const SESSION_OWNER_PID_ENV = "CODEX_COMPANION_SESSION_OWNER_PID"; +export const SESSION_OWNER_IDENTITY_ENV = "CODEX_COMPANION_SESSION_OWNER_IDENTITY"; + +const PLUGIN_DATA_ENV = "CLAUDE_PLUGIN_DATA"; +const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; +const REGISTRY_DIR_NAME = "broker-ownership-v1"; +const REGISTRY_LOCK_DIR_NAME = "registry.lock"; +const TERMINAL_FILE_NAME = "terminal.json"; + +function sha256(value) { + return createHash("sha256").update(String(value)).digest("hex"); +} + +function isSafePid(value) { + return Number.isSafeInteger(value) && value > 1; +} + +function isRegistryLockToken(value) { + return typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value); +} + +function isPidIdentity(pid, identity) { + return isSafePid(pid) && typeof identity === "string" && identity.startsWith(`${pid}@`) && identity.length > String(pid).length + 1; +} + +function requirePrivateDirectory(dirPath) { + const stat = fs.lstatSync(dirPath); + if (stat.isSymbolicLink() || !stat.isDirectory() || (stat.mode & 0o777) !== 0o700) { + const error = new Error(`Broker ownership registry directory is not private: ${dirPath}.`); + error.code = "BROKER_OWNERSHIP_PERMISSIONS"; + throw error; + } + return stat; +} + +function requirePrivateFile(filePath) { + const stat = fs.lstatSync(filePath); + if (stat.isSymbolicLink() || !stat.isFile() || (stat.mode & 0o777) !== 0o600) { + const error = new Error(`Broker ownership registry file is not private: ${filePath}.`); + error.code = "BROKER_OWNERSHIP_PERMISSIONS"; + throw error; + } + return stat; +} + +function ensurePrivateDir(dirPath) { + fs.mkdirSync(dirPath, { recursive: true, mode: 0o700 }); + const stat = fs.lstatSync(dirPath); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + const error = new Error(`Refusing non-directory broker ownership path: ${dirPath}.`); + error.code = "BROKER_OWNERSHIP_PERMISSIONS"; + throw error; + } + fs.chmodSync(dirPath, 0o700); + requirePrivateDirectory(dirPath); +} + +function immutableJsonBytes(payload) { + return `${JSON.stringify(payload, null, 2)}\n`; +} + +function createImmutableJson(filePath, payload) { + const bytes = immutableJsonBytes(payload); + ensurePrivateDir(path.dirname(filePath)); + if (fs.existsSync(filePath)) { + requirePrivateFile(filePath); + if (fs.readFileSync(filePath, "utf8") !== bytes) { + const error = new Error(`Broker ownership registry collision at ${filePath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + fs.chmodSync(filePath, 0o600); + return filePath; + } + + const tempPath = path.join(path.dirname(filePath), `.${path.basename(filePath)}.${process.pid}.${randomUUID()}.tmp`); + try { + fs.writeFileSync(tempPath, bytes, { encoding: "utf8", flag: "wx", mode: 0o600 }); + fs.chmodSync(tempPath, 0o600); + try { + fs.linkSync(tempPath, filePath); + } catch (error) { + if (error?.code !== "EEXIST") { + throw error; + } + requirePrivateFile(filePath); + if (fs.readFileSync(filePath, "utf8") !== bytes) { + throw error; + } + } + fs.chmodSync(filePath, 0o600); + requirePrivateFile(filePath); + return filePath; + } finally { + try { + fs.unlinkSync(tempPath); + } catch (error) { + if (error?.code !== "ENOENT") { + throw error; + } + } + } +} + +function readJson(filePath) { + requirePrivateFile(filePath); + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function registrationReference(registryRoot, brokerKey) { + return { + registered: true, + version: BROKER_OWNERSHIP_VERSION, + brokerKey, + registryRoot, + registryDir: path.join(registryRoot, brokerKey) + }; +} + +function validRegistrationReference(registration) { + return Boolean( + registration?.registered === true && + registration.version === BROKER_OWNERSHIP_VERSION && + typeof registration.brokerKey === "string" && + /^[a-f0-9]{64}$/.test(registration.brokerKey) && + typeof registration.registryRoot === "string" && + path.isAbsolute(registration.registryRoot) && + registration.registryDir === path.join(registration.registryRoot, registration.brokerKey) + ); +} + +function validRegistryLock(registration, lock) { + return Boolean( + validRegistrationReference(registration) && + lock?.acquired === true && + lock.brokerKey === registration.brokerKey && + isRegistryLockToken(lock.token) && + isSafePid(lock.pid) && + isPidIdentity(lock.pid, lock.pidIdentity) && + lock.path === path.join(registration.registryDir, REGISTRY_LOCK_DIR_NAME) + ); +} + +function createPreparedRegistryLock(registration, preparedRegistryDir, options = {}) { + const pid = options.pid ?? process.pid; + if (!isSafePid(pid)) { + return { acquired: false, reason: "registry-lock-owner-invalid" }; + } + const getProcessIdentityImpl = options.getProcessIdentityImpl ?? getProcessIdentity; + let pidIdentity = options.pidIdentity ?? null; + if (!pidIdentity) { + try { + pidIdentity = getProcessIdentityImpl(pid); + } catch { + return { acquired: false, reason: "registry-lock-owner-identity-unavailable" }; + } + } + if (!isPidIdentity(pid, pidIdentity)) { + return { acquired: false, reason: "registry-lock-owner-identity-unavailable" }; + } + const token = randomUUID(); + const preparedLockPath = path.join(preparedRegistryDir, REGISTRY_LOCK_DIR_NAME); + ensurePrivateDir(preparedLockPath); + createImmutableJson(path.join(preparedLockPath, "owner.json"), { + version: BROKER_OWNERSHIP_VERSION, + kind: "registry-lock", + brokerKey: registration.brokerKey, + token, + pid, + pidIdentity, + acquiredAt: (options.now ?? (() => new Date().toISOString()))() + }); + return { + acquired: true, + brokerKey: registration.brokerKey, + token, + pid, + pidIdentity, + path: path.join(registration.registryDir, REGISTRY_LOCK_DIR_NAME) + }; +} + +export function acquireBrokerRegistryLock( + registration, + { + now = () => new Date().toISOString(), + pid = process.pid, + pidIdentity: suppliedPidIdentity = null, + getProcessIdentityImpl = getProcessIdentity, + getLiveProcessPidsImpl = getLiveProcessPids, + hasLiveProcessIdentityImpl = hasLiveProcessIdentity + } = {} +) { + if (!validRegistrationReference(registration)) { + return { acquired: false, reason: "broker-registration-unavailable" }; + } + requirePrivateDirectory(registration.registryRoot); + requirePrivateDirectory(registration.registryDir); + let pidIdentity = suppliedPidIdentity; + if (!pidIdentity) { + try { + pidIdentity = getProcessIdentityImpl(pid); + } catch { + return { acquired: false, reason: "registry-lock-owner-identity-unavailable" }; + } + } + if (!isPidIdentity(pid, pidIdentity)) { + return { acquired: false, reason: "registry-lock-owner-identity-unavailable" }; + } + const lockPath = path.join(registration.registryDir, REGISTRY_LOCK_DIR_NAME); + const token = randomUUID(); + for (let attempt = 0; attempt < 3; attempt += 1) { + const preparedPath = path.join(registration.registryDir, `.registry-lock.${pid}.${randomUUID()}`); + try { + ensurePrivateDir(preparedPath); + createImmutableJson(path.join(preparedPath, "owner.json"), { + version: BROKER_OWNERSHIP_VERSION, + kind: "registry-lock", + brokerKey: registration.brokerKey, + token, + pid, + pidIdentity, + acquiredAt: now() + }); + try { + fs.renameSync(preparedPath, lockPath); + return { acquired: true, brokerKey: registration.brokerKey, token, pid, pidIdentity, path: lockPath }; + } catch (error) { + if (error?.code !== "EEXIST" && error?.code !== "ENOTEMPTY") { + throw error; + } + } + } finally { + if (fs.existsSync(preparedPath)) { + fs.rmSync(preparedPath, { recursive: true, force: true }); + } + } + + let existingOwner; + try { + requirePrivateDirectory(lockPath); + existingOwner = readJson(path.join(lockPath, "owner.json")); + } catch (error) { + if (error?.code === "ENOENT") { + continue; + } + return { acquired: false, reason: "registry-lock-malformed", path: lockPath }; + } + if ( + existingOwner?.version !== BROKER_OWNERSHIP_VERSION || + existingOwner.kind !== "registry-lock" || + existingOwner.brokerKey !== registration.brokerKey || + !isRegistryLockToken(existingOwner.token) || + !isSafePid(existingOwner.pid) || + (existingOwner.pidIdentity != null && !isPidIdentity(existingOwner.pid, existingOwner.pidIdentity)) + ) { + return { acquired: false, reason: "registry-lock-malformed", path: lockPath }; + } + let live = false; + try { + if (existingOwner.pidIdentity == null) { + const livePids = getLiveProcessPidsImpl([existingOwner.pid]); + if (!Array.isArray(livePids)) { + return { acquired: false, reason: "registry-lock-liveness-unavailable", path: lockPath }; + } + live = livePids.includes(existingOwner.pid); + } else { + live = hasLiveProcessIdentityImpl(existingOwner.pid, existingOwner.pidIdentity); + if (typeof live !== "boolean") { + return { acquired: false, reason: "registry-lock-liveness-unavailable", path: lockPath }; + } + } + } catch { + return { acquired: false, reason: "registry-lock-liveness-unavailable", path: lockPath }; + } + if (live) { + return { acquired: false, reason: "registry-busy", path: lockPath }; + } + + const staleRoot = path.join(registration.registryDir, "stale-locks"); + ensurePrivateDir(staleRoot); + const stalePath = path.join(staleRoot, `${existingOwner.pid}-${existingOwner.token}`); + try { + fs.renameSync(lockPath, stalePath); + } catch (error) { + if (error?.code === "ENOENT" || error?.code === "EEXIST" || error?.code === "ENOTEMPTY") { + continue; + } + return { acquired: false, reason: "registry-lock-quarantine-failed", path: lockPath }; + } + } + return { acquired: false, reason: "registry-lock-contention", path: lockPath }; +} + +export function releaseBrokerRegistryLock(registration, lock) { + if (!validRegistryLock(registration, lock)) { + return { released: false, reason: "registry-lock-unavailable" }; + } + const ownerPath = path.join(lock.path, "owner.json"); + try { + const owner = readJson(ownerPath); + if ( + owner?.version !== BROKER_OWNERSHIP_VERSION || + owner.kind !== "registry-lock" || + owner.brokerKey !== registration.brokerKey || + owner.token !== lock.token || + owner.pid !== lock.pid || + owner.pidIdentity !== lock.pidIdentity + ) { + return { released: false, reason: "registry-lock-mismatch" }; + } + fs.unlinkSync(ownerPath); + fs.rmdirSync(lock.path); + return { released: true }; + } catch (error) { + if (error?.code === "ENOENT") { + return { released: false, reason: "registry-lock-missing" }; + } + throw error; + } +} + +function withBrokerRegistryLock(registration, options, action) { + const suppliedLock = options.registryLock; + if (suppliedLock && !validRegistryLock(registration, suppliedLock)) { + return { ok: false, reason: "registry-lock-unavailable" }; + } + const lock = suppliedLock ?? acquireBrokerRegistryLock(registration, options); + if (lock.acquired !== true) { + return { ok: false, reason: lock.reason ?? "registry-busy" }; + } + try { + return { ok: true, value: action() }; + } finally { + if (!suppliedLock) { + const released = releaseBrokerRegistryLock(registration, lock); + if (released.released !== true) { + const error = new Error(`Unable to release broker registry lock (${released.reason}).`); + error.code = "BROKER_REGISTRY_LOCK_RELEASE_FAILED"; + throw error; + } + } + } +} + +export function resolveBrokerOwnershipRoot(env = process.env) { + const pluginDataDir = env?.[PLUGIN_DATA_ENV]; + if (typeof pluginDataDir !== "string" || !path.isAbsolute(pluginDataDir)) { + return null; + } + return path.join(pluginDataDir, "state", REGISTRY_DIR_NAME); +} + +export function brokerOwnershipKey(endpoint, brokerIdentity) { + return sha256(`${endpoint}\0${brokerIdentity}`); +} + +export function loadBrokerRegistration({ endpoint, brokerIdentity, env = process.env }) { + const registryRoot = resolveBrokerOwnershipRoot(env); + if (!registryRoot || typeof endpoint !== "string" || !endpoint || typeof brokerIdentity !== "string" || !brokerIdentity) { + return { registered: false, reason: "missing-registration-identity" }; + } + const registration = registrationReference(registryRoot, brokerOwnershipKey(endpoint, brokerIdentity)); + const brokerPath = path.join(registration.registryDir, "broker.json"); + if (!fs.existsSync(brokerPath)) { + return { registered: false, reason: "broker-record-absent" }; + } + try { + requirePrivateDirectory(registryRoot); + requirePrivateDirectory(registration.registryDir); + const broker = readJson(brokerPath); + if ( + broker?.version !== BROKER_OWNERSHIP_VERSION || + broker.kind !== "broker" || + broker.brokerKey !== registration.brokerKey || + broker.endpoint !== endpoint || + broker.pidIdentity !== brokerIdentity || + !isPidIdentity(broker.pid, broker.pidIdentity) + ) { + return { registered: false, reason: "broker-record-invalid" }; + } + return { ...registration, broker }; + } catch { + return { registered: false, reason: "broker-record-invalid" }; + } +} + +function validBrokerTerminal(record, filePath, registration) { + return Boolean( + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "terminal" && + record.brokerKey === registration.brokerKey && + record.pidIdentity === registration.broker?.pidIdentity && + record.decision === "cleanup-verified" && + typeof record.attemptId === "string" && + /^[a-zA-Z0-9._-]{1,128}$/.test(record.attemptId) && + record.receipt === `${record.attemptId}.json` && + typeof record.retiredAt === "string" && + record.retiredAt.length > 0 && + filePath === path.join(registration.registryDir, TERMINAL_FILE_NAME) + ); +} + +export function loadBrokerTerminal(registration) { + if (!validRegistrationReference(registration)) { + return { terminal: false, reason: "broker-registration-unavailable" }; + } + const terminalPath = path.join(registration.registryDir, TERMINAL_FILE_NAME); + if (!fs.existsSync(terminalPath)) { + return { terminal: false, reason: "terminal-absent", path: terminalPath }; + } + try { + const record = readJson(terminalPath); + if (!validBrokerTerminal(record, terminalPath, registration)) { + return { terminal: false, reason: "terminal-invalid", path: terminalPath }; + } + return { terminal: true, reason: "cleanup-verified", path: terminalPath, record }; + } catch { + return { terminal: false, reason: "terminal-invalid", path: terminalPath }; + } +} + +export function publishBrokerRegistration({ cwd, endpoint, pid, ownershipSnapshot, env = process.env, now = () => new Date().toISOString() }) { + const registryRoot = resolveBrokerOwnershipRoot(env); + if (!registryRoot) { + return { registered: false, reason: "plugin-data-unavailable" }; + } + if ( + typeof endpoint !== "string" || + !endpoint || + !isSafePid(pid) || + ownershipSnapshot?.rootPid !== pid || + !isPidIdentity(pid, ownershipSnapshot?.rootIdentity) || + !isSafePid(ownershipSnapshot?.processGroupId) + ) { + return { registered: false, reason: "broker-identity-unavailable" }; + } + + const brokerKey = brokerOwnershipKey(endpoint, ownershipSnapshot.rootIdentity); + const registration = registrationReference(registryRoot, brokerKey); + const existing = loadBrokerRegistration({ endpoint, brokerIdentity: ownershipSnapshot.rootIdentity, env }); + if (existing.registered === true) { + return existing; + } + const broker = { + version: BROKER_OWNERSHIP_VERSION, + kind: "broker", + brokerKey, + endpoint, + pid, + pidIdentity: ownershipSnapshot.rootIdentity, + processGroupId: ownershipSnapshot.processGroupId, + workspaceHash: sha256(path.resolve(cwd)), + createdAt: now() + }; + createImmutableJson(path.join(registration.registryDir, "broker.json"), broker); + return { ...registration, broker }; +} + +function ownerFromEnv(env) { + const sessionId = env?.[SESSION_ID_ENV]; + const pid = Number(env?.[SESSION_OWNER_PID_ENV]); + const identity = env?.[SESSION_OWNER_IDENTITY_ENV]; + if (typeof sessionId !== "string" || !sessionId || !isPidIdentity(pid, identity)) { + return null; + } + const ownerKey = sha256(`${sessionId}\0${identity}`); + return { ownerKey, sessionId, pid, identity }; +} + +export function hasLiveBrokerOwnerIdentity(env = process.env, options = {}) { + const owner = ownerFromEnv(env); + if (!owner) { + return false; + } + const hasLiveIdentity = options.hasLiveProcessIdentityImpl ?? hasLiveProcessIdentity; + try { + return hasLiveIdentity(owner.pid, owner.identity); + } catch { + return false; + } +} + +export function publishRegisteredBroker({ + cwd, + endpoint, + pid, + ownershipSnapshot, + env = process.env, + now = () => new Date().toISOString(), + hasLiveProcessIdentityImpl = hasLiveProcessIdentity, + getProcessIdentityImpl = getProcessIdentity, + retainRegistryLock = false +}) { + const registryRoot = resolveBrokerOwnershipRoot(env); + if (!registryRoot) { + return { registered: false, reason: "plugin-data-unavailable" }; + } + const owner = ownerFromEnv(env); + if (!owner) { + return { registered: false, reason: "session-owner-unavailable" }; + } + if (!hasLiveBrokerOwnerIdentity(env, { hasLiveProcessIdentityImpl })) { + return { registered: false, reason: "session-owner-not-live" }; + } + if ( + typeof endpoint !== "string" || + !endpoint || + !isSafePid(pid) || + ownershipSnapshot?.rootPid !== pid || + !isPidIdentity(pid, ownershipSnapshot?.rootIdentity) || + !isSafePid(ownershipSnapshot?.processGroupId) + ) { + return { registered: false, reason: "broker-identity-unavailable" }; + } + + const brokerKey = brokerOwnershipKey(endpoint, ownershipSnapshot.rootIdentity); + const registration = registrationReference(registryRoot, brokerKey); + const createdAt = now(); + const broker = { + version: BROKER_OWNERSHIP_VERSION, + kind: "broker", + brokerKey, + endpoint, + pid, + pidIdentity: ownershipSnapshot.rootIdentity, + processGroupId: ownershipSnapshot.processGroupId, + workspaceHash: sha256(path.resolve(cwd)), + createdAt + }; + const ownerRecord = { + version: BROKER_OWNERSHIP_VERSION, + kind: "owner", + brokerKey, + ownerKey: owner.ownerKey, + sessionId: owner.sessionId, + pid: owner.pid, + pidIdentity: owner.identity, + registeredAt: createdAt + }; + + ensurePrivateDir(registryRoot); + const preparedDir = path.join(registryRoot, `.${brokerKey}.${process.pid}.${randomUUID()}.prepared`); + fs.mkdirSync(preparedDir, { mode: 0o700 }); + try { + createImmutableJson(path.join(preparedDir, "broker.json"), broker); + createImmutableJson(path.join(preparedDir, "owners", `${owner.ownerKey}.json`), ownerRecord); + const registryLock = retainRegistryLock + ? createPreparedRegistryLock(registration, preparedDir, { now, getProcessIdentityImpl }) + : null; + if (retainRegistryLock && registryLock?.acquired !== true) { + return { registered: false, reason: registryLock?.reason ?? "registry-lock-unavailable" }; + } + // Publication is the point at which the reaper can first observe this + // broker. Revalidate both identities after all prepared bytes exist and + // immediately before the atomic rename so the visible initial state never + // begins with a dead sole owner or a reused broker PID. + if (!hasLiveProcessIdentityImpl(owner.pid, owner.identity)) { + return { registered: false, reason: "session-owner-not-live" }; + } + if (!hasLiveProcessIdentityImpl(pid, ownershipSnapshot.rootIdentity)) { + return { registered: false, reason: "broker-not-live" }; + } + fs.renameSync(preparedDir, registration.registryDir); + requirePrivateDirectory(registration.registryDir); + return { + ...registration, + broker, + ownerKey: owner.ownerKey, + ...(registryLock ? { registryLock } : {}) + }; + } finally { + if (fs.existsSync(preparedDir)) { + fs.rmSync(preparedDir, { recursive: true, force: true }); + } + } +} + +export function registerBrokerOwner(registration, options = {}) { + const { env = process.env, now = () => new Date().toISOString() } = options; + if (!validRegistrationReference(registration)) { + return { registered: false, reason: "broker-registration-unavailable" }; + } + const owner = ownerFromEnv(env); + if (!owner) { + return { registered: false, reason: "session-owner-unavailable" }; + } + const locked = withBrokerRegistryLock(registration, options, () => { + const hasLiveIdentity = options.hasLiveProcessIdentityImpl ?? hasLiveProcessIdentity; + const ownerIsLive = () => { + try { + return hasLiveIdentity(owner.pid, owner.identity) === true; + } catch { + return false; + } + }; + const ownerPath = path.join(registration.registryDir, "owners", `${owner.ownerKey}.json`); + if (fs.existsSync(ownerPath)) { + const existing = readJson(ownerPath); + if (!validOwnerRecord(existing, ownerPath, registration)) { + const error = new Error(`Invalid existing broker owner row at ${ownerPath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + if (!ownerIsLive()) { + return { registered: false, reason: "session-owner-not-live" }; + } + fs.chmodSync(ownerPath, 0o600); + return { registered: true, ownerKey: owner.ownerKey, path: ownerPath, owner: existing }; + } + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "owner", + brokerKey: registration.brokerKey, + ownerKey: owner.ownerKey, + sessionId: owner.sessionId, + pid: owner.pid, + pidIdentity: owner.identity, + registeredAt: now() + }; + if (!ownerIsLive()) { + return { registered: false, reason: "session-owner-not-live" }; + } + createImmutableJson(ownerPath, payload); + return { registered: true, ownerKey: owner.ownerKey, path: ownerPath, owner: payload }; + }); + return locked.ok ? locked.value : { registered: false, reason: locked.reason }; +} + +export function releaseBrokerOwner(registration, options = {}) { + const { env = process.env, now = () => new Date().toISOString() } = options; + if (!validRegistrationReference(registration)) { + return { released: false, reason: "broker-registration-unavailable" }; + } + const owner = ownerFromEnv(env); + if (!owner) { + return { released: false, reason: "session-owner-unavailable" }; + } + const locked = withBrokerRegistryLock(registration, options, () => { + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "release", + brokerKey: registration.brokerKey, + ownerKey: owner.ownerKey, + sessionId: owner.sessionId, + pid: owner.pid, + pidIdentity: owner.identity, + releasedAt: now() + }; + const releasePath = path.join(registration.registryDir, "releases", `${owner.ownerKey}.json`); + if (fs.existsSync(releasePath)) { + const existing = readJson(releasePath); + if (!validReleaseRecord(existing, releasePath, registration)) { + const error = new Error(`Invalid existing broker owner release at ${releasePath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + fs.chmodSync(releasePath, 0o600); + return { released: true, ownerKey: owner.ownerKey, path: releasePath, release: existing }; + } + createImmutableJson(releasePath, payload); + return { released: true, ownerKey: owner.ownerKey, path: releasePath, release: payload }; + }); + return locked.ok ? locked.value : { released: false, reason: locked.reason }; +} + +function listJsonFiles(dirPath) { + try { + requirePrivateDirectory(dirPath); + } catch (error) { + if (error?.code === "ENOENT") { + return []; + } + throw error; + } + return fs.readdirSync(dirPath) + .filter((name) => name.endsWith(".json") && !name.startsWith(".")) + .sort() + .map((name) => path.join(dirPath, name)); +} + +function validOwnerRecord(record, filePath, registration) { + return Boolean( + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "owner" && + record.brokerKey === registration.brokerKey && + typeof record.ownerKey === "string" && + path.basename(filePath) === `${record.ownerKey}.json` && + record.ownerKey === sha256(`${record.sessionId}\0${record.pidIdentity}`) && + isPidIdentity(record.pid, record.pidIdentity) && + typeof record.sessionId === "string" && + record.sessionId.length > 0 + ); +} + +function validReleaseRecord(record, filePath, registration) { + return Boolean( + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "release" && + record.brokerKey === registration.brokerKey && + typeof record.ownerKey === "string" && + path.basename(filePath) === `${record.ownerKey}.json` && + record.ownerKey === sha256(`${record.sessionId}\0${record.pidIdentity}`) && + isPidIdentity(record.pid, record.pidIdentity) && + typeof record.sessionId === "string" && + record.sessionId.length > 0 + ); +} + +function validSnapshotMember(record) { + return Boolean( + isSafePid(record?.pid) && + Number.isSafeInteger(record.parentPid) && + record.parentPid >= 0 && + isSafePid(record.processGroupId) && + (record.sessionId == null || isSafePid(record.sessionId)) && + typeof record.state === "string" && + record.state.length > 0 && + typeof record.startedAt === "string" && + record.startedAt.length > 0 && + isPidIdentity(record.pid, record.identity) && + Number.isSafeInteger(record.depth) && + record.depth >= 0 + ); +} + +function validOwnershipTree(snapshot, rootPid, rootIdentity) { + if (!Array.isArray(snapshot?.members) || snapshot.members.length === 0) { + return false; + } + const membersByPid = new Map(); + const memberIdentities = new Set(); + for (const member of snapshot.members) { + if ( + !validSnapshotMember(member) || + membersByPid.has(member.pid) || + memberIdentities.has(member.identity) + ) { + return false; + } + membersByPid.set(member.pid, member); + memberIdentities.add(member.identity); + } + const root = membersByPid.get(rootPid); + if (root?.identity !== rootIdentity || root.depth !== 0) { + return false; + } + for (const member of snapshot.members) { + if (member.pid === rootPid) { + continue; + } + const parent = membersByPid.get(member.parentPid); + if (!parent || member.depth !== parent.depth + 1) { + return false; + } + } + return true; +} + +function validChildRecord(record, filePath, registration) { + const snapshot = record?.ownershipSnapshot; + return Boolean( + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "child" && + record.brokerKey === registration.brokerKey && + typeof record.childKey === "string" && + record.childKey === sha256(record.pidIdentity) && + path.basename(filePath) === `${record.childKey}.json` && + isPidIdentity(record.pid, record.pidIdentity) && + isSafePid(record.processGroupId) && + snapshot?.rootPid === record.pid && + snapshot.rootIdentity === record.pidIdentity && + snapshot.processGroupId === record.processGroupId && + (snapshot.sessionId == null || + (isSafePid(snapshot.sessionId) && + snapshot.sessionId === snapshot.rootPid)) && + validOwnershipTree(snapshot, record.pid, record.pidIdentity) && + (snapshot.sessionId == null || snapshot.members.find((member) => member.pid === record.pid)?.sessionId === snapshot.sessionId) + ); +} + +function validChildObservation(record, filePath, registration, child) { + const snapshot = record?.ownershipSnapshot; + return Boolean( + child && + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "child-observation" && + record.brokerKey === registration.brokerKey && + record.childKey === child.childKey && + typeof record.observationKey === "string" && + record.observationKey === sha256(JSON.stringify(snapshot)) && + path.basename(filePath) === `${record.observationKey}.json` && + snapshot?.rootPid === child.pid && + snapshot.rootIdentity === child.pidIdentity && + snapshot.processGroupId === child.processGroupId && + (snapshot.sessionId == null || + (isSafePid(snapshot.sessionId) && snapshot.sessionId === snapshot.rootPid)) && + validOwnershipTree(snapshot, child.pid, child.pidIdentity) && + (snapshot.sessionId == null || snapshot.members.find((member) => member.pid === child.pid)?.sessionId === snapshot.sessionId) && + typeof record.observedAt === "string" && + record.observedAt.length > 0 + ); +} + +function mergeChildOwnership(child, observations) { + const members = new Map(); + for (const snapshot of [child.ownershipSnapshot, ...observations.map((observation) => observation.ownershipSnapshot)]) { + for (const member of snapshot.members) { + members.set(member.identity, member); + } + } + return { + ...child, + ownershipSnapshot: { + ...child.ownershipSnapshot, + members: [...members.values()] + } + }; +} + +function validChildReleaseRecord(record, filePath, registration, child) { + return Boolean( + child && + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "child-release" && + record.brokerKey === registration.brokerKey && + record.childKey === child.childKey && + path.basename(filePath) === `${record.childKey}.json` && + record.pid === child.pid && + record.pidIdentity === child.pidIdentity && + record.cleanupVerified === true + ); +} + +export function loadBrokerChildren(registration) { + if (!validRegistrationReference(registration)) { + return { valid: false, reason: "broker-registration-unavailable", children: [], malformed: [] }; + } + const children = []; + const releases = new Map(); + const malformed = []; + let childFiles; + let childReleaseFiles; + try { + childFiles = listJsonFiles(path.join(registration.registryDir, "children")); + childReleaseFiles = listJsonFiles(path.join(registration.registryDir, "child-releases")); + } catch { + return { + valid: false, + reason: "malformed-child-registry", + children: [], + releasedChildren: [], + malformed: [registration.registryDir] + }; + } + for (const filePath of childFiles) { + try { + const record = readJson(filePath); + if (!validChildRecord(record, filePath, registration)) { + malformed.push(filePath); + } else { + children.push(record); + } + } catch { + malformed.push(filePath); + } + } + const observedChildren = []; + for (const child of children) { + const observationDir = path.join(registration.registryDir, "child-observations", child.childKey); + const observations = []; + if (fs.existsSync(observationDir)) { + try { + requirePrivateDirectory(observationDir); + for (const filePath of listJsonFiles(observationDir)) { + const observation = readJson(filePath); + if (!validChildObservation(observation, filePath, registration, child)) { + malformed.push(filePath); + } else { + observations.push(observation); + } + } + } catch { + malformed.push(observationDir); + } + } + observedChildren.push(mergeChildOwnership(child, observations)); + } + const childrenByKey = new Map(observedChildren.map((child) => [child.childKey, child])); + for (const filePath of childReleaseFiles) { + try { + const record = readJson(filePath); + if (!validChildReleaseRecord(record, filePath, registration, childrenByKey.get(record?.childKey))) { + malformed.push(filePath); + } else { + releases.set(record.childKey, record); + } + } catch { + malformed.push(filePath); + } + } + return malformed.length > 0 + ? { valid: false, reason: "malformed-child-registry", children: [], releasedChildren: [], malformed } + : { + valid: true, + reason: "valid-child-registry", + children: observedChildren.filter((child) => !releases.has(child.childKey)), + releasedChildren: observedChildren.filter((child) => releases.has(child.childKey)), + malformed: [] + }; +} + +export function releaseBrokerChild( + registration, + options = {} +) { + const { child, cleanupOutcome, now = () => new Date().toISOString() } = options; + if (!validRegistrationReference(registration)) { + return { released: false, reason: "broker-registration-unavailable" }; + } + const childPath = path.join(registration.registryDir, "children", `${child?.childKey}.json`); + let persistedChild; + try { + persistedChild = readJson(childPath); + } catch { + return { released: false, reason: "child-cleanup-unverified" }; + } + if ( + !validChildRecord(persistedChild, childPath, registration) || + child?.childKey !== persistedChild.childKey || + child?.pidIdentity !== persistedChild.pidIdentity || + cleanupOutcome?.verified !== true || + (cleanupOutcome.survivors?.length ?? 0) > 0 || + (cleanupOutcome.survivorIdentities?.length ?? 0) > 0 + ) { + return { released: false, reason: "child-cleanup-unverified" }; + } + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "child-release", + brokerKey: registration.brokerKey, + childKey: child.childKey, + pid: child.pid, + pidIdentity: child.pidIdentity, + cleanupVerified: true, + releasedAt: now() + }; + const releasePath = path.join(registration.registryDir, "child-releases", `${child.childKey}.json`); + const locked = withBrokerRegistryLock(registration, options, () => { + if (fs.existsSync(releasePath)) { + const existing = readJson(releasePath); + if (!validChildReleaseRecord(existing, releasePath, registration, child)) { + const error = new Error(`Invalid existing broker child release at ${releasePath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + fs.chmodSync(releasePath, 0o600); + return { released: true, path: releasePath, release: existing }; + } + createImmutableJson(releasePath, payload); + return { released: true, path: releasePath, release: payload }; + }); + return locked.ok ? locked.value : { released: false, reason: locked.reason }; +} + +export function publishBrokerReaperReceipt( + registration, + { attemptId, decision, outcomes, residualIdentities, createdAt = new Date().toISOString() } +) { + if (!validRegistrationReference(registration)) { + return { published: false, reason: "broker-registration-unavailable" }; + } + if ( + typeof attemptId !== "string" || + !/^[a-zA-Z0-9._-]{1,128}$/.test(attemptId) || + typeof decision !== "string" || + !Array.isArray(outcomes) || + !Array.isArray(residualIdentities) + ) { + return { published: false, reason: "receipt-invalid" }; + } + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "reaper-receipt", + brokerKey: registration.brokerKey, + attemptId, + decision, + outcomes, + residualIdentities, + createdAt + }; + const receiptPath = path.join(registration.registryDir, "receipts", `${attemptId}.json`); + createImmutableJson(receiptPath, payload); + return { published: true, path: receiptPath, receipt: payload }; +} + +export function publishBrokerTerminal( + registration, + { attemptId, receiptPath, retiredAt = new Date().toISOString(), registryLock } +) { + if (!validRegistrationReference(registration)) { + return { terminal: false, reason: "broker-registration-unavailable" }; + } + const expectedReceiptPath = path.join(registration.registryDir, "receipts", `${attemptId}.json`); + if ( + typeof attemptId !== "string" || + !/^[a-zA-Z0-9._-]{1,128}$/.test(attemptId) || + receiptPath !== expectedReceiptPath || + typeof retiredAt !== "string" || + retiredAt.length === 0 + ) { + return { terminal: false, reason: "terminal-invalid" }; + } + let receipt; + try { + receipt = readJson(receiptPath); + } catch { + return { terminal: false, reason: "terminal-receipt-unavailable" }; + } + if ( + receipt?.version !== BROKER_OWNERSHIP_VERSION || + receipt.kind !== "reaper-receipt" || + receipt.brokerKey !== registration.brokerKey || + receipt.attemptId !== attemptId || + receipt.decision !== "cleanup-verified" || + !Array.isArray(receipt.residualIdentities) || + receipt.residualIdentities.length !== 0 + ) { + return { terminal: false, reason: "terminal-receipt-invalid" }; + } + const terminalPath = path.join(registration.registryDir, TERMINAL_FILE_NAME); + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "terminal", + brokerKey: registration.brokerKey, + pidIdentity: registration.broker.pidIdentity, + decision: "cleanup-verified", + attemptId, + receipt: path.basename(receiptPath), + retiredAt + }; + const locked = withBrokerRegistryLock(registration, { registryLock }, () => { + createImmutableJson(terminalPath, payload); + return { terminal: true, reason: "cleanup-verified", path: terminalPath, record: payload }; + }); + return locked.ok ? locked.value : { terminal: false, reason: locked.reason }; +} + +export function assessBrokerOwners(registration, { getLiveProcessPidsImpl = getLiveProcessPids } = {}) { + if (!validRegistrationReference(registration)) { + return { safeToShutdown: false, reason: "broker-registration-unavailable", liveOwners: [], deadOwners: [], releasedOwners: [], malformed: [] }; + } + + const malformed = []; + const owners = []; + const releases = new Map(); + let ownerFiles; + let releaseFiles; + try { + ownerFiles = listJsonFiles(path.join(registration.registryDir, "owners")); + releaseFiles = listJsonFiles(path.join(registration.registryDir, "releases")); + } catch { + return { + safeToShutdown: false, + reason: "malformed-registry", + liveOwners: [], + deadOwners: [], + releasedOwners: [], + malformed: [registration.registryDir] + }; + } + for (const filePath of ownerFiles) { + try { + const record = readJson(filePath); + if (!validOwnerRecord(record, filePath, registration)) { + malformed.push(filePath); + } else { + owners.push(record); + } + } catch { + malformed.push(filePath); + } + } + for (const filePath of releaseFiles) { + try { + const record = readJson(filePath); + if (!validReleaseRecord(record, filePath, registration)) { + malformed.push(filePath); + } else { + releases.set(record.ownerKey, record); + } + } catch { + malformed.push(filePath); + } + } + if (malformed.length > 0) { + return { safeToShutdown: false, reason: "malformed-registry", liveOwners: [], deadOwners: [], releasedOwners: [], malformed }; + } + if (owners.length === 0) { + return { safeToShutdown: false, reason: "no-registered-owner", liveOwners: [], deadOwners: [], releasedOwners: [], malformed: [] }; + } + + const liveOwners = []; + const deadOwners = []; + const releasedOwners = []; + for (const owner of owners) { + const release = releases.get(owner.ownerKey); + if (release) { + if ( + release.sessionId !== owner.sessionId || + release.pid !== owner.pid || + release.pidIdentity !== owner.pidIdentity + ) { + malformed.push(path.join(registration.registryDir, "releases", `${owner.ownerKey}.json`)); + continue; + } + releasedOwners.push(owner); + continue; + } + let live; + try { + live = getLiveProcessPidsImpl([owner.pid], { identities: [owner.pidIdentity] }); + } catch { + malformed.push(`liveness:${owner.ownerKey}`); + continue; + } + if (Array.isArray(live) && live.includes(owner.pid)) { + liveOwners.push(owner); + } else { + deadOwners.push(owner); + } + } + if (malformed.length > 0) { + return { safeToShutdown: false, reason: "owner-liveness-unavailable", liveOwners, deadOwners, releasedOwners, malformed }; + } + if (liveOwners.length > 0) { + return { safeToShutdown: false, reason: "live-owner", liveOwners, deadOwners, releasedOwners, malformed: [] }; + } + return { + safeToShutdown: true, + reason: "all-owners-dead-or-released", + liveOwners, + deadOwners, + releasedOwners, + malformed: [] + }; +} + +export function publishBrokerChild(registration, options = {}) { + const { ownershipSnapshot, now = () => new Date().toISOString() } = options; + if (!validRegistrationReference(registration)) { + return { registered: false, reason: "broker-registration-unavailable" }; + } + const pid = ownershipSnapshot?.rootPid; + const identity = ownershipSnapshot?.rootIdentity; + if ( + !isPidIdentity(pid, identity) || + !isSafePid(ownershipSnapshot?.processGroupId) || + (ownershipSnapshot?.sessionId != null && + (!isSafePid(ownershipSnapshot.sessionId) || ownershipSnapshot.sessionId !== pid)) || + !validOwnershipTree(ownershipSnapshot, pid, identity) + ) { + return { registered: false, reason: "child-identity-unavailable" }; + } + const locked = withBrokerRegistryLock(registration, options, () => { + const childKey = sha256(identity); + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "child", + brokerKey: registration.brokerKey, + childKey, + pid, + pidIdentity: identity, + processGroupId: ownershipSnapshot.processGroupId, + ownershipSnapshot, + registeredAt: now() + }; + const childPath = path.join(registration.registryDir, "children", `${childKey}.json`); + if (fs.existsSync(childPath)) { + const existing = readJson(childPath); + if ( + existing?.version !== BROKER_OWNERSHIP_VERSION || + existing.kind !== "child" || + existing.brokerKey !== registration.brokerKey || + existing.childKey !== childKey || + existing.pid !== pid || + existing.pidIdentity !== identity || + JSON.stringify(existing.ownershipSnapshot) !== JSON.stringify(ownershipSnapshot) + ) { + const error = new Error(`Invalid existing broker child row at ${childPath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + fs.chmodSync(childPath, 0o600); + return { registered: true, childKey, path: childPath, child: existing }; + } + createImmutableJson(childPath, payload); + return { registered: true, childKey, path: childPath, child: payload }; + }); + return locked.ok ? locked.value : { registered: false, reason: locked.reason }; +} + +export function publishBrokerChildObservation(registration, options = {}) { + const { + child, + ownershipSnapshot, + now = () => new Date().toISOString() + } = options; + if (!validRegistrationReference(registration)) { + return { observed: false, reason: "broker-registration-unavailable" }; + } + const childPath = path.join(registration.registryDir, "children", `${child?.childKey}.json`); + const locked = withBrokerRegistryLock(registration, options, () => { + const persistedChild = readJson(childPath); + if ( + !validChildRecord(persistedChild, childPath, registration) || + child?.childKey !== persistedChild.childKey || + child?.pidIdentity !== persistedChild.pidIdentity + ) { + return { observed: false, reason: "child-registration-invalid" }; + } + const observationKey = sha256(JSON.stringify(ownershipSnapshot)); + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "child-observation", + brokerKey: registration.brokerKey, + childKey: persistedChild.childKey, + observationKey, + ownershipSnapshot, + observedAt: now() + }; + const observationPath = path.join( + registration.registryDir, + "child-observations", + persistedChild.childKey, + `${observationKey}.json` + ); + if (fs.existsSync(observationPath)) { + const existing = readJson(observationPath); + if (!validChildObservation(existing, observationPath, registration, persistedChild)) { + return { observed: false, reason: "child-observation-invalid" }; + } + return { + observed: true, + reason: "child-already-observed", + path: observationPath, + observation: existing, + child: mergeChildOwnership(persistedChild, [existing]) + }; + } + if (!validChildObservation(payload, observationPath, registration, persistedChild)) { + return { observed: false, reason: "child-observation-invalid" }; + } + createImmutableJson(observationPath, payload); + const observedChild = mergeChildOwnership(persistedChild, [payload]); + return { + observed: true, + reason: "child-observed", + path: observationPath, + observation: payload, + child: observedChild + }; + }); + return locked.ok ? locked.value : { observed: false, reason: locked.reason }; +} diff --git a/plugins/codex/scripts/lib/codex.mjs b/plugins/codex/scripts/lib/codex.mjs index fead00cc4..5c9171c7b 100644 --- a/plugins/codex/scripts/lib/codex.mjs +++ b/plugins/codex/scripts/lib/codex.mjs @@ -20,6 +20,7 @@ * rejectCompletion: (error: unknown) => void, * finalTurn: Turn | null, * completed: boolean, + * inferredCompletion: boolean, * finalAnswerSeen: boolean, * pendingCollaborations: Set, * activeSubagentTurns: Set, @@ -40,8 +41,8 @@ import os from "node:os"; import path from "node:path"; import { readJsonFile } from "./fs.mjs"; -import { BROKER_BUSY_RPC_CODE, BROKER_ENDPOINT_ENV, CodexAppServerClient } from "./app-server.mjs"; -import { loadBrokerSession } from "./broker-lifecycle.mjs"; +import { BROKER_BUSY_RPC_CODE, BROKER_ENDPOINT_ENV, BROKER_OWNERSHIP_RPC_CODE, BROKER_STREAM_COMPLETED_METHOD, CodexAppServerClient } from "./app-server.mjs"; +import { loadReusableBrokerSession } from "./broker-lifecycle.mjs"; import { binaryAvailable } from "./process.mjs"; const SERVICE_NAME = "claude_code_codex_plugin"; @@ -321,6 +322,7 @@ function createTurnCaptureState(threadId, options = {}) { rejectCompletion, finalTurn: null, completed: false, + inferredCompletion: false, finalAnswerSeen: false, pendingCollaborations: new Set(), activeSubagentTurns: new Set(), @@ -364,6 +366,7 @@ function completeTurn(state, turn = null, options = {}) { } if (options.inferred) { + state.inferredCompletion = true; emitProgress(state.onProgress, "Turn completion inferred after the main thread finished and subagent work drained.", "finalizing"); } @@ -603,7 +606,14 @@ async function captureTurn(client, threadId, startRequest, options = {}) { completeTurn(state, response.turn); } - return await state.completion; + const completedState = await state.completion; + if (completedState.inferredCompletion && client.transport === "broker") { + client.notify(BROKER_STREAM_COMPLETED_METHOD, { + threadId: completedState.threadId, + turnId: completedState.turnId + }); + } + return completedState; } finally { clearCompletionTimer(state); client.setNotificationHandler(previousHandler ?? null); @@ -621,6 +631,7 @@ async function withAppServer(cwd, fn) { const brokerRequested = client?.transport === "broker" || Boolean(process.env[BROKER_ENDPOINT_ENV]); const shouldRetryDirect = (client?.transport === "broker" && error?.rpcCode === BROKER_BUSY_RPC_CODE) || + (client?.transport === "broker" && error?.rpcCode === BROKER_OWNERSHIP_RPC_CODE) || (brokerRequested && (error?.code === "ENOENT" || error?.code === "ECONNREFUSED")); if (client) { @@ -904,7 +915,7 @@ export function getCodexAvailability(cwd) { } export function getSessionRuntimeStatus(env = process.env, cwd = process.cwd()) { - const endpoint = env?.[BROKER_ENDPOINT_ENV] ?? loadBrokerSession(cwd)?.endpoint ?? null; + const endpoint = env?.[BROKER_ENDPOINT_ENV] ?? loadReusableBrokerSession(cwd, env)?.endpoint ?? null; if (endpoint) { return { mode: "shared", diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index dd8fc3751..a6a905fa7 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -54,14 +54,480 @@ function looksLikeMissingProcessMessage(text) { return /not found|no running instance|cannot find|does not exist|no such process/i.test(text); } -export function terminateProcessTree(pid, options = {}) { +const UNIX_PROCESS_TABLE_ARGS = ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]; +const UNIX_PS_COMMAND = "/bin/ps"; +const UNIX_PS_PATH_COMMAND = "ps"; + +function createProcessTableError(message) { + const error = new Error(message); + error.code = "PROCESS_TABLE_UNAVAILABLE"; + return error; +} + +function readUnixProcessTable(runCommandImpl, options = {}) { + let result = runCommandImpl(UNIX_PS_COMMAND, UNIX_PROCESS_TABLE_ARGS, { + cwd: options.cwd, + env: options.env + }); + if (result.error?.code === "ENOENT") { + result = runCommandImpl(UNIX_PS_PATH_COMMAND, UNIX_PROCESS_TABLE_ARGS, { + cwd: options.cwd, + env: options.env + }); + } + if (result.error || result.status !== 0) { + const detail = result.error?.message || result.stderr.trim() || result.stdout.trim() || `exit ${result.status}`; + throw createProcessTableError(`Unable to enumerate Unix processes: ${detail || `exit ${result.status}`}`); + } + + const processes = new Map(); + for (const line of result.stdout.split("\n")) { + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + const match = trimmedLine.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/); + const legacyMatch = match ? null : trimmedLine.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/); + const parsed = match ?? legacyMatch; + if (!parsed) { + throw createProcessTableError(`Unable to parse Unix process table line: ${trimmedLine}`); + } + const pid = Number(parsed[1]); + const parentPid = Number(parsed[2]); + const processGroupId = Number(parsed[3]); + const parsedSessionId = match ? Number(parsed[4]) : null; + // Darwin's `sess` is the security audit session, not the POSIX process + // session used for containment. Never treat it as a child-tree boundary. + const sessionId = + (options.platform ?? process.platform) !== "darwin" && + Number.isSafeInteger(parsedSessionId) && + parsedSessionId > 1 + ? parsedSessionId + : null; + const state = parsed[match ? 5 : 4]; + const startedAt = parsed[match ? 6 : 5].trim(); + if ( + !Number.isSafeInteger(pid) || + !Number.isSafeInteger(parentPid) || + !Number.isSafeInteger(processGroupId) || + !startedAt + ) { + throw createProcessTableError(`Unable to parse Unix process table line: ${trimmedLine}`); + } + processes.set(pid, { + pid, + parentPid, + processGroupId, + sessionId, + state, + startedAt, + identity: `${pid}@${startedAt}` + }); + } + return processes; +} + +function isRunningProcess(record) { + return record && !record.state.startsWith("Z"); +} + +function collectProcessTree(rootPid, processes, rootDepth = 0) { + const childrenByParent = new Map(); + for (const record of processes.values()) { + const children = childrenByParent.get(record.parentPid) ?? []; + children.push(record); + childrenByParent.set(record.parentPid, children); + } + + const records = []; + const visited = new Set(); + const visit = (parentPid, depth) => { + if (visited.has(parentPid)) { + return; + } + visited.add(parentPid); + for (const child of childrenByParent.get(parentPid) ?? []) { + visit(child.pid, depth + 1); + records.push({ ...child, depth: depth + 1 }); + } + }; + visit(rootPid, rootDepth); + const root = processes.get(rootPid); + if (root) { + records.push({ ...root, depth: rootDepth }); + } + return records; +} + +export function captureProcessOwnership(pid, options = {}) { + if (!Number.isFinite(pid) || (options.platform ?? process.platform) === "win32") { + return null; + } + + const processes = readUnixProcessTable(options.runCommandImpl ?? runCommand, options); + const root = processes.get(pid); + if (!root) { + return null; + } + + return { + rootPid: root.pid, + rootIdentity: root.identity, + processGroupId: root.processGroupId, + sessionId: root.sessionId, + members: collectProcessTree(pid, processes).map((record) => ({ ...record })) + }; +} + +export function captureStableSessionOwner(pid = process.pid, options = {}) { + if (!Number.isFinite(pid) || (options.platform ?? process.platform) === "win32") { + return null; + } + const processes = readUnixProcessTable(options.runCommandImpl ?? runCommand, options); + const current = processes.get(pid); + if (!isRunningProcess(current) || current.processGroupId === pid) { + return null; + } + const leader = processes.get(current.processGroupId); + if (!isRunningProcess(leader) || leader.pid !== leader.processGroupId) { + return null; + } + return { + pid: leader.pid, + identity: leader.identity, + processGroupId: leader.processGroupId + }; +} + +function recordsFromOwnershipSnapshot(snapshot) { + return (snapshot?.members ?? []).filter((record) => { + return Number.isFinite(record.pid) && typeof record.identity === "string" && record.identity.length > 0; + }); +} + +function identitiesByPid(identities) { + const result = new Map(); + for (const identity of identities ?? []) { + const match = String(identity).match(/^(\d+)@/); + if (match) { + result.set(Number(match[1]), String(identity)); + } + } + return result; +} + +export function normalizeProcessCleanupOutcome(outcome = {}) { + return { + attempted: Boolean(outcome.attempted), + delivered: Boolean(outcome.delivered), + verified: outcome.verified === true, + degraded: Boolean(outcome.degraded), + method: outcome.method ?? null, + escalated: Boolean(outcome.escalated), + targets: Array.isArray(outcome.targets) ? outcome.targets : [], + targetIdentities: Array.isArray(outcome.targetIdentities) ? outcome.targetIdentities : [], + survivors: Array.isArray(outcome.survivors) ? outcome.survivors : [], + survivorIdentities: Array.isArray(outcome.survivorIdentities) ? outcome.survivorIdentities : [], + ...(outcome.identityMismatch ? { identityMismatch: true } : {}) + }; +} + +function sleep(milliseconds) { + if (milliseconds <= 0) { + return Promise.resolve(); + } + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +export function getProcessIdentity(pid, options = {}) { + if (!Number.isFinite(pid) || (options.platform ?? process.platform) === "win32") { + return null; + } + return readUnixProcessTable(options.runCommandImpl ?? runCommand, options).get(pid)?.identity ?? null; +} + +export function hasLiveProcessIdentity(pid, identity, options = {}) { + if ( + !Number.isFinite(pid) || + typeof identity !== "string" || + !identity || + (options.platform ?? process.platform) === "win32" + ) { + return false; + } + const current = readUnixProcessTable(options.runCommandImpl ?? runCommand, options).get(pid); + return isRunningProcess(current) && current.identity === identity; +} + +export function getLiveProcessPids(pids, options = {}) { + const candidates = [...new Set((pids ?? []).filter((pid) => Number.isFinite(pid)))]; + const expectedIdentities = identitiesByPid(options.identities); + if (candidates.length === 0) { + return []; + } + if ((options.platform ?? process.platform) === "win32") { + const killImpl = options.killImpl ?? process.kill.bind(process); + return candidates.filter((pid) => { + try { + killImpl(pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } + }); + } + + try { + const processes = readUnixProcessTable(options.runCommandImpl ?? runCommand, options); + return candidates.filter((pid) => { + const current = processes.get(pid); + return isRunningProcess(current) && (!expectedIdentities.has(pid) || expectedIdentities.get(pid) === current.identity); + }); + } catch { + // An unverified cleanup remains blocked when liveness cannot be checked. + return candidates; + } +} + +function mergeTrackedDescendants(tracked, processes, rootPid, rootIdentity) { + const trackedByPid = new Map([...tracked.values()].map((record) => [record.pid, record.identity])); + const roots = []; + const currentRoot = processes.get(rootPid); + if (currentRoot?.identity === rootIdentity) { + roots.push({ pid: rootPid, depth: 0 }); + } + for (const trackedRecord of tracked.values()) { + const current = processes.get(trackedRecord.pid); + if (current?.identity === trackedRecord.identity) { + roots.push({ pid: current.pid, depth: trackedRecord.depth }); + } + } + for (const root of roots) { + const records = collectProcessTree(root.pid, processes, root.depth).sort((left, right) => left.depth - right.depth); + for (const record of records) { + const priorIdentity = trackedByPid.get(record.pid); + if (priorIdentity && priorIdentity !== record.identity) { + continue; + } + if (record.pid !== root.pid) { + const parentIdentity = trackedByPid.get(record.parentPid); + if (!parentIdentity || processes.get(record.parentPid)?.identity !== parentIdentity) { + continue; + } + } + if (!tracked.has(record.identity)) { + tracked.set(record.identity, record); + trackedByPid.set(record.pid, record.identity); + } + } + } +} + +function listLiveTracked(tracked, processes) { + return [...tracked.values()].filter((record) => { + const current = processes.get(record.pid); + return current?.identity === record.identity && isRunningProcess(current); + }); +} + +function buildSignalUnits(records) { + const groupLeaders = new Map(); + for (const record of records) { + if (record.pid === record.processGroupId) { + groupLeaders.set(record.processGroupId, record); + } + } + + const units = []; + for (const record of records) { + const groupLeader = groupLeaders.get(record.processGroupId); + if (groupLeader && groupLeader.identity !== record.identity) { + continue; + } + units.push({ + record, + group: record.pid === record.processGroupId + }); + } + units.sort((left, right) => right.record.depth - left.record.depth); + return units; +} + +function mergeTrackedGroupMembers(tracked, processes, units) { + const trackedByPid = new Map([...tracked.values()].map((record) => [record.pid, record.identity])); + for (const unit of units) { + if (!unit.group) { + continue; + } + for (const record of processes.values()) { + if (record.processGroupId !== unit.record.processGroupId) { + continue; + } + const priorIdentity = trackedByPid.get(record.pid); + if (priorIdentity && priorIdentity !== record.identity) { + continue; + } + if (!tracked.has(record.identity)) { + tracked.set(record.identity, { ...record, depth: unit.record.depth + 1 }); + trackedByPid.set(record.pid, record.identity); + } + } + } +} + +function signalVerifiedUnit(unit, signal, processes, killImpl) { + const current = processes.get(unit.record.pid); + if (current?.identity !== unit.record.identity || !isRunningProcess(current)) { + return false; + } + if (unit.group && current.processGroupId !== current.pid) { + return false; + } + + const target = unit.group ? -current.pid : current.pid; + try { + killImpl(target, signal); + return true; + } catch (error) { + if (error?.code === "ESRCH" || error?.code === "EPERM") { + // On macOS a process-group signal can race the group leader's exit and + // return EPERM. Never infer success from the errno; the caller's fresh + // process-table poll decides whether cleanup is verified. + return false; + } + if (error instanceof Error) { + error.message = `Unable to signal owned process ${unit.record.identity} with ${signal} via target ${target}: ${error.message}`; + } + throw error; + } +} + +function withoutExcluded(records, excludePids) { + if (!excludePids || excludePids.size === 0) { + return records; + } + return records.filter((record) => !excludePids.has(record.pid)); +} + +function signalTracked(tracked, signal, options) { + const processes = readUnixProcessTable(options.runCommandImpl, options); + mergeTrackedDescendants(tracked, processes, options.rootPid, options.rootIdentity); + mergeTrackedGroupMembers(tracked, processes, buildSignalUnits(listLiveTracked(tracked, processes))); + const units = buildSignalUnits(withoutExcluded(listLiveTracked(tracked, processes), options.excludePids)); + let delivered = false; + for (const unit of units) { + delivered = signalVerifiedUnit(unit, signal, processes, options.killImpl) || delivered; + } + return delivered; +} + +async function pollTracked(tracked, options, attempts) { + let live = []; + for (let attempt = 0; attempt < attempts; attempt += 1) { + const processes = readUnixProcessTable(options.runCommandImpl, options); + mergeTrackedDescendants(tracked, processes, options.rootPid, options.rootIdentity); + live = withoutExcluded(listLiveTracked(tracked, processes), options.excludePids); + if (live.length === 0) { + break; + } + if (attempt + 1 < attempts) { + await options.sleepImpl(options.pollIntervalMs); + } + } + return live; +} + +function warnProcessCleanup(message, options) { + const warnImpl = options.warnImpl ?? ((warning) => process.stderr.write(`${warning}\n`)); + try { + warnImpl(message); + } catch { + // Cleanup warnings must not turn a best-effort kill into a host failure. + } +} + +function degradedDirectChildKill(pid, options, killImpl, reason) { + const ownershipSnapshot = options.ownershipSnapshot ?? null; + const ownerHoldsLiveHandle = options.ownerHoldsLiveHandle === true; + const canSignalOwnedGroup = + ownerHoldsLiveHandle && + ownershipSnapshot?.rootPid === pid && + ownershipSnapshot?.processGroupId === pid && + ownershipSnapshot?.rootIdentity === (options.expectedRootIdentity ?? ownershipSnapshot?.rootIdentity); + if (!ownerHoldsLiveHandle) { + warnProcessCleanup( + `Unable to verify Unix process cleanup for PID ${pid}; deferred signalling until process enumeration recovers (${String(reason).replace(/\s+/g, " ").trim()}). Surviving PIDs: ${pid}.`, + options + ); + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified: false, + escalated: false, + degraded: true, + method: "deferred", + targets: [], + targetIdentities: options.expectedRootIdentity ? [options.expectedRootIdentity] : [], + survivors: [pid], + survivorIdentities: options.expectedRootIdentity ? [options.expectedRootIdentity] : [] + }); + } + const directKillImpl = options.directKillImpl ?? ((signal) => killImpl(pid, signal)); + let delivered = false; + try { + delivered = canSignalOwnedGroup + ? killImpl(-pid, "SIGKILL") !== false + : directKillImpl("SIGKILL") !== false; + } catch { + // The direct child may already have exited. + } + warnProcessCleanup( + `Unable to verify Unix process cleanup for PID ${pid}; used ${canSignalOwnedGroup ? "process-group" : "direct-child"} kill fallback (${String(reason).replace(/\s+/g, " ").trim()}). Surviving PIDs: none known.`, + options + ); + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered, + verified: false, + escalated: false, + degraded: true, + method: canSignalOwnedGroup ? "process-group" : "direct-child", + targets: canSignalOwnedGroup + ? recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.pid) + : [pid], + targetIdentities: canSignalOwnedGroup + ? recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.identity) + : options.expectedRootIdentity ? [options.expectedRootIdentity] : [], + survivors: [], + survivorIdentities: [] + }); +} + +export async function terminateProcessTree(pid, options = {}) { if (!Number.isFinite(pid)) { - return { attempted: false, delivered: false, method: null }; + const ownershipSnapshot = options.ownershipSnapshot ?? null; + if (Number.isFinite(ownershipSnapshot?.rootPid)) { + return terminateProcessTree(ownershipSnapshot.rootPid, options); + } + const ownershipEstablished = Boolean(options.expectedRootIdentity || options.requireVerifiedOwnership); + return normalizeProcessCleanupOutcome({ + attempted: false, + delivered: false, + verified: !ownershipEstablished, + degraded: ownershipEstablished, + method: null + }); } const platform = options.platform ?? process.platform; const runCommandImpl = options.runCommandImpl ?? runCommand; const killImpl = options.killImpl ?? process.kill.bind(process); + const ownershipSnapshot = options.ownershipSnapshot ?? null; + const expectedRootIdentity = options.expectedRootIdentity ?? ownershipSnapshot?.rootIdentity ?? null; + const ownershipCaptureFailed = options.requireVerifiedOwnership === true; + const captureFailureCleanupAllowed = + ownershipCaptureFailed && options.ownerHoldsLiveHandle === true; + const ownershipEstablished = Boolean(ownershipSnapshot || expectedRootIdentity || ownershipCaptureFailed); if (platform === "win32") { const result = runCommandImpl("taskkill", ["/PID", String(pid), "/T", "/F"], { @@ -70,21 +536,21 @@ export function terminateProcessTree(pid, options = {}) { }); if (!result.error && result.status === 0) { - return { attempted: true, delivered: true, method: "taskkill", result }; + return { ...normalizeProcessCleanupOutcome({ attempted: true, delivered: true, verified: true, method: "taskkill" }), result }; } const combinedOutput = `${result.stderr}\n${result.stdout}`.trim(); if (!result.error && looksLikeMissingProcessMessage(combinedOutput)) { - return { attempted: true, delivered: false, method: "taskkill", result }; + return { ...normalizeProcessCleanupOutcome({ attempted: true, delivered: false, verified: true, method: "taskkill" }), result }; } if (result.error?.code === "ENOENT") { try { killImpl(pid); - return { attempted: true, delivered: true, method: "kill" }; + return normalizeProcessCleanupOutcome({ attempted: true, delivered: true, verified: true, method: "kill" }); } catch (error) { if (error?.code === "ESRCH") { - return { attempted: true, delivered: false, method: "kill" }; + return normalizeProcessCleanupOutcome({ attempted: true, delivered: false, verified: true, method: "kill" }); } throw error; } @@ -97,23 +563,293 @@ export function terminateProcessTree(pid, options = {}) { throw new Error(formatCommandFailure(result)); } + let initialProcesses; try { - killImpl(-pid, "SIGTERM"); - return { attempted: true, delivered: true, method: "process-group" }; + initialProcesses = readUnixProcessTable(runCommandImpl, options); } catch (error) { - if (error?.code !== "ESRCH") { - try { - killImpl(pid, "SIGTERM"); - return { attempted: true, delivered: true, method: "process" }; - } catch (innerError) { - if (innerError?.code === "ESRCH") { - return { attempted: true, delivered: false, method: "process" }; - } - throw innerError; + if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { + throw error; + } + if (!expectedRootIdentity && !captureFailureCleanupAllowed) { + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified: false, + degraded: true, + method: "process-tree", + targets: [], + survivors: [pid], + survivorIdentities: [] + }); + } + return degradedDirectChildKill(pid, options, killImpl, error.message); + } + const root = initialProcesses.get(pid); + if (!expectedRootIdentity && !captureFailureCleanupAllowed) { + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified: false, + degraded: true, + method: "process-tree", + targets: [], + survivors: [pid], + survivorIdentities: [] + }); + } + if (root && expectedRootIdentity && root.identity !== expectedRootIdentity) { + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified: false, + degraded: ownershipEstablished, + identityMismatch: true, + method: "process-tree", + targets: [] + }); + } + + const tracked = new Map(); + for (const record of recordsFromOwnershipSnapshot(ownershipSnapshot)) { + tracked.set(record.identity, record); + } + for (const record of root ? collectProcessTree(pid, initialProcesses) : []) { + tracked.set(record.identity, record); + } + const sortedTracked = () => + [...tracked.values()].sort((left, right) => right.depth - left.depth); + if (!root && listLiveTracked(tracked, initialProcesses).length === 0) { + const degradedWithoutDurableSnapshot = options.priorCleanupDegraded === true && recordsFromOwnershipSnapshot(ownershipSnapshot).length === 0; + const verified = !options.requireVerifiedOwnership && ownershipEstablished && !degradedWithoutDurableSnapshot; + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified, + degraded: ownershipEstablished && !verified, + method: "process-tree", + targets: sortedTracked().map((record) => record.pid), + targetIdentities: sortedTracked().map((record) => record.identity) + }); + } + let cleanupRootIdentity = expectedRootIdentity; + if (!cleanupRootIdentity && captureFailureCleanupAllowed && root) { + // A live, unreaped child handle prevents PID reuse. Its owner may use the + // current identity for best-effort cleanup, but the result stays unverified. + cleanupRootIdentity = root.identity; + } + const unixOptions = { + rootPid: pid, + rootIdentity: cleanupRootIdentity, + runCommandImpl, + killImpl, + cwd: options.cwd, + env: options.env, + sleepImpl: options.sleepImpl ?? sleep, + pollIntervalMs: options.pollIntervalMs ?? 25 + }; + + try { + let delivered = false; + let escalated = false; + let live = []; + // Terminate descendants first and wait for them while their parent is + // still alive, so the parent can reap them; killing the whole tree + // back-to-back leaves permanent zombies where PID 1 does not reap orphans + // (containers). Both phases share the one tracked map so PID-reuse + // identity memory carries across phases; the descendant phase only + // excludes the root pid from signaling and liveness. + const hasDescendants = [...tracked.values()].some((record) => record.pid !== pid); + const phases = hasDescendants + ? [{ ...unixOptions, excludePids: new Set([pid]) }, unixOptions] + : [unixOptions]; + for (const phaseOptions of phases) { + delivered = signalTracked(tracked, "SIGTERM", phaseOptions) || delivered; + let phaseLive = await pollTracked(tracked, phaseOptions, options.termPollAttempts ?? 11); + if (phaseLive.length > 0) { + escalated = true; + delivered = signalTracked(tracked, "SIGKILL", phaseOptions) || delivered; + phaseLive = await pollTracked(tracked, phaseOptions, options.killPollAttempts ?? 11); } + live = phaseLive; } - return { attempted: true, delivered: false, method: "process-group" }; + const verified = + live.length === 0 && + !options.requireVerifiedOwnership && + ownershipEstablished; + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered, + verified, + degraded: ownershipEstablished && !verified, + escalated, + method: "process-tree", + // The algorithm covers same-process-group descendants plus those observed at scan time. + // A post-scan setsid descendant can escape the tracked process tree. + targets: sortedTracked().map((record) => record.pid), + targetIdentities: sortedTracked().map((record) => record.identity), + survivors: live.map((record) => record.pid), + survivorIdentities: live.map((record) => record.identity) + }); + } catch (error) { + if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { + throw error; + } + return degradedDirectChildKill(pid, options, killImpl, error.message); + } +} + +export async function terminateProcessGroup(pgid, options = {}) { + if (!Number.isFinite(pgid)) { + return normalizeProcessCleanupOutcome({ attempted: false, delivered: false, verified: false, degraded: true }); + } + if ((options.platform ?? process.platform) === "win32") { + return normalizeProcessCleanupOutcome({ attempted: false, delivered: false, verified: false, degraded: true }); + } + + const runCommandImpl = options.runCommandImpl ?? runCommand; + const killImpl = options.killImpl ?? process.kill.bind(process); + + let processes; + try { + processes = readUnixProcessTable(runCommandImpl, options); + } catch (error) { + if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { + throw error; + } + warnProcessCleanup( + `Unable to enumerate Unix processes while reclaiming process group ${pgid}; surviving PIDs unknown.`, + options + ); + return normalizeProcessCleanupOutcome({ + attempted: false, + delivered: false, + verified: false, + degraded: true, + survivors: recordsFromOwnershipSnapshot(options.ownershipSnapshot).map((record) => record.pid), + survivorIdentities: recordsFromOwnershipSnapshot(options.ownershipSnapshot).map((record) => record.identity) + }); + } + + const tracked = new Map(); + const ownershipSnapshot = options.ownershipSnapshot ?? null; + const ownershipEstablished = Boolean(ownershipSnapshot); + const ownershipSessionId = + Number.isSafeInteger(ownershipSnapshot?.sessionId) && + ownershipSnapshot.sessionId > 1 && + ownershipSnapshot.rootPid === ownershipSnapshot.sessionId + ? ownershipSnapshot.sessionId + : null; + const cleanupMethod = ownershipSessionId ? "process-session" : "process-group"; + // If a live process holds this group id but is not the root we recorded, the + // id has been reused and nothing in the group is ours. Refuse before + // admitting any member: members of an unrelated group are absent from the + // snapshot, so the per-record check below would admit them, and the leader + // identity check runs only after signals have already been delivered. + // A missing leader is not a mismatch — that is the crashed-root case where + // surviving helpers still need reclaiming. + const groupLeader = processes.get(pgid); + if (ownershipSnapshot && groupLeader && groupLeader.identity !== ownershipSnapshot.rootIdentity) { + return normalizeProcessCleanupOutcome({ + attempted: false, + delivered: false, + verified: false, + degraded: true, + identityMismatch: true, + method: cleanupMethod, + survivors: recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.pid), + survivorIdentities: recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.identity) + }); + } + const sessionLeader = ownershipSessionId ? processes.get(ownershipSessionId) : null; + if (ownershipSessionId && sessionLeader && sessionLeader.identity !== ownershipSnapshot.rootIdentity) { + return normalizeProcessCleanupOutcome({ + attempted: false, + delivered: false, + verified: false, + degraded: true, + identityMismatch: true, + method: cleanupMethod, + survivors: recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.pid), + survivorIdentities: recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.identity) + }); + } + for (const record of recordsFromOwnershipSnapshot(ownershipSnapshot)) { + tracked.set(record.identity, record); + } + let ownershipScopeMembersFound = false; + let ownershipSelectionFound = false; + for (const record of processes.values()) { + const inOwnedScope = + record.processGroupId === pgid || + (ownershipSessionId !== null && record.sessionId === ownershipSessionId); + if (!inOwnedScope || !isRunningProcess(record)) { + continue; + } + ownershipScopeMembersFound = true; + const snapshotRecord = recordsFromOwnershipSnapshot(ownershipSnapshot).find((candidate) => candidate.pid === record.pid); + if (ownershipSnapshot && snapshotRecord && snapshotRecord.identity !== record.identity) { + continue; + } + ownershipSelectionFound = true; + tracked.set(record.identity, { + ...record, + depth: Number.isSafeInteger(record.depth) ? record.depth : record.pid === pgid ? 0 : 1 + }); + } + if (tracked.size === 0) { + return normalizeProcessCleanupOutcome({ + attempted: false, + delivered: false, + verified: !ownershipEstablished, + degraded: ownershipEstablished, + survivors: [], + survivorIdentities: [] + }); + } + + const unixOptions = { + runCommandImpl, + killImpl, + cwd: options.cwd, + env: options.env, + sleepImpl: options.sleepImpl ?? sleep, + pollIntervalMs: options.pollIntervalMs ?? 25 + }; + + try { + let delivered = signalTracked(tracked, "SIGTERM", unixOptions); + let live = await pollTracked(tracked, unixOptions, options.termPollAttempts ?? 11); + let escalated = false; + if (live.length > 0) { + escalated = true; + delivered = signalTracked(tracked, "SIGKILL", unixOptions) || delivered; + live = await pollTracked(tracked, unixOptions, options.killPollAttempts ?? 11); + } + const root = processes.get(pgid); + const rootIdentityMatches = !root || root.identity === ownershipSnapshot?.rootIdentity; + const ownedScopeAccountedFor = ownershipSelectionFound || !ownershipScopeMembersFound; + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered, + verified: live.length === 0 && (!ownershipEstablished || (rootIdentityMatches && ownedScopeAccountedFor)), + degraded: ownershipEstablished && (!rootIdentityMatches || !ownedScopeAccountedFor), + escalated, + method: cleanupMethod, + targets: [...tracked.values()].map((record) => record.pid), + targetIdentities: [...tracked.values()].map((record) => record.identity), + survivors: live.map((record) => record.pid), + survivorIdentities: live.map((record) => record.identity) + }); + } catch (error) { + if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { + throw error; + } + warnProcessCleanup( + `Unable to verify Unix process-group cleanup for pgid ${pgid}; surviving PIDs unknown.`, + options + ); + return normalizeProcessCleanupOutcome({ attempted: true, delivered: true, verified: false, degraded: true, survivors: [], survivorIdentities: [] }); } } diff --git a/plugins/codex/scripts/lib/state.mjs b/plugins/codex/scripts/lib/state.mjs index 2da23498f..63329e4e7 100644 --- a/plugins/codex/scripts/lib/state.mjs +++ b/plugins/codex/scripts/lib/state.mjs @@ -107,7 +107,7 @@ export function saveState(cwd, state) { if (retainedIds.has(job.id)) { continue; } - removeJobFile(resolveJobFile(cwd, job.id)); + removeJobFile(cwd, job.id, { preserveCancelFlag: hasCancelFlag(cwd, job.id) }); removeFileIfExists(job.logFile); } @@ -174,9 +174,30 @@ export function readJobFile(jobFile) { return JSON.parse(fs.readFileSync(jobFile, "utf8")); } -function removeJobFile(jobFile) { - if (fs.existsSync(jobFile)) { - fs.unlinkSync(jobFile); +export function writeCancelFlag(cwd, jobId) { + const cancelFlag = resolveCancelFlag(cwd, jobId); + try { + fs.writeFileSync(cancelFlag, "", { flag: "wx" }); + } catch (error) { + if (error?.code !== "EEXIST") { + throw error; + } + } + return cancelFlag; +} + +export function hasCancelFlag(cwd, jobId) { + return fs.existsSync(resolveCancelFlag(cwd, jobId)); +} + +export function removeCancelFlag(cwd, jobId) { + removeFileIfExists(resolveCancelFlag(cwd, jobId)); +} + +function removeJobFile(cwd, jobId, options = {}) { + removeFileIfExists(resolveJobFile(cwd, jobId)); + if (options.preserveCancelFlag !== true) { + removeCancelFlag(cwd, jobId); } } @@ -189,3 +210,7 @@ export function resolveJobFile(cwd, jobId) { ensureStateDir(cwd); return path.join(resolveJobsDir(cwd), `${jobId}.json`); } + +function resolveCancelFlag(cwd, jobId) { + return resolveJobFile(cwd, jobId).replace(/\.json$/, ".cancelled"); +} diff --git a/plugins/codex/scripts/lib/tracked-jobs.mjs b/plugins/codex/scripts/lib/tracked-jobs.mjs index 902869012..995ede1bc 100644 --- a/plugins/codex/scripts/lib/tracked-jobs.mjs +++ b/plugins/codex/scripts/lib/tracked-jobs.mjs @@ -1,9 +1,10 @@ import fs from "node:fs"; import process from "node:process"; -import { readJobFile, resolveJobFile, resolveJobLogFile, upsertJob, writeJobFile } from "./state.mjs"; +import { hasCancelFlag, readJobFile, removeCancelFlag, resolveJobFile, resolveJobLogFile, upsertJob, writeJobFile } from "./state.mjs"; export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; +const JOB_CANCELLED_CODE = "JOB_CANCELLED"; export function nowIso() { return new Date().toISOString(); @@ -139,6 +140,13 @@ function readStoredJobOrNull(workspaceRoot, jobId) { return readJobFile(jobFile); } +function createJobCancelledError(jobId) { + const error = new Error(`Job ${jobId} was cancelled before execution.`); + error.name = "JobCancelledError"; + error.code = JOB_CANCELLED_CODE; + return error; +} + export async function runTrackedJob(job, runner, options = {}) { const runningRecord = { ...job, @@ -152,6 +160,9 @@ export async function runTrackedJob(job, runner, options = {}) { upsertJob(job.workspaceRoot, runningRecord); try { + if (hasCancelFlag(job.workspaceRoot, job.id)) { + throw createJobCancelledError(job.id); + } const execution = await runner(); const completionStatus = execution.exitStatus === 0 ? "completed" : "failed"; const completedAt = nowIso(); @@ -182,10 +193,11 @@ export async function runTrackedJob(job, runner, options = {}) { const errorMessage = error instanceof Error ? error.message : String(error); const existing = readStoredJobOrNull(job.workspaceRoot, job.id) ?? runningRecord; const completedAt = nowIso(); + const terminalStatus = error?.code === JOB_CANCELLED_CODE ? "cancelled" : "failed"; writeJobFile(job.workspaceRoot, job.id, { ...existing, - status: "failed", - phase: "failed", + status: terminalStatus, + phase: terminalStatus, errorMessage, pid: null, completedAt, @@ -193,12 +205,18 @@ export async function runTrackedJob(job, runner, options = {}) { }); upsertJob(job.workspaceRoot, { id: job.id, - status: "failed", - phase: "failed", + status: terminalStatus, + phase: terminalStatus, pid: null, errorMessage, completedAt }); + if (terminalStatus === "cancelled") { + // The worker has now durably acknowledged the cancellation. Until this + // point the tombstone must survive state pruning so a worker that read + // the queued request cannot cross the read-to-start race. + removeCancelFlag(job.workspaceRoot, job.id); + } throw error; } } diff --git a/plugins/codex/scripts/registered-broker-reaper.mjs b/plugins/codex/scripts/registered-broker-reaper.mjs new file mode 100644 index 000000000..53777c1cf --- /dev/null +++ b/plugins/codex/scripts/registered-broker-reaper.mjs @@ -0,0 +1,334 @@ +#!/usr/bin/env node + +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; + +import { + acquireBrokerRegistryLock, + assessBrokerOwners, + loadBrokerChildren, + loadBrokerRegistration, + loadBrokerTerminal, + publishBrokerReaperReceipt, + publishBrokerTerminal, + releaseBrokerChild, + releaseBrokerRegistryLock, + resolveBrokerOwnershipRoot +} from "./lib/broker-ownership.mjs"; +import { + getLiveProcessPids, + terminateProcessGroup, + terminateProcessTree +} from "./lib/process.mjs"; + +const APPLY_MODE = "apply-registered"; +const REPORT_MODE = "report-only"; + +function listRegistrationCandidates(env) { + const registryRoot = resolveBrokerOwnershipRoot(env); + if (!registryRoot || !fs.existsSync(registryRoot)) { + return []; + } + try { + const rootStat = fs.lstatSync(registryRoot); + if (rootStat.isSymbolicLink() || !rootStat.isDirectory() || (rootStat.mode & 0o777) !== 0o700) { + return [{ valid: false, brokerKey: null, registryDir: registryRoot, reason: "registry-root-invalid" }]; + } + } catch { + return [{ valid: false, brokerKey: null, registryDir: registryRoot, reason: "registry-root-invalid" }]; + } + const candidates = []; + for (const entry of fs.readdirSync(registryRoot, { withFileTypes: true })) { + if (!/^[a-f0-9]{64}$/.test(entry.name)) { + continue; + } + const registryDir = path.join(registryRoot, entry.name); + if (!entry.isDirectory()) { + candidates.push({ valid: false, brokerKey: entry.name, registryDir, reason: "broker-directory-invalid" }); + continue; + } + const brokerPath = path.join(registryDir, "broker.json"); + try { + const brokerStat = fs.lstatSync(brokerPath); + if (brokerStat.isSymbolicLink() || !brokerStat.isFile() || (brokerStat.mode & 0o777) !== 0o600) { + throw new Error("broker record is not a private regular file"); + } + const broker = JSON.parse(fs.readFileSync(brokerPath, "utf8")); + const registration = loadBrokerRegistration({ + endpoint: broker?.endpoint, + brokerIdentity: broker?.pidIdentity, + env + }); + if ( + registration.registered !== true || + registration.brokerKey !== entry.name || + registration.registryDir !== registryDir + ) { + candidates.push({ valid: false, brokerKey: entry.name, registryDir, reason: "broker-record-invalid" }); + } else { + const terminal = loadBrokerTerminal(registration); + if (terminal.terminal === true) { + continue; + } + if (terminal.reason !== "terminal-absent") { + candidates.push({ valid: false, brokerKey: entry.name, registryDir, reason: terminal.reason }); + } else { + candidates.push({ valid: true, registration }); + } + } + } catch { + candidates.push({ valid: false, brokerKey: entry.name, registryDir, reason: "broker-record-invalid" }); + } + } + return candidates; +} + +function targetRecords(registration, children) { + const broker = registration.broker; + const byIdentity = new Map([[broker.pidIdentity, { pid: broker.pid, identity: broker.pidIdentity }]]); + for (const child of children) { + for (const member of child.ownershipSnapshot.members) { + byIdentity.set(member.identity, { pid: member.pid, identity: member.identity }); + } + } + return [...byIdentity.values()]; +} + +function brokerOwnershipSnapshot(broker) { + const startedAt = broker.pidIdentity.slice(String(broker.pid).length + 1); + return { + rootPid: broker.pid, + rootIdentity: broker.pidIdentity, + processGroupId: broker.processGroupId, + members: [ + { + pid: broker.pid, + parentPid: 1, + processGroupId: broker.processGroupId, + state: "registered", + startedAt, + identity: broker.pidIdentity, + depth: 0 + } + ] + }; +} + +function refusal(candidate, reason, extra = {}) { + return { + status: "report-only", + brokerKey: candidate.registration?.brokerKey ?? candidate.brokerKey ?? null, + pid: candidate.registration?.broker?.pid ?? null, + reason, + ...extra + }; +} + +async function processRegistration(candidate, options) { + if (!candidate.valid) { + return refusal(candidate, candidate.reason); + } + + const registration = candidate.registration; + const getLive = options.getLiveProcessPidsImpl ?? getLiveProcessPids; + const assess = options.assessBrokerOwnersImpl ?? assessBrokerOwners; + const initialAssessment = assess(registration, { getLiveProcessPidsImpl: getLive }); + if (initialAssessment.safeToShutdown !== true) { + return refusal(candidate, initialAssessment.reason, { assessment: initialAssessment }); + } + const initialChildren = loadBrokerChildren(registration); + if (initialChildren.valid !== true) { + return refusal(candidate, initialChildren.reason, { malformed: initialChildren.malformed }); + } + if (options.mode !== APPLY_MODE) { + return refusal(candidate, "eligible-but-apply-not-enabled", { assessment: initialAssessment }); + } + + const acquireLock = options.acquireBrokerRegistryLockImpl ?? acquireBrokerRegistryLock; + const releaseLock = options.releaseBrokerRegistryLockImpl ?? releaseBrokerRegistryLock; + const registryLock = acquireLock(registration); + if (registryLock?.acquired !== true) { + return refusal(candidate, registryLock?.reason ?? "registry-busy"); + } + + const outcomes = []; + let decision = "cleanup-unverified"; + let residualIdentities = []; + let result; + try { + const lockedRegistration = loadBrokerRegistration({ + endpoint: registration.broker.endpoint, + brokerIdentity: registration.broker.pidIdentity, + env: options.env ?? process.env + }); + if ( + lockedRegistration.registered !== true || + lockedRegistration.brokerKey !== registration.brokerKey || + lockedRegistration.registryDir !== registration.registryDir + ) { + result = refusal(candidate, "locked-broker-record-invalid"); + } else { + const finalAssessment = assess(lockedRegistration, { getLiveProcessPidsImpl: getLive }); + if (finalAssessment.safeToShutdown !== true) { + result = refusal(candidate, `locked-${finalAssessment.reason}`, { assessment: finalAssessment }); + } else { + const children = loadBrokerChildren(lockedRegistration); + if (children.valid !== true) { + result = refusal(candidate, children.reason, { malformed: children.malformed }); + } else { + const terminateGroup = options.terminateProcessGroupImpl ?? terminateProcessGroup; + const terminateTree = options.terminateProcessTreeImpl ?? terminateProcessTree; + let cleanupVerified = true; + for (const child of children.children) { + const outcome = await terminateGroup(child.processGroupId, { + ownershipSnapshot: child.ownershipSnapshot, + termPollAttempts: options.termPollAttempts, + killPollAttempts: options.killPollAttempts, + pollIntervalMs: options.pollIntervalMs + }); + if (outcome?.verified !== true) { + outcomes.push({ target: "child", pid: child.pid, pidIdentity: child.pidIdentity, outcome }); + cleanupVerified = false; + break; + } + let childRelease; + try { + childRelease = (options.releaseBrokerChildImpl ?? releaseBrokerChild)(lockedRegistration, { + child, + cleanupOutcome: outcome, + now: options.now, + registryLock + }); + } catch (error) { + childRelease = { released: false, reason: `child-release-error:${error.message}` }; + } + outcomes.push({ + target: "child", + pid: child.pid, + pidIdentity: child.pidIdentity, + outcome, + childRelease: { + released: childRelease?.released === true, + reason: childRelease?.reason ?? null + } + }); + if (childRelease?.released !== true) { + cleanupVerified = false; + break; + } + } + + if (cleanupVerified) { + const broker = lockedRegistration.broker; + const outcome = await terminateTree(broker.pid, { + expectedRootIdentity: broker.pidIdentity, + ownershipSnapshot: brokerOwnershipSnapshot(broker), + termPollAttempts: options.termPollAttempts, + killPollAttempts: options.killPollAttempts, + pollIntervalMs: options.pollIntervalMs + }); + outcomes.push({ target: "broker", pid: broker.pid, pidIdentity: broker.pidIdentity, outcome }); + cleanupVerified = outcome?.verified === true; + } + + const targets = targetRecords(lockedRegistration, children.children); + const residualPids = getLive( + targets.map((target) => target.pid), + { identities: targets.map((target) => target.identity) } + ); + const residualSet = new Set(residualPids); + residualIdentities = targets + .filter((target) => residualSet.has(target.pid)) + .map((target) => target.identity); + if (cleanupVerified && residualIdentities.length === 0) { + decision = "cleanup-verified"; + } + + const attemptId = (options.attemptIdFactory ?? (() => randomUUID()))(); + const receipt = (options.publishBrokerReaperReceiptImpl ?? publishBrokerReaperReceipt)(lockedRegistration, { + attemptId, + decision, + outcomes, + residualIdentities, + createdAt: (options.now ?? (() => new Date().toISOString()))() + }); + const terminal = + decision === "cleanup-verified" && receipt?.published === true + ? (options.publishBrokerTerminalImpl ?? publishBrokerTerminal)(lockedRegistration, { + attemptId, + receiptPath: receipt.path, + retiredAt: (options.now ?? (() => new Date().toISOString()))(), + registryLock + }) + : null; + const terminalRecorded = decision !== "cleanup-verified" || terminal?.terminal === true; + result = { + status: decision === "cleanup-verified" && receipt?.published === true && terminalRecorded ? "reaped" : "report-only", + brokerKey: lockedRegistration.brokerKey, + pid: lockedRegistration.broker.pid, + reason: + receipt?.published !== true + ? `${decision}-receipt-unavailable` + : terminalRecorded + ? decision + : `${decision}-${terminal?.reason ?? "terminal-unavailable"}`, + outcomes, + residualIdentities, + receiptPath: receipt?.published === true ? receipt.path : null, + terminalPath: terminal?.terminal === true ? terminal.path : null + }; + } + } + } + } finally { + const released = releaseLock(registration, registryLock); + if (released?.released !== true) { + if (result) { + result.status = "report-only"; + result.reason = `registry-lock-release-${released?.reason ?? "failed"}`; + } else { + result = refusal(candidate, `registry-lock-release-${released?.reason ?? "failed"}`); + } + } + } + return result; +} + +export async function runRegisteredBrokerReaper(options = {}) { + const mode = options.mode === APPLY_MODE ? APPLY_MODE : REPORT_MODE; + const candidates = listRegistrationCandidates(options.env ?? process.env); + const results = []; + for (const candidate of candidates) { + results.push(await processRegistration(candidate, { ...options, mode })); + } + return { + mode, + scanned: candidates.length, + reaped: results.filter((result) => result.status === "reaped").length, + reported: results.filter((result) => result.status !== "reaped").length, + results + }; +} + +async function main() { + const mode = process.argv.slice(2).includes("--apply-registered") ? APPLY_MODE : REPORT_MODE; + const summary = await runRegisteredBrokerReaper({ mode }); + for (const result of summary.results) { + process.stdout.write( + `${result.status.toUpperCase()} broker_key=${result.brokerKey ?? "unknown"} pid=${result.pid ?? "unknown"} reason=${result.reason}\n` + ); + } + process.stdout.write( + `scanned=${summary.scanned} reaped=${summary.reaped} reported=${summary.reported} mode=${summary.mode}\n` + ); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 778571e6c..ba436744a 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -2,9 +2,19 @@ import fs from "node:fs"; import process from "node:process"; +import { pathToFileURL } from "node:url"; -import { terminateProcessTree } from "./lib/process.mjs"; +import { captureStableSessionOwner, terminateProcessTree } from "./lib/process.mjs"; import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; +import { + acquireBrokerRegistryLock, + assessBrokerOwners, + loadBrokerRegistration, + releaseBrokerOwner, + releaseBrokerRegistryLock, + SESSION_OWNER_IDENTITY_ENV, + SESSION_OWNER_PID_ENV +} from "./lib/broker-ownership.mjs"; import { clearBrokerSession, LOG_FILE_ENV, @@ -13,9 +23,10 @@ import { sendBrokerShutdown, teardownBrokerSession } from "./lib/broker-lifecycle.mjs"; -import { loadState, resolveStateFile, saveState } from "./lib/state.mjs"; +import { loadState, resolveStateFile, saveState, writeCancelFlag, writeJobFile } from "./lib/state.mjs"; import { TRANSCRIPT_PATH_ENV } from "./lib/claude-session-transfer.mjs"; import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; +import { runRegisteredBrokerReaper } from "./registered-broker-reaper.mjs"; export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; const PLUGIN_DATA_ENV = "CLAUDE_PLUGIN_DATA"; @@ -39,51 +50,134 @@ function appendEnvVar(name, value) { fs.appendFileSync(process.env.CLAUDE_ENV_FILE, `export ${name}=${shellEscape(value)}\n`, "utf8"); } -function cleanupSessionJobs(cwd, sessionId) { +export async function cleanupSessionJobs(cwd, sessionId, dependencies = {}) { if (!cwd || !sessionId) { - return; + return { verified: true, failures: [] }; } const workspaceRoot = resolveWorkspaceRoot(cwd); const stateFile = resolveStateFile(workspaceRoot); if (!fs.existsSync(stateFile)) { - return; + return { verified: true, failures: [] }; } const state = loadState(workspaceRoot); const removedJobs = state.jobs.filter((job) => job.sessionId === sessionId); if (removedJobs.length === 0) { - return; + return { verified: true, failures: [] }; } + const terminate = dependencies.terminateProcessTreeImpl ?? terminateProcessTree; + const retainedJobs = []; + const failures = []; for (const job of removedJobs) { const stillRunning = job.status === "queued" || job.status === "running"; if (!stillRunning) { continue; } + if (job.status === "queued" && !Number.isFinite(job.pid)) { + writeCancelFlag(workspaceRoot, job.id); + retainedJobs.push(job); + continue; + } try { - terminateProcessTree(job.pid ?? Number.NaN); - } catch { - // Ignore teardown failures during session shutdown. + const expectedRootIdentity = job.processIdentity ?? null; + const ownershipCaptureFailed = job.ownershipCaptureFailed === true; + const outcome = await terminate(job.pid ?? Number.NaN, { + expectedRootIdentity, + ownershipSnapshot: null, + requireVerifiedOwnership: ownershipCaptureFailed, + priorCleanupDegraded: job.cleanupOutcome?.degraded === true + }); + if (outcome?.verified === true) { + continue; + } + const cleanupFailure = + ownershipCaptureFailed && !expectedRootIdentity + ? `Job ${job.id} could not be verified as owned and was left alone.` + : "Session cleanup could not verify process termination."; + const retainedJob = { + ...job, + phase: "cleanup-pending", + cleanupOutcome: outcome, + cleanupFailure + }; + writeJobFile(workspaceRoot, job.id, retainedJob); + retainedJobs.push(retainedJob); + } catch (error) { + if (error?.code === "ESRCH") { + continue; + } + const retainedJob = { + ...job, + phase: "cleanup-pending", + cleanupOutcome: { + attempted: true, + delivered: false, + verified: false, + degraded: true, + survivors: [], + survivorIdentities: [] + }, + cleanupFailure: error instanceof Error ? error.message : String(error) + }; + writeJobFile(workspaceRoot, job.id, retainedJob); + retainedJobs.push(retainedJob); + failures.push(error); } } saveState(workspaceRoot, { ...state, - jobs: state.jobs.filter((job) => job.sessionId !== sessionId) + jobs: state.jobs.filter((job) => job.sessionId !== sessionId).concat(retainedJobs) }); + if (failures.length > 0) { + throw failures[0]; + } + return { verified: retainedJobs.length === 0, failures: [] }; } -function handleSessionStart(input) { +export async function handleSessionStart(input, dependencies = {}) { appendEnvVar(SESSION_ID_ENV, input.session_id); appendEnvVar(TRANSCRIPT_PATH_ENV, input.transcript_path); appendEnvVar(PLUGIN_DATA_ENV, process.env[PLUGIN_DATA_ENV]); + if ((dependencies.platform ?? process.platform) === "win32") { + return; + } + try { + const captureOwner = dependencies.captureStableSessionOwnerImpl ?? captureStableSessionOwner; + const owner = captureOwner(dependencies.pid ?? process.pid, { + cwd: input.cwd || process.cwd(), + env: process.env, + platform: dependencies.platform + }); + if (owner?.pid && owner.identity) { + appendEnvVar(SESSION_OWNER_PID_ENV, owner.pid); + appendEnvVar(SESSION_OWNER_IDENTITY_ENV, owner.identity); + } + } catch { + // Missing owner identity leaves this session in the report-only class. + } + // A killed session cannot run SessionEnd. Reap only immutable registered + // ownership on the next harness start so crash residue converges without a + // process-name sweep, cron job, or manual cleanup. + try { + const reapRegistered = dependencies.runRegisteredBrokerReaperImpl ?? runRegisteredBrokerReaper; + await reapRegistered({ + mode: "apply-registered", + env: dependencies.env ?? process.env + }); + } catch (error) { + const warn = dependencies.warnImpl ?? ((message) => process.stderr.write(`${message}\n`)); + warn(`Registered Codex broker cleanup remains report-only: ${error instanceof Error ? error.message : String(error)}`); + } } -async function handleSessionEnd(input) { +export async function handleSessionEnd(input, dependencies = {}) { const cwd = input.cwd || process.cwd(); + const loadBroker = dependencies.loadBrokerSessionImpl ?? loadBrokerSession; const brokerSession = - loadBrokerSession(cwd) ?? + loadBroker(cwd) ?? (process.env[BROKER_ENDPOINT_ENV] ? { endpoint: process.env[BROKER_ENDPOINT_ENV], @@ -96,21 +190,114 @@ async function handleSessionEnd(input) { const logFile = brokerSession?.logFile ?? null; const sessionDir = brokerSession?.sessionDir ?? null; const pid = brokerSession?.pid ?? null; + const pidIdentity = brokerSession?.pidIdentity ?? null; + const ownershipSnapshot = brokerSession?.ownershipSnapshot ?? null; + const requireVerifiedOwnership = brokerSession?.ownershipCaptureFailed === true; + const registry = brokerSession?.registry?.registered === true ? brokerSession.registry : null; + let registryAssessment = null; + let registryLock = null; - if (brokerEndpoint) { - await sendBrokerShutdown(brokerEndpoint); + if (registry) { + const acquireRegistryLock = dependencies.acquireBrokerRegistryLockImpl ?? acquireBrokerRegistryLock; + registryLock = acquireRegistryLock(registry); + if (registryLock?.acquired === true) { + const loadRegistration = dependencies.loadBrokerRegistrationImpl ?? loadBrokerRegistration; + const lockedRegistration = loadRegistration({ + endpoint: brokerEndpoint, + brokerIdentity: pidIdentity, + env: dependencies.env ?? process.env + }); + if ( + lockedRegistration?.registered !== true || + lockedRegistration.brokerKey !== registry.brokerKey || + lockedRegistration.registryDir !== registry.registryDir + ) { + registryAssessment = { + safeToShutdown: false, + reason: "broker-record-invalid", + liveOwners: [], + deadOwners: [], + releasedOwners: [], + malformed: [registry.registryDir] + }; + } else { + const releaseOwner = dependencies.releaseBrokerOwnerImpl ?? releaseBrokerOwner; + const assessOwners = dependencies.assessBrokerOwnersImpl ?? assessBrokerOwners; + const ownerRelease = releaseOwner(lockedRegistration, { + env: dependencies.env ?? process.env, + registryLock + }); + registryAssessment = ownerRelease?.released === true + ? assessOwners(lockedRegistration) + : { + safeToShutdown: false, + reason: `owner-release-${ownerRelease?.reason ?? "failed"}`, + liveOwners: [], + deadOwners: [], + releasedOwners: [], + malformed: [] + }; + } + } else { + registryAssessment = { + safeToShutdown: false, + reason: registryLock?.reason ?? "registry-busy", + liveOwners: [], + deadOwners: [], + releasedOwners: [], + malformed: [] + }; + } } - cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); - teardownBrokerSession({ - endpoint: brokerEndpoint, - pidFile, - logFile, - sessionDir, - pid, - killProcess: terminateProcessTree - }); - clearBrokerSession(cwd); + const brokerShutdownAllowed = registryAssessment?.safeToShutdown === true; + let brokerCleanup = { verified: true }; + let registryLockReleaseFailure = null; + try { + if (brokerEndpoint && brokerShutdownAllowed) { + await (dependencies.sendBrokerShutdownImpl ?? sendBrokerShutdown)(brokerEndpoint); + } + if (brokerShutdownAllowed) { + const teardownBroker = dependencies.teardownBrokerSessionImpl ?? teardownBrokerSession; + brokerCleanup = await teardownBroker({ + endpoint: brokerEndpoint, + pidFile, + logFile, + sessionDir, + pid, + pidIdentity, + ownershipSnapshot, + requireVerifiedOwnership, + killProcess: dependencies.terminateProcessTreeImpl ?? terminateProcessTree + }); + if (brokerCleanup?.verified === true) { + (dependencies.clearBrokerSessionImpl ?? clearBrokerSession)(cwd); + } + } + } finally { + if (registry && registryLock?.acquired === true) { + const releaseRegistryLock = dependencies.releaseBrokerRegistryLockImpl ?? releaseBrokerRegistryLock; + const released = releaseRegistryLock(registry, registryLock); + if (released?.released !== true) { + registryLockReleaseFailure = released?.reason ?? "unknown"; + } + } + } + + const cleanupJobs = dependencies.cleanupSessionJobsImpl ?? cleanupSessionJobs; + const jobCleanup = await cleanupJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); + if (registryLockReleaseFailure) { + throw new Error(`Broker registry lock release failed (${registryLockReleaseFailure}).`); + } + if (jobCleanup?.verified !== true) { + throw new Error("Session cleanup remains pending because process termination could not be verified."); + } + if (brokerCleanup?.verified !== true) { + throw new Error("Broker cleanup remains pending because process termination could not be verified."); + } + if (registry && !brokerShutdownAllowed && registryAssessment?.reason !== "live-owner") { + throw new Error(`Broker cleanup remains report-only because registered ownership is ambiguous (${registryAssessment?.reason ?? "unknown"}).`); + } } async function main() { @@ -118,7 +305,7 @@ async function main() { const eventName = process.argv[2] ?? input.hook_event_name ?? ""; if (eventName === "SessionStart") { - handleSessionStart(input); + await handleSessionStart(input); return; } @@ -127,7 +314,9 @@ async function main() { } } -main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - process.exit(1); -}); +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/tests/broker-ownership.test.mjs b/tests/broker-ownership.test.mjs new file mode 100644 index 000000000..451088cb5 --- /dev/null +++ b/tests/broker-ownership.test.mjs @@ -0,0 +1,719 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + acquireBrokerRegistryLock as acquireBrokerRegistryLockImpl, + assessBrokerOwners, + loadBrokerChildren, + publishBrokerChild as publishBrokerChildImpl, + publishBrokerChildObservation as publishBrokerChildObservationImpl, + publishBrokerRegistration, + publishRegisteredBroker, + registerBrokerOwner as registerBrokerOwnerImpl, + releaseBrokerChild as releaseBrokerChildImpl, + releaseBrokerOwner as releaseBrokerOwnerImpl, + releaseBrokerRegistryLock +} from "../plugins/codex/scripts/lib/broker-ownership.mjs"; + +const TEST_LOCK_PID = 4900; +const TEST_LOCK_IDENTITY = "4900@Mon Jul 27 00:00:40 2026"; + +function withTestRegistryLock(options = {}) { + const pid = options.pid ?? TEST_LOCK_PID; + const pidIdentity = options.pidIdentity ?? (pid === TEST_LOCK_PID + ? TEST_LOCK_IDENTITY + : `${pid}@Mon Jul 27 00:00:40 2026`); + return { + ...options, + pid, + pidIdentity, + hasLiveProcessIdentityImpl: () => true, + ...(options.hasLiveProcessIdentityImpl + ? { hasLiveProcessIdentityImpl: options.hasLiveProcessIdentityImpl } + : {}) + }; +} + +function acquireBrokerRegistryLock(registration, options = {}) { + return acquireBrokerRegistryLockImpl(registration, withTestRegistryLock(options)); +} + +function registerBrokerOwner(registration, options = {}) { + return registerBrokerOwnerImpl(registration, withTestRegistryLock(options)); +} + +function releaseBrokerOwner(registration, options = {}) { + return releaseBrokerOwnerImpl(registration, withTestRegistryLock(options)); +} + +function publishBrokerChild(registration, options = {}) { + return publishBrokerChildImpl(registration, withTestRegistryLock(options)); +} + +function publishBrokerChildObservation(registration, options = {}) { + return publishBrokerChildObservationImpl(registration, withTestRegistryLock(options)); +} + +function releaseBrokerChild(registration, options = {}) { + return releaseBrokerChildImpl(registration, withTestRegistryLock(options)); +} + +function makeFixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-ownership-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = { CLAUDE_PLUGIN_DATA: root }; + const brokerIdentity = "4100@Mon Jul 27 00:00:00 2026"; + const ownershipSnapshot = { + rootPid: 4100, + rootIdentity: brokerIdentity, + processGroupId: 4100, + members: [ + { + pid: 4100, + parentPid: 1, + processGroupId: 4100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: brokerIdentity, + depth: 0 + } + ] + }; + const registration = publishBrokerRegistration({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot, + env, + now: () => "2026-07-27T00:00:00.000Z" + }); + assert.equal(registration.registered, true); + return { root, env, registration }; +} + +function ownerEnv(env, sessionId, pid, startedAt) { + return { + ...env, + CODEX_COMPANION_SESSION_ID: sessionId, + CODEX_COMPANION_SESSION_OWNER_PID: String(pid), + CODEX_COMPANION_SESSION_OWNER_IDENTITY: `${pid}@${startedAt}` + }; +} + +test("initial broker and owner records become visible atomically", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-atomic-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-atomic", 5050, "Mon Jul 27 00:00:45 2026"); + const brokerIdentity = "4100@Mon Jul 27 00:00:00 2026"; + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: brokerIdentity, + processGroupId: 4100, + members: [] + }, + env, + now: () => "2026-07-27T00:00:00.000Z", + hasLiveProcessIdentityImpl: (pid, identity) => + (pid === 5050 && identity === "5050@Mon Jul 27 00:00:45 2026") || + (pid === 4100 && identity === brokerIdentity) + }); + + assert.equal(registration.registered, true); + assert.equal(fs.existsSync(path.join(registration.registryDir, "broker.json")), true); + assert.equal(fs.readdirSync(path.join(registration.registryDir, "owners")).length, 1); + assert.equal(fs.readdirSync(registration.registryRoot).some((name) => name.endsWith(".prepared")), false); +}); + +test("initial publication can make its transaction lock visible atomically", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-atomic-lock-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const brokerIdentity = "4100@Mon Jul 27 00:00:00 2026"; + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: brokerIdentity, + processGroupId: 4100, + members: [] + }, + env: ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-atomic-lock", 5050, "Mon Jul 27 00:00:45 2026"), + retainRegistryLock: true, + hasLiveProcessIdentityImpl: () => true, + getProcessIdentityImpl: (pid) => `${pid}@Mon Jul 27 00:00:41 2026` + }); + + assert.equal(registration.registered, true); + assert.equal(registration.registryLock?.acquired, true); + assert.equal(fs.existsSync(registration.registryLock.path), true); + assert.equal(acquireBrokerRegistryLock(registration).acquired, false); + assert.equal(releaseBrokerRegistryLock(registration, registration.registryLock).released, true); +}); + +test("missing owner identity prevents any broker registry publication", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-no-owner-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: "4100@Mon Jul 27 00:00:00 2026", + processGroupId: 4100, + members: [] + }, + env: { CLAUDE_PLUGIN_DATA: root } + }); + + assert.deepEqual(registration, { registered: false, reason: "session-owner-unavailable" }); + assert.equal(fs.existsSync(path.join(root, "state", "broker-ownership-v1")), false); +}); + +test("stale owner identity prevents any broker registry publication", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-stale-owner-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: "4100@Mon Jul 27 00:00:00 2026", + processGroupId: 4100, + members: [] + }, + env: ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-stale", 5050, "Mon Jul 27 00:00:45 2026"), + hasLiveProcessIdentityImpl: () => false + }); + + assert.deepEqual(registration, { registered: false, reason: "session-owner-not-live" }); + assert.equal(fs.existsSync(path.join(root, "state", "broker-ownership-v1")), false); +}); + +test("initial publication revalidates the sole owner after preparing registry bytes", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-owner-revalidation-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-race", 5050, "Mon Jul 27 00:00:45 2026"); + let ownerChecks = 0; + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: "4100@Mon Jul 27 00:00:00 2026", + processGroupId: 4100, + members: [] + }, + env, + hasLiveProcessIdentityImpl(pid) { + if (pid === 5050) { + ownerChecks += 1; + return ownerChecks === 1; + } + return true; + } + }); + + assert.deepEqual(registration, { registered: false, reason: "session-owner-not-live" }); + assert.equal(fs.existsSync(path.join(root, "state", "broker-ownership-v1")), true); + assert.deepEqual(fs.readdirSync(path.join(root, "state", "broker-ownership-v1")), []); +}); + +test("initial publication revalidates the broker identity immediately before visibility", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-process-revalidation-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-race", 5050, "Mon Jul 27 00:00:45 2026"); + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: "4100@Mon Jul 27 00:00:00 2026", + processGroupId: 4100, + members: [] + }, + env, + hasLiveProcessIdentityImpl(pid) { + return pid === 5050; + } + }); + + assert.deepEqual(registration, { registered: false, reason: "broker-not-live" }); + assert.deepEqual(fs.readdirSync(path.join(root, "state", "broker-ownership-v1")), []); +}); + +test("owner publication revalidates the exact owner while holding the registry lock", (t) => { + const { registration, env } = makeFixture(t); + const sessionEnv = ownerEnv(env, "session-revalidated", 5060, "Mon Jul 27 00:00:46 2026"); + const lockOptions = { + pid: 5061, + pidIdentity: "5061@Mon Jul 27 00:00:47 2026" + }; + + const rejectedCreate = registerBrokerOwner(registration, { + env: sessionEnv, + ...lockOptions, + hasLiveProcessIdentityImpl: () => false + }); + assert.deepEqual(rejectedCreate, { registered: false, reason: "session-owner-not-live" }); + + const registered = registerBrokerOwner(registration, { + env: sessionEnv, + ...lockOptions, + hasLiveProcessIdentityImpl: () => true + }); + assert.equal(registered.registered, true); + + const rejectedExisting = registerBrokerOwner(registration, { + env: sessionEnv, + ...lockOptions, + hasLiveProcessIdentityImpl: () => false + }); + assert.deepEqual(rejectedExisting, { registered: false, reason: "session-owner-not-live" }); +}); + +test("owner publication fails closed while cleanup holds the registry lock", (t) => { + const { registration, env } = makeFixture(t); + const lock = acquireBrokerRegistryLock(registration, { + pid: 5000, + now: () => "2026-07-27T00:00:30.000Z" + }); + assert.equal(lock.acquired, true); + assert.equal(fs.statSync(lock.path).mode & 0o777, 0o700); + + const blocked = registerBrokerOwner( + registration, + { + env: ownerEnv(env, "session-racing", 5050, "Mon Jul 27 00:00:45 2026"), + getLiveProcessPidsImpl: () => [5000] + } + ); + assert.deepEqual(blocked, { registered: false, reason: "registry-busy" }); + + assert.deepEqual(releaseBrokerRegistryLock(registration, lock), { released: true }); + const registered = registerBrokerOwner( + registration, + { env: ownerEnv(env, "session-racing", 5050, "Mon Jul 27 00:00:45 2026") } + ); + assert.equal(registered.registered, true); +}); + +test("a well-formed lock whose creator is absent is quarantined before retry", (t) => { + const { registration, env } = makeFixture(t); + const stale = acquireBrokerRegistryLock(registration, { + pid: 5001, + now: () => "2026-07-27T00:00:31.000Z" + }); + assert.equal(stale.acquired, true); + + const registered = registerBrokerOwner(registration, { + env: ownerEnv(env, "session-after-stale-lock", 5051, "Mon Jul 27 00:00:46 2026"), + hasLiveProcessIdentityImpl: (pid) => pid !== 5001 + }); + assert.equal(registered.registered, true); + const staleRoot = path.join(registration.registryDir, "stale-locks"); + assert.equal(fs.readdirSync(staleRoot).length, 1); + assert.equal(fs.existsSync(stale.path), false); +}); + +test("a reused lock-owner PID does not keep an abandoned registry lock busy", (t) => { + const { registration } = makeFixture(t); + const staleIdentity = "5006@Mon Jul 27 00:00:36 2026"; + const stale = acquireBrokerRegistryLock(registration, { + pid: 5006, + pidIdentity: staleIdentity, + now: () => "2026-07-27T00:00:36.000Z" + }); + assert.equal(stale.acquired, true); + + const contender = acquireBrokerRegistryLock(registration, { + pid: 5007, + pidIdentity: "5007@Mon Jul 27 00:00:37 2026", + now: () => "2026-07-27T00:00:37.000Z", + getLiveProcessPidsImpl: () => [5006], + hasLiveProcessIdentityImpl(pid, identity) { + assert.equal(pid, 5006); + assert.equal(identity, staleIdentity); + return false; + } + }); + + assert.equal(contender.acquired, true); + const owner = JSON.parse(fs.readFileSync(path.join(contender.path, "owner.json"), "utf8")); + assert.equal(owner.pidIdentity, "5007@Mon Jul 27 00:00:37 2026"); + assert.deepEqual(releaseBrokerRegistryLock(registration, contender), { released: true }); +}); + +test("a stale-lock contender cannot quarantine a replacement live lock", (t) => { + const { registration } = makeFixture(t); + const stale = acquireBrokerRegistryLock(registration, { + pid: 5002, + now: () => "2026-07-27T00:00:32.000Z" + }); + assert.equal(stale.acquired, true); + + let replacement; + let livenessChecks = 0; + const contender = acquireBrokerRegistryLock(registration, { + pid: 5004, + now: () => "2026-07-27T00:00:34.000Z", + hasLiveProcessIdentityImpl(pid) { + livenessChecks += 1; + if (livenessChecks === 1) { + const staleRoot = path.join(registration.registryDir, "stale-locks"); + fs.mkdirSync(staleRoot, { recursive: true, mode: 0o700 }); + fs.chmodSync(staleRoot, 0o700); + fs.renameSync(stale.path, path.join(staleRoot, `${5002}-${stale.token}`)); + replacement = acquireBrokerRegistryLock(registration, { + pid: 5003, + now: () => "2026-07-27T00:00:33.000Z" + }); + assert.equal(replacement.acquired, true); + return false; + } + return pid === 5003; + } + }); + + assert.equal(contender.acquired, false); + assert.equal(contender.reason, "registry-busy"); + const liveOwner = JSON.parse(fs.readFileSync(path.join(replacement.path, "owner.json"), "utf8")); + assert.equal(liveOwner.token, replacement.token); + assert.deepEqual(releaseBrokerRegistryLock(registration, replacement), { released: true }); +}); + +test("registered broker with a live owner is not eligible for cleanup", (t) => { + const { registration, env } = makeFixture(t); + const liveOwner = ownerEnv(env, "session-live", 5100, "Mon Jul 27 00:01:00 2026"); + const owner = registerBrokerOwner(registration, { env: liveOwner, now: () => "2026-07-27T00:01:00.000Z" }); + + const assessment = assessBrokerOwners(registration, { + getLiveProcessPidsImpl(pids, options) { + assert.deepEqual(pids, [5100]); + assert.deepEqual(options.identities, ["5100@Mon Jul 27 00:01:00 2026"]); + return [5100]; + } + }); + + assert.equal(owner.registered, true); + assert.equal(assessment.safeToShutdown, false); + assert.equal(assessment.reason, "live-owner"); + assert.deepEqual(assessment.liveOwners.map((candidate) => candidate.sessionId), ["session-live"]); + assert.equal(fs.statSync(owner.path).mode & 0o777, 0o600); +}); + +test("registered broker is eligible only after every owner is dead or released", (t) => { + const { registration, env } = makeFixture(t); + const ownerA = ownerEnv(env, "session-a", 5200, "Mon Jul 27 00:02:00 2026"); + const ownerB = ownerEnv(env, "session-b", 5300, "Mon Jul 27 00:03:00 2026"); + registerBrokerOwner(registration, { env: ownerA }); + registerBrokerOwner(registration, { env: ownerB }); + + const mixed = assessBrokerOwners(registration, { + getLiveProcessPidsImpl(pids) { + return pids[0] === 5300 ? [5300] : []; + } + }); + assert.equal(mixed.safeToShutdown, false); + assert.deepEqual(mixed.liveOwners.map((candidate) => candidate.sessionId), ["session-b"]); + + releaseBrokerOwner(registration, { env: ownerB, now: () => "2026-07-27T00:04:00.000Z" }); + const released = assessBrokerOwners(registration, { + getLiveProcessPidsImpl() { + return []; + } + }); + assert.equal(released.safeToShutdown, true); + assert.equal(released.reason, "all-owners-dead-or-released"); + assert.deepEqual(released.releasedOwners.map((candidate) => candidate.sessionId), ["session-b"]); +}); + +test("owner PID reuse with a different identity does not keep a broker alive", (t) => { + const { registration, env } = makeFixture(t); + registerBrokerOwner(registration, { + env: ownerEnv(env, "session-reused", 5400, "Mon Jul 27 00:05:00 2026") + }); + + const assessment = assessBrokerOwners(registration, { + getLiveProcessPidsImpl() { + return []; + } + }); + assert.equal(assessment.safeToShutdown, true); + assert.deepEqual(assessment.deadOwners.map((candidate) => candidate.sessionId), ["session-reused"]); +}); + +test("malformed owner state blocks cleanup instead of being skipped", (t) => { + const { registration } = makeFixture(t); + const ownersDir = path.join(registration.registryDir, "owners"); + fs.mkdirSync(ownersDir, { recursive: true }); + fs.writeFileSync(path.join(ownersDir, "malformed.json"), "{not-json\n", { mode: 0o600 }); + + const assessment = assessBrokerOwners(registration, { + getLiveProcessPidsImpl() { + throw new Error("malformed rows must stop before liveness checks"); + } + }); + assert.equal(assessment.safeToShutdown, false); + assert.equal(assessment.reason, "malformed-registry"); + assert.equal(assessment.malformed.length, 1); +}); + +test("symlinked or overly permissive registry rows block cleanup", (t) => { + const permissiveFixture = makeFixture(t); + const registered = registerBrokerOwner(permissiveFixture.registration, { + env: ownerEnv(permissiveFixture.env, "session-permissive", 5450, "Mon Jul 27 00:05:30 2026") + }); + fs.chmodSync(registered.path, 0o644); + const permissive = assessBrokerOwners(permissiveFixture.registration, { + getLiveProcessPidsImpl: () => [] + }); + assert.equal(permissive.safeToShutdown, false); + assert.equal(permissive.reason, "malformed-registry"); + + const symlinkFixture = makeFixture(t); + const ownersDir = path.join(symlinkFixture.registration.registryDir, "owners"); + fs.mkdirSync(ownersDir, { recursive: true, mode: 0o700 }); + fs.chmodSync(ownersDir, 0o700); + const target = path.join(symlinkFixture.root, "outside.json"); + fs.writeFileSync(target, "{}\n", { mode: 0o600 }); + fs.symlinkSync(target, path.join(ownersDir, "linked.json")); + const symlinked = assessBrokerOwners(symlinkFixture.registration, { + getLiveProcessPidsImpl: () => [] + }); + assert.equal(symlinked.safeToShutdown, false); + assert.equal(symlinked.reason, "malformed-registry"); +}); + +test("a broker record with no owner rows remains unregistered and report-only", (t) => { + const { registration } = makeFixture(t); + const assessment = assessBrokerOwners(registration, { + getLiveProcessPidsImpl() { + throw new Error("an ownerless broker must not reach liveness checks"); + } + }); + assert.equal(assessment.safeToShutdown, false); + assert.equal(assessment.reason, "no-registered-owner"); +}); + +test("broker child ownership is immutable and identity keyed", (t) => { + const { registration } = makeFixture(t); + const childSnapshot = { + rootPid: 6100, + rootIdentity: "6100@Mon Jul 27 00:06:00 2026", + processGroupId: 6100, + members: [ + { + pid: 6100, + parentPid: 4100, + processGroupId: 6100, + state: "S", + startedAt: "Mon Jul 27 00:06:00 2026", + identity: "6100@Mon Jul 27 00:06:00 2026", + depth: 0 + } + ] + }; + const first = publishBrokerChild(registration, { + ownershipSnapshot: childSnapshot, + now: () => "2026-07-27T00:06:00.000Z" + }); + const second = publishBrokerChild(registration, { + ownershipSnapshot: childSnapshot, + now: () => "2026-07-27T00:06:00.000Z" + }); + + assert.equal(first.registered, true); + assert.equal(second.path, first.path); + assert.equal(fs.statSync(first.path).mode & 0o777, 0o600); + assert.deepEqual(JSON.parse(fs.readFileSync(first.path, "utf8")).ownershipSnapshot, childSnapshot); + + const refused = releaseBrokerChild(registration, { + child: first.child, + cleanupOutcome: { verified: false, survivors: [6100] } + }); + assert.equal(refused.released, false); + assert.equal(loadBrokerChildren(registration).children.length, 1); + + const released = releaseBrokerChild(registration, { + child: first.child, + cleanupOutcome: { verified: true, survivors: [], survivorIdentities: [] }, + now: () => "2026-07-27T00:07:00.000Z" + }); + assert.equal(released.released, true); + assert.equal(fs.statSync(released.path).mode & 0o777, 0o600); + const children = loadBrokerChildren(registration); + assert.equal(children.children.length, 0); + assert.equal(children.releasedChildren.length, 1); +}); + +test("post-activation child observations extend durable cleanup ownership", (t) => { + const { registration } = makeFixture(t); + const rootIdentity = "6200@Mon Jul 27 00:08:00 2026"; + const child = publishBrokerChild(registration, { + ownershipSnapshot: { + rootPid: 6200, + rootIdentity, + processGroupId: 6200, + members: [ + { + pid: 6200, + parentPid: 4100, + processGroupId: 6200, + state: "S", + startedAt: "Mon Jul 27 00:08:00 2026", + identity: rootIdentity, + depth: 0 + } + ] + } + }); + const helperIdentity = "6202@Mon Jul 27 00:08:02 2026"; + const helperParentIdentity = "6201@Mon Jul 27 00:08:01 2026"; + const observed = publishBrokerChildObservation(registration, { + child: child.child, + ownershipSnapshot: { + rootPid: 6200, + rootIdentity, + processGroupId: 6200, + members: [ + { + pid: 6200, + parentPid: 4100, + processGroupId: 6200, + state: "S", + startedAt: "Mon Jul 27 00:08:00 2026", + identity: rootIdentity, + depth: 0 + }, + { + pid: 6201, + parentPid: 6200, + processGroupId: 6200, + state: "S", + startedAt: "Mon Jul 27 00:08:01 2026", + identity: helperParentIdentity, + depth: 1 + }, + { + pid: 6202, + parentPid: 6201, + processGroupId: 6202, + state: "S", + startedAt: "Mon Jul 27 00:08:02 2026", + identity: helperIdentity, + depth: 2 + } + ] + }, + now: () => "2026-07-27T00:08:03.000Z" + }); + + assert.equal(observed.observed, true); + assert.equal(fs.statSync(observed.path).mode & 0o777, 0o600); + const repeated = publishBrokerChildObservation(registration, { + child: child.child, + ownershipSnapshot: observed.observation.ownershipSnapshot, + now: () => "2026-07-27T00:08:04.000Z" + }); + assert.equal(repeated.observed, true); + assert.equal(repeated.reason, "child-already-observed"); + assert.equal(repeated.path, observed.path); + const loaded = loadBrokerChildren(registration); + assert.equal(loaded.valid, true); + assert.deepEqual(loaded.children[0].ownershipSnapshot.members.map((member) => member.identity), [ + rootIdentity, + helperParentIdentity, + helperIdentity + ]); + + const released = releaseBrokerChild(registration, { + child: loaded.children[0], + cleanupOutcome: { verified: true, survivors: [], survivorIdentities: [] } + }); + assert.equal(released.released, true); + assert.equal(loadBrokerChildren(registration).releasedChildren[0].ownershipSnapshot.members.length, 3); +}); + +test("an observation member outside the immutable child tree makes the registry report-only", (t) => { + const { registration } = makeFixture(t); + const rootIdentity = "6300@Mon Jul 27 00:09:00 2026"; + const child = publishBrokerChild(registration, { + ownershipSnapshot: { + rootPid: 6300, + rootIdentity, + processGroupId: 6300, + members: [ + { + pid: 6300, + parentPid: 4100, + processGroupId: 6300, + state: "S", + startedAt: "Mon Jul 27 00:09:00 2026", + identity: rootIdentity, + depth: 0 + } + ] + } + }); + const ownershipSnapshot = { + rootPid: 6300, + rootIdentity, + processGroupId: 6300, + members: [ + child.child.ownershipSnapshot.members[0], + { + pid: 9900, + parentPid: 1, + processGroupId: 9900, + state: "S", + startedAt: "Mon Jul 27 00:09:01 2026", + identity: "9900@Mon Jul 27 00:09:01 2026", + depth: 1 + } + ] + }; + + const refused = publishBrokerChildObservation(registration, { + child: child.child, + ownershipSnapshot + }); + assert.equal(refused.observed, false); + assert.equal(refused.reason, "child-observation-invalid"); + + const observationKey = createHash("sha256").update(JSON.stringify(ownershipSnapshot)).digest("hex"); + const observationDir = path.join(registration.registryDir, "child-observations", child.child.childKey); + const observationPath = path.join(observationDir, `${observationKey}.json`); + fs.mkdirSync(observationDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync( + observationPath, + `${JSON.stringify({ + version: 1, + kind: "child-observation", + brokerKey: registration.brokerKey, + childKey: child.child.childKey, + observationKey, + ownershipSnapshot, + observedAt: "2026-07-27T00:09:02.000Z" + })}\n`, + { mode: 0o600 } + ); + + const loaded = loadBrokerChildren(registration); + assert.equal(loaded.valid, false); + assert.equal(loaded.reason, "malformed-child-registry"); + assert.deepEqual(loaded.children, []); + assert.deepEqual(loaded.malformed, [observationPath]); +}); diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index f83c96a0d..3dd3752bd 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -15,7 +15,10 @@ const readline = require("node:readline"); const STATE_PATH = ${JSON.stringify(statePath)}; const BEHAVIOR = ${JSON.stringify(behavior)}; + const DETACHED_FIXTURE_TTL_MS = 5 * 60 * 1000; + const SELF_EXPIRING_KEEPALIVE = "setTimeout(() => process.exit(0), " + DETACHED_FIXTURE_TTL_MS + "); setInterval(() => {}, 1000)"; const interruptibleTurns = new Map(); + const { spawn } = require("node:child_process"); function loadState() { if (!fs.existsSync(STATE_PATH)) { @@ -272,6 +275,22 @@ if (args[0] !== "app-server") { } const bootState = loadState(); bootState.appServerStarts = (bootState.appServerStarts || 0) + 1; +if (BEHAVIOR === "with-helper-child" || BEHAVIOR === "slow-task-with-helper-child" || BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "with-resistant-helper" || BEHAVIOR === "crash-with-post-snapshot-helper") { + if (BEHAVIOR !== "crash-with-post-snapshot-helper") { + const helperCode = BEHAVIOR === "with-resistant-helper" + ? "process.on('SIGTERM', () => {}); " + SELF_EXPIRING_KEEPALIVE + : SELF_EXPIRING_KEEPALIVE; + const helper = spawn(process.execPath, ["-e", helperCode], { + detached: process.platform !== "win32", + stdio: "ignore" + }); + helper.unref(); + bootState.helperPids = [...(bootState.helperPids || []), helper.pid]; + } +} +if (BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "crash-with-post-snapshot-helper" || BEHAVIOR === "crash-with-post-activation-regrouped-helper" || BEHAVIOR === "post-activation-helper-on-thread-list" || BEHAVIOR === "streaming-helper-after-response") { + bootState.appServerPids = [...(bootState.appServerPids || []), process.pid]; +} saveState(bootState); const rl = readline.createInterface({ input: process.stdin }); @@ -287,8 +306,37 @@ rl.on("line", (line) => { switch (message.method) { case "initialize": state.capabilities = message.params.capabilities || null; + if (BEHAVIOR === "crash-with-post-activation-regrouped-helper") { + const helper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { + detached: process.platform !== "win32", + stdio: "ignore" + }); + helper.unref(); + state.helperPids = [...(state.helperPids || []), helper.pid]; + } saveState(state); send({ id: message.id, result: { userAgent: "fake-codex-app-server" } }); + if (BEHAVIOR === "crash-with-regrouped-helper") { + setTimeout(() => process.exit(1), 100); + } + if (BEHAVIOR === "crash-with-post-snapshot-helper") { + setTimeout(() => { + const helper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { + detached: false, + stdio: "ignore" + }); + helper.unref(); + const current = loadState(); + current.helperPids = [...(current.helperPids || []), helper.pid]; + saveState(current); + setTimeout(() => process.exit(1), 100); + }, 250); + } + if (BEHAVIOR === "crash-with-post-activation-regrouped-helper") { + // Leave enough time for the broker to persist the post-response + // ownership observation before simulating the later crash. + setTimeout(() => process.exit(1), 500); + } break; case "initialized": @@ -328,6 +376,15 @@ rl.on("line", (line) => { } case "thread/list": { + if (BEHAVIOR === "post-activation-helper-on-thread-list" && !state.helperPids?.length) { + const helper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { + detached: process.platform !== "win32", + stdio: "ignore" + }); + helper.unref(); + state.helperPids = [helper.pid]; + saveState(state); + } let threads = state.threads.slice(); if (message.params.cwd) { threads = threads.filter((thread) => thread.cwd === message.params.cwd); @@ -454,6 +511,22 @@ rl.on("line", (line) => { saveState(state); send({ id: message.id, result: { turn: buildTurn(turnId) } }); + if (BEHAVIOR === "streaming-helper-after-response") { + setTimeout(() => { + const helper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { + detached: process.platform !== "win32", + stdio: "ignore" + }); + helper.unref(); + const current = loadState(); + current.helperPids = [...(current.helperPids || []), helper.pid]; + saveState(current); + send({ method: "item/started", params: { threadId: thread.id, turnId, item: { type: "commandExecution", id: "late-helper" } } }); + setTimeout(() => process.exit(1), 750); + }, 100); + break; + } + const payload = message.params.outputSchema && message.params.outputSchema.properties && message.params.outputSchema.properties.verdict ? structuredReviewPayload(prompt) : taskPayload(prompt, thread.name && thread.name.startsWith("Codex Companion Task") && prompt.includes("Continue from the current thread state")); @@ -600,7 +673,7 @@ rl.on("line", (line) => { send({ method: "turn/completed", params: { threadId: thread.id, turn: buildTurn(turnId, "completed") } }); }, 5000); interruptibleTurns.set(turnId, { threadId: thread.id, timer }); - } else if (BEHAVIOR === "slow-task") { + } else if (BEHAVIOR === "slow-task" || BEHAVIOR === "slow-task-with-helper-child") { emitTurnCompletedLater(thread.id, turnId, items, 400); } else { emitTurnCompleted(thread.id, turnId, items); @@ -653,6 +726,7 @@ export function buildEnv(binDir) { const sep = process.platform === "win32" ? ";" : ":"; return { ...process.env, + CODEX_COMPANION_TEST_BROKER_TTL_MS: "30000", PATH: `${binDir}${sep}${process.env.PATH}` }; } diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 80e0715b0..916c073d9 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -1,11 +1,105 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; +import { + captureProcessOwnership, + captureStableSessionOwner, + getLiveProcessPids, + hasLiveProcessIdentity, + terminateProcessGroup, + terminateProcessTree +} from "../plugins/codex/scripts/lib/process.mjs"; -test("terminateProcessTree uses taskkill on Windows", () => { +test("captureProcessOwnership never treats a Darwin audit session as process containment", () => { + const snapshot = captureProcessOwnership(7300, { + platform: "darwin", + runCommandImpl(command, args) { + assert.equal(command, "/bin/ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]); + return { + command, + args, + status: 0, + signal: null, + stdout: "7300 1 7300 44001 S Mon Jul 27 00:09:00 2026\n", + stderr: "", + error: null + }; + } + }); + + assert.equal(snapshot.sessionId, null); + assert.equal(snapshot.members[0].sessionId, null); +}); + +test("captureStableSessionOwner records the hook process-group leader", () => { + const owner = captureStableSessionOwner(7101, { + platform: "darwin", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: [ + "7100 1 7100 S Mon Jul 27 00:07:00 2026", + "7101 7100 7100 S Mon Jul 27 00:07:01 2026" + ].join("\n"), + stderr: "", + error: null + }; + } + }); + + assert.deepEqual(owner, { + pid: 7100, + identity: "7100@Mon Jul 27 00:07:00 2026", + processGroupId: 7100 + }); +}); + +test("captureStableSessionOwner refuses a hook that is its own process-group leader", () => { + const owner = captureStableSessionOwner(7200, { + platform: "darwin", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "7200 1 7200 S Mon Jul 27 00:08:00 2026\n", + stderr: "", + error: null + }; + } + }); + + assert.equal(owner, null); +}); + +test("hasLiveProcessIdentity excludes a matching zombie process", () => { + const identity = "7300@Mon Jul 27 00:09:00 2026"; + const live = hasLiveProcessIdentity(7300, identity, { + platform: "darwin", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "7300 1 7300 Z Mon Jul 27 00:09:00 2026\n", + stderr: "", + error: null + }; + } + }); + + assert.equal(live, false); +}); + +test("terminateProcessTree uses taskkill on Windows", async () => { let captured = null; - const outcome = terminateProcessTree(1234, { + const outcome = await terminateProcessTree(1234, { platform: "win32", runCommandImpl(command, args) { captured = { command, args }; @@ -32,8 +126,28 @@ test("terminateProcessTree uses taskkill on Windows", () => { assert.equal(outcome.method, "taskkill"); }); -test("terminateProcessTree treats missing Windows processes as already stopped", () => { - const outcome = terminateProcessTree(1234, { +test("getLiveProcessPids ignores a survivor PID reused by a different process", () => { + const live = getLiveProcessPids([900], { + platform: "darwin", + identities: ["900@old"], + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "900 1 900 S Mon Jul 27 00:00:02 2026\n", + stderr: "", + error: null + }; + } + }); + + assert.deepEqual(live, []); +}); + +test("terminateProcessTree treats missing Windows processes as already stopped", async () => { + const outcome = await terminateProcessTree(1234, { platform: "win32", runCommandImpl(command, args) { return { @@ -53,3 +167,1014 @@ test("terminateProcessTree treats missing Windows processes as already stopped", assert.equal(outcome.result.status, 128); assert.match(outcome.result.stdout, /not found/i); }); + +test("terminateProcessTree terminates Unix descendant groups deepest-first", async () => { + const signals = []; + const alive = new Set([1234, 1235, 1236, 1237]); + const parents = new Map([[1234, 1], [1235, 1234], [1236, 1235], [1237, 1234]]); + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", + runCommandImpl(command, args) { + assert.equal(command, "/bin/ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]); + const stdout = [...alive] + .map((pid) => `${pid} ${parents.get(pid)} ${pid} S Mon Jul 27 00:00:0${pid - 1234} 2026`) + .join("\n"); + return { + command, + args, + status: 0, + signal: null, + stdout: stdout ? `${stdout}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive.delete(Math.abs(pid)); + } + }); + + assert.deepEqual(signals, [ + [-1236, "SIGTERM"], + [-1235, "SIGTERM"], + [-1237, "SIGTERM"], + [-1234, "SIGTERM"] + ]); + assert.equal(outcome.delivered, true); + assert.equal(outcome.method, "process-tree"); + assert.deepEqual(outcome.targets, [1236, 1235, 1237, 1234]); +}); + +test("terminateProcessTree signals a Unix PID directly when it is not a group leader", async () => { + const signals = []; + let alive = true; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: alive ? "1234 1 999 S Mon Jul 27 00:00:00 2026\n" : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive = false; + } + }); + + assert.deepEqual(signals, [[1234, "SIGTERM"]]); + assert.equal(outcome.delivered, true); + assert.equal(outcome.verified, true); + assert.equal(outcome.method, "process-tree"); +}); + +test("terminateProcessTree verifies an EPERM group-signal race only after the target disappears", async () => { + let scans = 0; + const identity = "100@Mon Jul 27 00:00:00 2026"; + const outcome = await terminateProcessTree(100, { + platform: "darwin", + expectedRootIdentity: identity, + ownershipSnapshot: { + rootPid: 100, + rootIdentity: identity, + processGroupId: 100, + members: [{ pid: 100, parentPid: 1, processGroupId: 100, state: "S", startedAt: "Mon Jul 27 00:00:00 2026", identity, depth: 0 }] + }, + runCommandImpl() { + scans += 1; + return { + status: 0, + stdout: scans <= 2 ? "100 1 100 S Mon Jul 27 00:00:00 2026\n" : "", + stderr: "", + error: null + }; + }, + killImpl() { + const error = new Error("kill EPERM"); + error.code = "EPERM"; + throw error; + }, + pollIntervalMs: 0, + termPollAttempts: 1 + }); + + assert.equal(outcome.delivered, false); + assert.equal(outcome.verified, true); + assert.deepEqual(outcome.survivors, []); +}); + +test("terminateProcessTree keeps an EPERM group-signal survivor unverified", async () => { + const identity = "100@Mon Jul 27 00:00:00 2026"; + const outcome = await terminateProcessTree(100, { + platform: "darwin", + expectedRootIdentity: identity, + ownershipSnapshot: { + rootPid: 100, + rootIdentity: identity, + processGroupId: 100, + members: [{ pid: 100, parentPid: 1, processGroupId: 100, state: "S", startedAt: "Mon Jul 27 00:00:00 2026", identity, depth: 0 }] + }, + runCommandImpl() { + return { + status: 0, + stdout: "100 1 100 S Mon Jul 27 00:00:00 2026\n", + stderr: "", + error: null + }; + }, + killImpl() { + const error = new Error("kill EPERM"); + error.code = "EPERM"; + throw error; + }, + pollIntervalMs: 0, + termPollAttempts: 1, + killPollAttempts: 1 + }); + + assert.equal(outcome.delivered, false); + assert.equal(outcome.verified, false); + assert.deepEqual(outcome.survivorIdentities, [identity]); +}); + +test("terminateProcessTree parses a captured Linux procps process table", async () => { + const signals = []; + const alive = new Set([42001, 42002]); + const sample = [ + "42001 1 42001 Ss Mon Jul 27 12:34:56 2026", + "42002 42001 42002 S Mon Jul 27 12:34:57 2026" + ].join("\n"); + const outcome = await terminateProcessTree(42001, { + platform: "linux", + expectedRootIdentity: "42001@Mon Jul 27 12:34:56 2026", + runCommandImpl(command, args) { + assert.equal(command, "/bin/ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]); + const stdout = [...alive] + .map((pid) => sample.split("\n").find((line) => line.startsWith(String(pid)))) + .filter(Boolean) + .join("\n"); + return { + command, + args, + status: 0, + signal: null, + stdout: stdout ? `${stdout}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive.delete(Math.abs(pid)); + } + }); + + assert.deepEqual(signals, [[-42002, "SIGTERM"], [-42001, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.deepEqual(outcome.targets, [42002, 42001]); +}); + +test("terminateProcessTree defers persisted cleanup when Unix process enumeration fails", async () => { + const signals = []; + const warnings = []; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", + warnImpl(message) { + warnings.push(message); + }, + runCommandImpl(command, args) { + assert.equal(command, "/bin/ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]); + return { + command, + args, + status: 1, + signal: null, + stdout: "", + stderr: "ps denied", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.equal(outcome.method, "deferred"); + assert.deepEqual(outcome.survivors, [1234]); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /deferred signalling.*1234/i); +}); + +test("terminateProcessTree preserves an owned detached group during degraded live-handle cleanup", async () => { + const signals = []; + const warnings = []; + const ownershipSnapshot = { + rootPid: 1234, + rootIdentity: "1234@Mon Jul 27 00:00:00 2026", + processGroupId: 1234, + members: [ + { + pid: 1234, + parentPid: 1, + processGroupId: 1234, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "1234@Mon Jul 27 00:00:00 2026", + depth: 0 + } + ] + }; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + ownershipSnapshot, + ownerHoldsLiveHandle: true, + warnImpl(message) { + warnings.push(message); + }, + runCommandImpl(command, args) { + return { + command, + args, + status: 1, + signal: null, + stdout: "", + stderr: "ps denied", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, [[-1234, "SIGKILL"]]); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.equal(outcome.method, "process-group"); + assert.match(warnings[0], /process-group kill fallback/i); +}); + +test("terminateProcessTree does not convert a degraded root-only cleanup into later verified success", async () => { + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", + priorCleanupDegraded: true, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "", + stderr: "", + error: null + }; + }, + killImpl() { + throw new Error("an absent root must not be signaled"); + } + }); + + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); +}); + +test("terminateProcessTree refuses a reused root PID", async () => { + const signals = []; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Sun Jul 26 00:00:00 2026", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "1234 1 1234 S Mon Jul 27 00:00:00 2026\n", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.equal(outcome.identityMismatch, true); + const ownershipSnapshot = { + rootPid: 1234, + rootIdentity: "1234@Mon Jul 27 00:00:00 2026", + processGroupId: 1234, + members: [ + { + pid: 1234, + parentPid: 1, + processGroupId: 1234, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "1234@Mon Jul 27 00:00:00 2026", + depth: 0 + }, + { + pid: 1235, + parentPid: 1234, + processGroupId: 1235, + state: "S", + startedAt: "Mon Jul 27 00:00:01 2026", + identity: "1235@Mon Jul 27 00:00:01 2026", + depth: 1 + } + ] + }; + const cleanOutcome = await terminateProcessTree(1234, { + platform: "darwin", + ownershipSnapshot, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "", + stderr: "", + error: null + }; + }, + killImpl() { + throw new Error("no process should be signaled"); + } + }); + + assert.equal(cleanOutcome.verified, true); + assert.equal(cleanOutcome.degraded, false); + assert.deepEqual(cleanOutcome.survivors, []); + assert.deepEqual(cleanOutcome.survivorIdentities, []); +}); + +test("terminateProcessTree refuses a PID without persisted ownership", async () => { + const signals = []; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: "1234 1 1234 S Mon Jul 27 00:00:00 2026\n", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.attempted, true); + assert.equal(outcome.delivered, false); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.deepEqual(outcome.survivors, [1234]); + assert.deepEqual(outcome.survivorIdentities, []); +}); + +test("terminateProcessTree refuses an absent PID without persisted ownership", async () => { + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "", + stderr: "", + error: null + }; + }, + killImpl() { + throw new Error("a PID without persisted ownership must not be signaled"); + } + }); + + assert.equal(outcome.attempted, true); + assert.equal(outcome.delivered, false); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.deepEqual(outcome.survivors, [1234]); + assert.deepEqual(outcome.survivorIdentities, []); +}); + +test("terminateProcessTree refuses capture-failure cleanup without a live owner handle", async () => { + const identityObservedAtSpawn = "1234@Sun Jul 26 00:00:00 2026"; + const identityNowHoldingPid = "1234@Mon Jul 27 00:00:00 2026"; + const persistedRecord = { + ownershipCaptureFailed: true, + processIdentity: null, + ownershipSnapshot: null + }; + const signals = []; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: persistedRecord.processIdentity, + ownershipSnapshot: persistedRecord.ownershipSnapshot, + requireVerifiedOwnership: persistedRecord.ownershipCaptureFailed, + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: `1234 1 1234 S ${identityNowHoldingPid.slice(identityNowHoldingPid.indexOf("@") + 1)}\n`, + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.notEqual(identityNowHoldingPid, identityObservedAtSpawn); + assert.deepEqual(signals, []); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); +}); + +test("terminateProcessTree permits capture-failure cleanup with a live owner handle", async () => { + const identityObservedAtSpawn = "1234@Sun Jul 26 00:00:00 2026"; + const identityNowHoldingPid = "1234@Mon Jul 27 00:00:00 2026"; + const persistedRecord = { + ownershipCaptureFailed: true, + processIdentity: null, + ownershipSnapshot: null + }; + const signals = []; + let alive = true; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: persistedRecord.processIdentity, + ownershipSnapshot: persistedRecord.ownershipSnapshot, + requireVerifiedOwnership: persistedRecord.ownershipCaptureFailed, + ownerHoldsLiveHandle: true, + pollIntervalMs: 0, + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: alive + ? `1234 1 1234 S ${identityNowHoldingPid.slice(identityNowHoldingPid.indexOf("@") + 1)}\n` + : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive = false; + } + }); + + assert.notEqual(identityNowHoldingPid, identityObservedAtSpawn); + assert.deepEqual(signals, [[-1234, "SIGTERM"]]); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); +}); + +test("terminateProcessTree revalidates descendant identities before signaling", async () => { + const signals = []; + let rootAlive = true; + let snapshots = 0; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", + termPollAttempts: 1, + killPollAttempts: 1, + sleepImpl() {}, + runCommandImpl(command, args) { + snapshots += 1; + const root = rootAlive ? "1234 1 1234 S Mon Jul 27 00:00:00 2026\n" : ""; + const childStart = snapshots === 1 ? "Mon Jul 27 00:00:01 2026" : "Mon Jul 27 00:01:01 2026"; + return { + command, + args, + status: 0, + signal: null, + stdout: `${root}1235 1234 1235 S ${childStart}\n`, + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + if (pid === -1234) { + rootAlive = false; + } + } + }); + + assert.deepEqual(signals, [[-1234, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.equal(outcome.escalated, false); +}); + +test("terminateProcessTree escalates resistant descendants and verifies exit", async () => { + const signals = []; + const alive = new Set([1234, 1235]); + const pgids = new Map([[1234, 1234], [1235, 1235]]); + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", + termPollAttempts: 1, + killPollAttempts: 1, + sleepImpl() {}, + runCommandImpl(command, args) { + const stdout = [ + alive.has(1234) ? "1234 1 1234 S Mon Jul 27 00:00:00 2026" : null, + alive.has(1235) ? "1235 1234 1235 S Mon Jul 27 00:00:01 2026" : null + ].filter(Boolean).join("\n"); + return { + command, + args, + status: 0, + signal: null, + stdout: stdout ? `${stdout}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + if (signal !== "SIGKILL") { + return; + } + const targetPgid = pid < 0 ? -pid : pgids.get(pid); + for (const candidate of [...alive]) { + if (candidate === pid || pgids.get(candidate) === targetPgid) { + alive.delete(candidate); + } + } + } + }); + + assert.deepEqual(signals, [ + [-1235, "SIGTERM"], + [-1235, "SIGKILL"], + [-1234, "SIGTERM"], + [-1234, "SIGKILL"] + ]); + assert.equal(outcome.verified, true); + assert.equal(outcome.escalated, true); +}); + +test("terminateProcessTree keeps the root alive until its descendants are reaped", async () => { + const signals = []; + const alive = new Set([100, 200]); + const outcome = await terminateProcessTree(100, { + platform: "darwin", + expectedRootIdentity: "100@Mon Jul 27 00:00:00 2026", + pollIntervalMs: 0, + runCommandImpl(command, args) { + const rows = []; + if (alive.has(100)) { + rows.push("100 1 100 S Mon Jul 27 00:00:00 2026"); + } + if (alive.has(200)) { + rows.push("200 100 200 S Mon Jul 27 00:00:01 2026"); + } + return { + command, + args, + status: 0, + signal: null, + stdout: rows.length ? `${rows.join("\n")}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + // The helper resists SIGTERM so the descendant phase must escalate + // before the root may be signaled at all. + if (Math.abs(pid) === 200 && signal === "SIGTERM") { + return; + } + alive.delete(Math.abs(pid)); + } + }); + + assert.deepEqual(signals, [ + [-200, "SIGTERM"], + [-200, "SIGKILL"], + [-100, "SIGTERM"] + ]); + assert.equal(outcome.verified, true); + assert.equal(outcome.escalated, true); +}); + +test("terminateProcessTree tracks reparented members of a signaled process group", async () => { + async function runScenario(helperSurvivesKill) { + const signals = []; + const alive = new Set([100, 200]); + const outcome = await terminateProcessTree(100, { + platform: "darwin", + expectedRootIdentity: "100@Mon Jul 27 00:00:00 2026", + termPollAttempts: 1, + killPollAttempts: 1, + sleepImpl() {}, + runCommandImpl(command, args) { + const rows = []; + if (alive.has(100)) { + rows.push("100 1 100 S Mon Jul 27 00:00:00 2026"); + } + if (alive.has(200)) { + rows.push("200 1 100 S Mon Jul 27 00:00:01 2026"); + } + return { + command, + args, + status: 0, + signal: null, + stdout: rows.length ? `${rows.join("\n")}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + if (signal === "SIGTERM") { + alive.delete(100); + } else if (!helperSurvivesKill) { + alive.delete(200); + } + } + }); + + assert.deepEqual(signals, [[-100, "SIGTERM"], [200, "SIGKILL"]]); + assert.equal(outcome.escalated, true); + assert.equal(outcome.verified, !helperSurvivesKill); + assert.deepEqual(outcome.survivors, helperSurvivesKill ? [200] : []); + } + + await runScenario(false); + await runScenario(true); +}); + +test("terminateProcessGroup reclaims orphaned members of a dead leader's group", async () => { + const signals = []; + const alive = new Set([202]); + const outcome = await terminateProcessGroup(200, { + platform: "darwin", + pollIntervalMs: 0, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: alive.has(202) ? "202 1 200 S Mon Jul 27 00:00:02 2026\n" : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive.delete(Math.abs(pid)); + } + }); + + assert.deepEqual(signals, [[202, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.equal(outcome.method, "process-group"); + assert.deepEqual(outcome.targets, [202]); +}); + +test("terminateProcessGroup converges when an owned group is already absent", async () => { + const signals = []; + const ownershipSnapshot = { + rootPid: 200, + rootIdentity: "200@Mon Jul 27 00:00:00 2026", + processGroupId: 200, + members: [ + { + pid: 200, + parentPid: 1, + processGroupId: 200, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "200@Mon Jul 27 00:00:00 2026", + depth: 0 + } + ] + }; + const outcome = await terminateProcessGroup(200, { + platform: "darwin", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.verified, true); + assert.equal(outcome.degraded, false); + assert.deepEqual(outcome.survivors, []); +}); + +test("terminateProcessGroup hunts an observed regrouped helper after its root exits", async () => { + const signals = []; + const alive = new Set([200]); + const ownershipSnapshot = { + rootPid: 100, + rootIdentity: "100@Mon Jul 27 00:00:00 2026", + processGroupId: 100, + members: [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "100@Mon Jul 27 00:00:00 2026", + depth: 0 + }, + { + pid: 200, + parentPid: 100, + processGroupId: 200, + state: "S", + startedAt: "Mon Jul 27 00:00:01 2026", + identity: "200@Mon Jul 27 00:00:01 2026", + depth: 1 + } + ] + }; + const outcome = await terminateProcessGroup(100, { + platform: "darwin", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: alive.has(200) ? "200 1 200 S Mon Jul 27 00:00:01 2026\n" : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive.delete(Math.abs(pid)); + } + }); + + assert.deepEqual(signals, [[-200, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.equal(outcome.degraded, false); + assert.deepEqual(outcome.survivors, []); +}); + +test("terminateProcessGroup excludes a reused snapshot PID", async () => { + const signals = []; + const ownershipSnapshot = { + rootPid: 100, + rootIdentity: "100@Mon Jul 27 00:00:00 2026", + processGroupId: 100, + members: [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "100@Mon Jul 27 00:00:00 2026", + depth: 0 + }, + { + pid: 200, + parentPid: 100, + processGroupId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:01 2026", + identity: "200@Mon Jul 27 00:00:01 2026", + depth: 1 + } + ] + }; + const outcome = await terminateProcessGroup(100, { + platform: "darwin", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: "200 1 100 S Mon Jul 27 00:01:01 2026\n", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); +}); + +test("terminateProcessGroup reclaims a post-snapshot member of the owned group", async () => { + const signals = []; + let alive = true; + const ownershipSnapshot = { + rootPid: 100, + rootIdentity: "100@Mon Jul 27 00:00:00 2026", + processGroupId: 100, + members: [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "100@Mon Jul 27 00:00:00 2026", + depth: 0 + } + ] + }; + const outcome = await terminateProcessGroup(100, { + platform: "darwin", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: alive ? "300 1 100 S Mon Jul 27 00:00:02 2026\n" : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive = false; + } + }); + + assert.deepEqual(signals, [[300, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.equal(outcome.degraded, false); + assert.deepEqual(outcome.survivors, []); +}); + +test("terminateProcessGroup reclaims a post-activation group in the owned Unix session", async () => { + const signals = []; + const alive = new Set([100, 300]); + const ownershipSnapshot = { + rootPid: 100, + rootIdentity: "100@Mon Jul 27 00:00:00 2026", + processGroupId: 100, + sessionId: 100, + members: [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + sessionId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "100@Mon Jul 27 00:00:00 2026", + depth: 0 + } + ] + }; + const outcome = await terminateProcessGroup(100, { + platform: "linux", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl(command, args) { + assert.equal(command, "/bin/ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]); + const rows = []; + if (alive.has(100)) { + rows.push("100 1 100 100 S Mon Jul 27 00:00:00 2026"); + } + if (alive.has(300)) { + rows.push("300 1 300 100 S Mon Jul 27 00:00:02 2026"); + } + return { + command, + args, + status: 0, + signal: null, + stdout: rows.length > 0 ? `${rows.join("\n")}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + const groupId = Math.abs(pid); + for (const candidate of [...alive]) { + if (candidate === groupId) { + alive.delete(candidate); + } + } + } + }); + + assert.deepEqual(signals, [[-300, "SIGTERM"], [-100, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.equal(outcome.degraded, false); + assert.equal(outcome.method, "process-session"); + assert.deepEqual(outcome.survivors, []); +}); + +test("terminateProcessGroup signals nothing when the group id was reused by another leader", async () => { + const signals = []; + const ownershipSnapshot = { + rootPid: 100, + rootIdentity: "100@Mon Jul 27 00:00:00 2026", + processGroupId: 100, + members: [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "100@Mon Jul 27 00:00:00 2026", + depth: 0 + } + ] + }; + // pid 100 now belongs to an unrelated process that leads its own group, and + // pid 301 is a member of that stranger's group. 301 is absent from the + // snapshot, so the per-record check cannot exclude it. + const outcome = await terminateProcessGroup(100, { + platform: "darwin", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: + "100 1 100 S Tue Jul 28 09:00:00 2026\n301 100 100 S Tue Jul 28 09:00:01 2026\n", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.deepEqual(outcome.survivors, [100]); +}); diff --git a/tests/registered-broker-reaper.test.mjs b/tests/registered-broker-reaper.test.mjs new file mode 100644 index 000000000..1d119cdcb --- /dev/null +++ b/tests/registered-broker-reaper.test.mjs @@ -0,0 +1,374 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + acquireBrokerRegistryLock as acquireBrokerRegistryLockImpl, + loadBrokerChildren, + publishBrokerChild as publishBrokerChildImpl, + publishBrokerRegistration, + registerBrokerOwner as registerBrokerOwnerImpl, + releaseBrokerOwner as releaseBrokerOwnerImpl, + releaseBrokerRegistryLock +} from "../plugins/codex/scripts/lib/broker-ownership.mjs"; +import { runRegisteredBrokerReaper as runRegisteredBrokerReaperImpl } from "../plugins/codex/scripts/registered-broker-reaper.mjs"; + +const TEST_LOCK_PID = 4950; +const TEST_LOCK_IDENTITY = "4950@Mon Jul 27 00:00:50 2026"; + +function withTestRegistryLock(options = {}) { + return { + pid: options.pid ?? TEST_LOCK_PID, + pidIdentity: options.pidIdentity ?? TEST_LOCK_IDENTITY, + hasLiveProcessIdentityImpl: () => true, + ...options + }; +} + +function acquireBrokerRegistryLock(registration, options = {}) { + return acquireBrokerRegistryLockImpl(registration, withTestRegistryLock(options)); +} + +function publishBrokerChild(registration, options = {}) { + return publishBrokerChildImpl(registration, withTestRegistryLock(options)); +} + +function registerBrokerOwner(registration, options = {}) { + return registerBrokerOwnerImpl(registration, withTestRegistryLock(options)); +} + +function releaseBrokerOwner(registration, options = {}) { + return releaseBrokerOwnerImpl(registration, withTestRegistryLock(options)); +} + +function runRegisteredBrokerReaper(options = {}) { + return runRegisteredBrokerReaperImpl({ + ...options, + acquireBrokerRegistryLockImpl: + options.acquireBrokerRegistryLockImpl ?? ((registration) => acquireBrokerRegistryLock(registration)) + }); +} + +function ownerEnv(env, sessionId, pid) { + const startedAt = `Mon Jul 27 00:${String(pid % 60).padStart(2, "0")}:00 2026`; + return { + ...env, + CODEX_COMPANION_SESSION_ID: sessionId, + CODEX_COMPANION_SESSION_OWNER_PID: String(pid), + CODEX_COMPANION_SESSION_OWNER_IDENTITY: `${pid}@${startedAt}` + }; +} + +function snapshot(pid, minute) { + const startedAt = `Mon Jul 27 01:${String(minute).padStart(2, "0")}:00 2026`; + const identity = `${pid}@${startedAt}`; + return { + rootPid: pid, + rootIdentity: identity, + processGroupId: pid, + members: [ + { + pid, + parentPid: 1, + processGroupId: pid, + state: "S", + startedAt, + identity, + depth: 0 + } + ] + }; +} + +function makeFixture(t, { child = false } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "registered-broker-reaper-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = { CLAUDE_PLUGIN_DATA: root }; + const brokerSnapshot = snapshot(4100, 1); + const registration = publishBrokerRegistration({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: brokerSnapshot.rootPid, + ownershipSnapshot: brokerSnapshot, + env, + now: () => "2026-07-27T01:01:00.000Z" + }); + assert.equal(registration.registered, true); + let childRecord = null; + if (child) { + childRecord = publishBrokerChild(registration, { + ownershipSnapshot: snapshot(4200, 2), + now: () => "2026-07-27T01:02:00.000Z" + }).child; + } + return { root, env, registration, childRecord }; +} + +function addReleasedOwner(registration, env, sessionId = "session-released", pid = 5100) { + const sessionEnv = ownerEnv(env, sessionId, pid); + assert.equal(registerBrokerOwner(registration, { env: sessionEnv }).registered, true); + assert.equal(releaseBrokerOwner(registration, { env: sessionEnv }).released, true); +} + +test("report-only mode never calls a cleanup function for an eligible registry", async (t) => { + const { env, registration } = makeFixture(t); + addReleasedOwner(registration, env); + let cleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + env, + terminateProcessGroupImpl: async () => { cleanupCalls += 1; }, + terminateProcessTreeImpl: async () => { cleanupCalls += 1; } + }); + assert.equal(summary.mode, "report-only"); + assert.equal(summary.reaped, 0); + assert.equal(summary.results[0].reason, "eligible-but-apply-not-enabled"); + assert.equal(cleanupCalls, 0); +}); + +test("one live owner blocks cleanup even when every other owner is dead", async (t) => { + const { env, registration } = makeFixture(t); + const live = ownerEnv(env, "session-live", 5200); + const dead = ownerEnv(env, "session-dead", 5300); + registerBrokerOwner(registration, { env: live }); + registerBrokerOwner(registration, { env: dead }); + let cleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl(pids, options) { + return pids.filter((pid) => pid === 5200 && options.identities.some((identity) => identity.startsWith("5200@"))); + }, + terminateProcessTreeImpl: async () => { cleanupCalls += 1; } + }); + assert.equal(summary.results[0].reason, "live-owner"); + assert.equal(cleanupCalls, 0); +}); + +test("registered cleanup reclaims children before the broker and writes a mode-0600 receipt", async (t) => { + const { env, registration, childRecord } = makeFixture(t, { child: true }); + addReleasedOwner(registration, env); + const calls = []; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessGroupImpl: async (pgid, options) => { + calls.push(`group:${pgid}:${options.ownershipSnapshot.rootIdentity}`); + return { attempted: true, delivered: true, verified: true, degraded: false, targetIdentities: [childRecord.pidIdentity] }; + }, + terminateProcessTreeImpl: async (pid, options) => { + calls.push(`tree:${pid}:${options.expectedRootIdentity}`); + return { attempted: true, delivered: true, verified: true, degraded: false, targetIdentities: [options.expectedRootIdentity] }; + }, + attemptIdFactory: () => "attempt-success", + now: () => "2026-07-27T02:00:00.000Z" + }); + assert.equal(summary.reaped, 1); + assert.deepEqual(calls, [ + `group:4200:${childRecord.pidIdentity}`, + `tree:4100:${registration.broker.pidIdentity}` + ]); + const receiptPath = summary.results[0].receiptPath; + assert.ok(receiptPath); + assert.equal(fs.statSync(receiptPath).mode & 0o777, 0o600); + assert.equal(JSON.parse(fs.readFileSync(receiptPath, "utf8")).decision, "cleanup-verified"); + + const receiptNames = fs.readdirSync(path.dirname(receiptPath)); + const repeated = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessGroupImpl: async () => { + throw new Error("a terminal registry must not retry child cleanup"); + }, + terminateProcessTreeImpl: async () => { + throw new Error("a terminal registry must not retry broker cleanup"); + }, + attemptIdFactory: () => "attempt-should-not-run" + }); + assert.equal(repeated.scanned, 0); + assert.deepEqual(fs.readdirSync(path.dirname(receiptPath)), receiptNames); +}); + +test("each verified child is released before a later target can block convergence", async (t) => { + const { env, registration } = makeFixture(t, { child: true }); + publishBrokerChild(registration, { + ownershipSnapshot: snapshot(4300, 3), + now: () => "2026-07-27T01:03:00.000Z" + }); + addReleasedOwner(registration, env); + const orderedChildren = loadBrokerChildren(registration).children; + assert.equal(orderedChildren.length, 2); + const firstProcessedChild = orderedChildren[0]; + const blockedChild = orderedChildren[1]; + + let firstPassGroupCalls = 0; + let firstPassBrokerCalls = 0; + const firstPass = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessGroupImpl: async () => { + firstPassGroupCalls += 1; + if (firstPassGroupCalls === 1) { + return { attempted: true, delivered: true, verified: true, degraded: false, survivors: [], survivorIdentities: [] }; + } + return { attempted: true, delivered: true, verified: false, degraded: true, survivors: [blockedChild.pid], survivorIdentities: [blockedChild.pidIdentity] }; + }, + terminateProcessTreeImpl: async () => { firstPassBrokerCalls += 1; }, + attemptIdFactory: () => "attempt-partial-child-release" + }); + assert.equal(firstPass.reaped, 0); + assert.equal(firstPassGroupCalls, 2); + assert.equal(firstPassBrokerCalls, 0); + const afterFirstPass = loadBrokerChildren(registration); + assert.equal(afterFirstPass.valid, true); + assert.deepEqual(afterFirstPass.releasedChildren.map((child) => child.childKey), [firstProcessedChild.childKey]); + assert.deepEqual(afterFirstPass.children.map((child) => child.childKey), [blockedChild.childKey]); + + const secondPassCalls = []; + const secondPass = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessGroupImpl: async (pgid) => { + secondPassCalls.push(`group:${pgid}`); + return { attempted: true, delivered: true, verified: true, degraded: false, survivors: [], survivorIdentities: [] }; + }, + terminateProcessTreeImpl: async (pid) => { + secondPassCalls.push(`tree:${pid}`); + return { attempted: true, delivered: true, verified: true, degraded: false, survivors: [], survivorIdentities: [] }; + }, + attemptIdFactory: () => "attempt-converged-child-release" + }); + assert.equal(secondPass.reaped, 1); + assert.deepEqual(secondPassCalls, [`group:${blockedChild.processGroupId}`, `tree:${registration.broker.pid}`]); +}); + +test("a contended registry lock closes the registration-to-signal race", async (t) => { + const { env, registration } = makeFixture(t); + addReleasedOwner(registration, env); + const lock = acquireBrokerRegistryLock(registration); + t.after(() => releaseBrokerRegistryLock(registration, lock)); + let cleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessTreeImpl: async () => { cleanupCalls += 1; } + }); + assert.equal(summary.results[0].reason, "registry-busy"); + assert.equal(cleanupCalls, 0); +}); + +test("the broker record is revalidated after the cleanup lock is acquired", async (t) => { + const { env, registration } = makeFixture(t); + addReleasedOwner(registration, env); + let cleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + acquireBrokerRegistryLockImpl(candidate) { + const lock = acquireBrokerRegistryLock(candidate); + fs.unlinkSync(path.join(candidate.registryDir, "broker.json")); + return lock; + }, + terminateProcessTreeImpl: async () => { cleanupCalls += 1; } + }); + assert.equal(summary.results[0].reason, "locked-broker-record-invalid"); + assert.equal(cleanupCalls, 0); +}); + +test("a reused or ambiguous child group blocks broker cleanup", async (t) => { + const { env, registration } = makeFixture(t, { child: true }); + addReleasedOwner(registration, env); + let brokerCleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: (pids) => pids, + terminateProcessGroupImpl: async () => ({ + attempted: false, + delivered: false, + verified: false, + degraded: true, + identityMismatch: true + }), + terminateProcessTreeImpl: async () => { brokerCleanupCalls += 1; }, + attemptIdFactory: () => "attempt-child-reused" + }); + assert.equal(summary.reaped, 0); + assert.equal(summary.results[0].reason, "cleanup-unverified"); + assert.equal(brokerCleanupCalls, 0); + assert.ok(summary.results[0].residualIdentities.length > 0); +}); + +test("malformed owner or child state is report-only", async (t) => { + const ownerFixture = makeFixture(t); + fs.mkdirSync(path.join(ownerFixture.registration.registryDir, "owners"), { recursive: true }); + fs.writeFileSync(path.join(ownerFixture.registration.registryDir, "owners", "bad.json"), "{}\n", "utf8"); + const ownerSummary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env: ownerFixture.env, + getLiveProcessPidsImpl: () => [] + }); + assert.equal(ownerSummary.results[0].reason, "malformed-registry"); + + const childFixture = makeFixture(t); + addReleasedOwner(childFixture.registration, childFixture.env); + fs.mkdirSync(path.join(childFixture.registration.registryDir, "children"), { recursive: true }); + fs.writeFileSync(path.join(childFixture.registration.registryDir, "children", "bad.json"), "{}\n", "utf8"); + const childSummary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env: childFixture.env, + getLiveProcessPidsImpl: () => [] + }); + assert.equal(childSummary.results[0].reason, "malformed-child-registry"); +}); + +test("missing and unregistered broker rows never reach a signal path", async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "registered-broker-reaper-missing-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = { CLAUDE_PLUGIN_DATA: root }; + const registryRoot = path.join(root, "state", "broker-ownership-v1"); + const missingBrokerDir = path.join(registryRoot, "a".repeat(64)); + fs.mkdirSync(missingBrokerDir, { recursive: true, mode: 0o700 }); + fs.chmodSync(registryRoot, 0o700); + fs.chmodSync(missingBrokerDir, 0o700); + let cleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + terminateProcessTreeImpl: async () => { cleanupCalls += 1; } + }); + assert.equal(summary.results[0].reason, "broker-record-invalid"); + assert.equal(cleanupCalls, 0); + + const emptyRoot = fs.mkdtempSync(path.join(os.tmpdir(), "registered-broker-reaper-empty-")); + t.after(() => fs.rmSync(emptyRoot, { recursive: true, force: true })); + const empty = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env: { CLAUDE_PLUGIN_DATA: emptyRoot } + }); + assert.equal(empty.scanned, 0); +}); + +test("an overly permissive registry root is report-only", async (t) => { + const { env, registration } = makeFixture(t); + addReleasedOwner(registration, env); + const registryRoot = path.dirname(registration.registryDir); + fs.chmodSync(registryRoot, 0o755); + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessTreeImpl: async () => { + throw new Error("invalid roots must not reach cleanup"); + } + }); + assert.equal(summary.reaped, 0); + assert.equal(summary.results[0].reason, "registry-root-invalid"); +}); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 8f276835b..5928706e2 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -1,4 +1,5 @@ import fs from "node:fs"; +import net from "node:net"; import path from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; @@ -6,15 +7,1759 @@ import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; -import { initGitRepo, makeTempDir, run } from "./helpers.mjs"; -import { loadBrokerSession, saveBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; -import { resolveStateDir } from "../plugins/codex/scripts/lib/state.mjs"; +import { initGitRepo, makeTempDir as createTempDir, run } from "./helpers.mjs"; +import { enqueueBackgroundTask, handleCancel, handleTaskWorker } from "../plugins/codex/scripts/codex-companion.mjs"; +import { cleanupSessionJobs, handleSessionEnd, handleSessionStart } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; +import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; +import { isBrokerRequestAllowedDuringShutdown } from "../plugins/codex/scripts/app-server-broker.mjs"; +import { acquireBrokerRegistryLock, loadBrokerChildren, loadBrokerRegistration, publishBrokerChild, publishRegisteredBroker, registerBrokerOwner, releaseBrokerOwner, releaseBrokerRegistryLock, SESSION_OWNER_IDENTITY_ENV, SESSION_OWNER_PID_ENV } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; +import { acquireBrokerLaunchLock, activateBrokerProcess, brokerLaunchLockPort, clearBrokerSession, ensureBrokerSession, loadBrokerSession, loadReusableBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession, waitForBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { readStoredJob } from "../plugins/codex/scripts/lib/job-control.mjs"; +import { captureProcessOwnership, getProcessIdentity, hasLiveProcessIdentity, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; +import { hasCancelFlag, listJobs, loadState, resolveStateDir, resolveStateFile, saveState, upsertJob, writeCancelFlag, writeJobFile } from "../plugins/codex/scripts/lib/state.mjs"; +import { runTrackedJob } from "../plugins/codex/scripts/lib/tracked-jobs.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const PLUGIN_ROOT = path.join(ROOT, "plugins", "codex"); const SCRIPT = path.join(PLUGIN_ROOT, "scripts", "codex-companion.mjs"); const STOP_HOOK = path.join(PLUGIN_ROOT, "scripts", "stop-review-gate-hook.mjs"); const SESSION_HOOK = path.join(PLUGIN_ROOT, "scripts", "session-lifecycle-hook.mjs"); +const BROKER_SCRIPT = path.join(PLUGIN_ROOT, "scripts", "app-server-broker.mjs"); +const DETACHED_FIXTURE_TTL_MS = 5 * 60 * 1000; +const SELF_EXPIRING_KEEPALIVE = selfExpiringKeepaliveCode(); +const runtimeTempDirs = new Set(); +const runtimePluginDataDir = createTempDir("codex-plugin-runtime-state-"); +process.env.CLAUDE_PLUGIN_DATA = runtimePluginDataDir; +let brokerOwnerSequence = 0; + +function makeTempDir(prefix) { + const tempDir = createTempDir(prefix); + runtimeTempDirs.add(tempDir); + return tempDir; +} + +function withBrokerOwner(env, label) { + brokerOwnerSequence += 1; + return { + ...env, + CODEX_COMPANION_SESSION_ID: `runtime-${label}-${brokerOwnerSequence}`, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: getProcessIdentity(process.pid) + }; +} + +function selfExpiringKeepaliveCode(ttlMs = DETACHED_FIXTURE_TTL_MS) { + return `setTimeout(() => process.exit(0), ${ttlMs}); setInterval(() => {}, 1000)`; +} + +test.after(async () => { + const cleanupFailures = []; + + for (const cwd of [ROOT, ...runtimeTempDirs]) { + const brokerSession = loadBrokerSession(cwd); + if (!brokerSession) { + continue; + } + + await sendBrokerShutdown(brokerSession.endpoint).catch(() => {}); + const cleanup = await teardownBrokerSession({ + endpoint: brokerSession.endpoint, + pidFile: brokerSession.pidFile, + logFile: brokerSession.logFile, + sessionDir: brokerSession.sessionDir, + pid: brokerSession.pid, + pidIdentity: brokerSession.pidIdentity, + ownershipSnapshot: brokerSession.ownershipSnapshot, + requireVerifiedOwnership: brokerSession.ownershipCaptureFailed === true, + killProcess: terminateProcessTree + }); + if (cleanup?.verified === true) { + clearBrokerSession(cwd); + } + if (cleanup?.verified !== true || loadBrokerSession(cwd)) { + cleanupFailures.push({ cwd, cleanup }); + } + } + + assert.deepEqual(cleanupFailures, []); +}); + +test("broker rejects queued work after shutdown begins", () => { + assert.equal(isBrokerRequestAllowedDuringShutdown(true, { id: 2, method: "thread/list" }), false); + assert.equal(isBrokerRequestAllowedDuringShutdown(true, { id: 3, method: "broker/shutdown" }), true); + assert.equal(isBrokerRequestAllowedDuringShutdown(false, { id: 4, method: "thread/list" }), true); +}); + +test("SessionStart exports the stable process-group owner identity and runs registered cleanup", async () => { + const envFile = path.join(makeTempDir(), "session.env"); + const previousEnvFile = process.env.CLAUDE_ENV_FILE; + process.env.CLAUDE_ENV_FILE = envFile; + try { + let reaperMode = null; + await handleSessionStart( + { + session_id: "session-owner-export", + transcript_path: "/tmp/transcript.jsonl", + cwd: ROOT + }, + { + pid: 7101, + platform: "darwin", + captureStableSessionOwnerImpl() { + return { + pid: 7100, + identity: "7100@Mon Jul 27 00:07:00 2026", + processGroupId: 7100 + }; + }, + async runRegisteredBrokerReaperImpl(options) { + reaperMode = options.mode; + return { mode: options.mode, scanned: 0, reaped: 0, reported: 0, results: [] }; + } + } + ); + assert.equal(reaperMode, "apply-registered"); + } finally { + if (previousEnvFile == null) { + delete process.env.CLAUDE_ENV_FILE; + } else { + process.env.CLAUDE_ENV_FILE = previousEnvFile; + } + } + + const source = fs.readFileSync(envFile, "utf8"); + assert.match(source, /export CODEX_COMPANION_SESSION_ID='session-owner-export'/); + assert.match(source, new RegExp(`export ${SESSION_OWNER_PID_ENV}='7100'`)); + assert.match(source, new RegExp(`export ${SESSION_OWNER_IDENTITY_ENV}='7100@Mon Jul 27 00:07:00 2026'`)); +}); + +test("SessionEnd leaves a shared broker untouched while another registered owner is live", async () => { + const calls = []; + await handleSessionEnd( + { cwd: ROOT, session_id: "session-ending" }, + { + env: {}, + loadBrokerSessionImpl() { + return { + endpoint: "unix:/tmp/shared-broker.sock", + pid: 4100, + pidIdentity: "4100@Mon Jul 27 00:00:00 2026", + registry: { + registered: true, + version: 1, + brokerKey: "a".repeat(64), + registryRoot: "/tmp/registry", + registryDir: `/tmp/registry/${"a".repeat(64)}` + } + }; + }, + acquireBrokerRegistryLockImpl() { + calls.push("acquire-lock"); + return { acquired: true }; + }, + loadBrokerRegistrationImpl() { + calls.push("validate-registry"); + return { + registered: true, + version: 1, + brokerKey: "a".repeat(64), + registryRoot: "/tmp/registry", + registryDir: `/tmp/registry/${"a".repeat(64)}` + }; + }, + releaseBrokerOwnerImpl() { + calls.push("release-owner"); + return { released: true }; + }, + assessBrokerOwnersImpl() { + return { safeToShutdown: false, reason: "live-owner", liveOwners: [{ sessionId: "session-other" }] }; + }, + sendBrokerShutdownImpl() { + calls.push("shutdown-broker"); + }, + cleanupSessionJobsImpl() { + calls.push("cleanup-jobs"); + return { verified: true, failures: [] }; + }, + teardownBrokerSessionImpl() { + calls.push("teardown-broker"); + return { verified: true }; + }, + clearBrokerSessionImpl() { + calls.push("clear-broker"); + }, + releaseBrokerRegistryLockImpl() { + calls.push("release-lock"); + return { released: true }; + } + } + ); + + assert.deepEqual(calls, ["acquire-lock", "validate-registry", "release-owner", "release-lock", "cleanup-jobs"]); +}); + +test("SessionEnd leaves an unregistered or externally supplied broker report-only", async () => { + const calls = []; + await handleSessionEnd( + { cwd: ROOT, session_id: "session-unregistered" }, + { + loadBrokerSessionImpl() { + return { + endpoint: "unix:/tmp/unregistered-broker.sock", + pid: 4150, + pidIdentity: "4150@Mon Jul 27 00:00:50 2026" + }; + }, + sendBrokerShutdownImpl() { + calls.push("shutdown-broker"); + }, + teardownBrokerSessionImpl() { + calls.push("teardown-broker"); + return { verified: true }; + }, + clearBrokerSessionImpl() { + calls.push("clear-broker"); + }, + cleanupSessionJobsImpl() { + calls.push("cleanup-jobs"); + return { verified: true, failures: [] }; + } + } + ); + + assert.deepEqual(calls, ["cleanup-jobs"]); +}); + +test("detached fixture keepalive self-expires when parent cleanup is skipped", async (t) => { + if (process.platform === "win32") { + t.skip("Unix detached fixture behavior is not available on Windows."); + return; + } + + const sleeper = spawn(process.execPath, ["-e", selfExpiringKeepaliveCode(500)], { + detached: true, + stdio: "ignore" + }); + sleeper.unref(); + assert.doesNotThrow(() => process.kill(sleeper.pid, 0)); + + await waitFor(() => { + try { + process.kill(sleeper.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }, { timeoutMs: 3000 }); +}); + +test("background task is persisted before its worker can start", () => { + const workspace = makeTempDir(); + const job = { + id: "task-persist-before-spawn", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Exercise the startup race", + createdAt: "2026-07-28T08:00:00.000Z" + }; + const request = { + cwd: workspace, + prompt: "Exercise the startup race", + jobId: job.id + }; + const workerProgress = { + status: "running", + phase: "investigating", + pid: 43210, + progressMarker: "worker-read-succeeded" + }; + let observedQueuedRecord = null; + + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl() { + observedQueuedRecord = readStoredJob(workspace, job.id); + writeJobFile(workspace, job.id, { + ...observedQueuedRecord, + ...workerProgress + }); + upsertJob(workspace, { + id: job.id, + ...workerProgress + }); + } + }); + + assert.equal(observedQueuedRecord.status, "queued"); + assert.equal(observedQueuedRecord.phase, "queued"); + assert.equal(observedQueuedRecord.pid, null); + assert.deepEqual(observedQueuedRecord.request, request); + assert.equal(Object.hasOwn(observedQueuedRecord, "processIdentity"), false); + assert.equal(Object.hasOwn(observedQueuedRecord, "ownershipSnapshot"), false); + assert.equal(Object.hasOwn(observedQueuedRecord, "ownershipCaptureFailed"), false); + + const storedJob = readStoredJob(workspace, job.id); + assert.equal(storedJob.status, workerProgress.status); + assert.equal(storedJob.phase, workerProgress.phase); + assert.equal(storedJob.pid, workerProgress.pid); + assert.equal(storedJob.progressMarker, workerProgress.progressMarker); + + const indexedJob = listJobs(workspace).find((candidate) => candidate.id === job.id); + assert.equal(indexedJob.status, workerProgress.status); + assert.equal(indexedJob.phase, workerProgress.phase); + assert.equal(indexedJob.pid, workerProgress.pid); + assert.equal(indexedJob.progressMarker, workerProgress.progressMarker); +}); + +test("background task marks the queued record failed when worker spawn throws", () => { + const workspace = makeTempDir(); + const job = { + id: "task-sync-spawn-failure", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Exercise synchronous spawn failure", + createdAt: "2026-07-30T03:00:00.000Z" + }; + const request = { + cwd: workspace, + prompt: "Exercise synchronous spawn failure", + jobId: job.id + }; + const spawnError = Object.assign(new Error("spawn EAGAIN"), { code: "EAGAIN" }); + + assert.throws( + () => + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl() { + throw spawnError; + } + }), + spawnError + ); + + const storedJob = readStoredJob(workspace, job.id); + assert.equal(storedJob.status, "failed"); + assert.equal(storedJob.phase, "failed"); + assert.equal(storedJob.pid, null); + assert.equal(storedJob.errorMessage, "spawn EAGAIN"); + assert.ok(storedJob.completedAt); + + const indexedJob = listJobs(workspace).find((candidate) => candidate.id === job.id); + assert.equal(indexedJob.status, "failed"); + assert.equal(indexedJob.phase, "failed"); + assert.equal(indexedJob.pid, null); + assert.equal(indexedJob.errorMessage, "spawn EAGAIN"); +}); + +test("background task marks the queued record failed when worker emits a spawn error", () => { + const workspace = makeTempDir(); + const job = { + id: "task-async-spawn-failure", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Exercise asynchronous spawn failure", + createdAt: "2026-07-30T03:01:00.000Z" + }; + const request = { + cwd: workspace, + prompt: "Exercise asynchronous spawn failure", + jobId: job.id + }; + let spawnErrorHandler = null; + const worker = { + once(event, handler) { + if (event === "error") { + spawnErrorHandler = handler; + } + return this; + } + }; + + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl() { + return worker; + } + }); + assert.equal(typeof spawnErrorHandler, "function"); + spawnErrorHandler(Object.assign(new Error("spawn EMFILE"), { code: "EMFILE" })); + + const storedJob = readStoredJob(workspace, job.id); + assert.equal(storedJob.status, "failed"); + assert.equal(storedJob.phase, "failed"); + assert.equal(storedJob.pid, null); + assert.equal(storedJob.errorMessage, "spawn EMFILE"); + assert.ok(storedJob.completedAt); + + const indexedJob = listJobs(workspace).find((candidate) => candidate.id === job.id); + assert.equal(indexedJob.status, "failed"); + assert.equal(indexedJob.phase, "failed"); + assert.equal(indexedJob.pid, null); + assert.equal(indexedJob.errorMessage, "spawn EMFILE"); +}); + +test("background task stays runnable when worker identity capture fails", async () => { + const workspace = makeTempDir(); + const job = { + id: "task-capture-failure", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Exercise ownership capture failure", + createdAt: "2026-07-28T08:01:00.000Z" + }; + const request = { + cwd: workspace, + prompt: "Exercise ownership capture failure", + jobId: job.id + }; + let workerJob = null; + let workPerformed = false; + + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl() {} + }); + await handleTaskWorker(["--cwd", workspace, "--job-id", job.id], { + getProcessIdentityImpl() { + throw new Error("injected ownership capture failure"); + }, + runTrackedJobImpl(candidate, _runner, options) { + workerJob = candidate; + return runTrackedJob( + candidate, + async () => { + workPerformed = true; + return { + exitStatus: 0, + payload: { ok: true }, + rendered: "Worker completed.", + summary: "Worker completed." + }; + }, + options + ); + } + }); + + assert.equal(workPerformed, true); + assert.equal(workerJob.ownershipCaptureFailed, true); + assert.equal(Object.hasOwn(workerJob, "processIdentity"), false); + assert.equal(Object.hasOwn(workerJob, "ownershipSnapshot"), false); + + const storedJob = readStoredJob(workspace, job.id); + assert.equal(storedJob.status, "completed"); + assert.equal(storedJob.ownershipCaptureFailed, true); + assert.equal(Object.hasOwn(storedJob, "processIdentity"), false); + assert.equal(Object.hasOwn(storedJob, "ownershipSnapshot"), false); +}); + +test("fake app-server crash reclaims an observed regrouped helper without replacement", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "crash-with-regrouped-helper"); + const env = buildEnv(binDir); + const child = spawn("codex", ["app-server"], { + cwd: repo, + env, + detached: true, + stdio: ["pipe", "pipe", "ignore"] + }); + child.stdout.setEncoding("utf8"); + let buffer = ""; + const initialized = new Promise((resolve, reject) => { + child.stdout.on("data", (chunk) => { + buffer += chunk; + const newline = buffer.indexOf("\n"); + if (newline === -1) { + return; + } + resolve(JSON.parse(buffer.slice(0, newline))); + }); + child.once("error", reject); + }); + child.stdin.write(`${JSON.stringify({ id: 1, method: "initialize", params: { capabilities: {} } })}\n`); + const response = await initialized; + assert.equal(response.id, 1); + await waitFor(() => fs.existsSync(fakeStatePath) && JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).helperPids?.length === 1); + const helperPid = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).helperPids[0]; + t.after(() => { + try { + process.kill(helperPid, "SIGKILL"); + } catch { + // Ignore the helper after cleanup. + } + }); + let ownershipSnapshot; + try { + ownershipSnapshot = captureProcessOwnership(child.pid, { env }); + } catch (error) { + if (error?.code === "PROCESS_TABLE_UNAVAILABLE") { + t.skip(`process table unavailable: ${error.message}`); + return; + } + throw error; + } + await new Promise((resolve) => child.once("exit", resolve)); + const outcome = await terminateProcessGroup(child.pid, { ownershipSnapshot, env }); + + assert.equal(outcome.verified, true); + assert.deepEqual(outcome.survivors, []); + assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 1); +}); + +test("shared broker durably observes and reclaims a post-activation regrouped helper", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "crash-with-post-activation-regrouped-helper"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" + }, "post-snapshot-helper"); + const brokerSession = await ensureBrokerSession(repo, { env }); + if (!brokerSession) { + t.skip("broker socket unavailable in this sandbox"); + return; + } + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + if (fs.existsSync(fakeStatePath)) { + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + for (const helperPid of state.helperPids || []) { + try { + process.kill(helperPid, "SIGKILL"); + } catch { + // Ignore helpers already terminated by group cleanup. + } + } + } + }); + + function isLive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } + } + + const client = await CodexAppServerClient.connect(repo, { env }); + await client.request("thread/list", { cwd: repo }); + await waitFor(() => { + if (!fs.existsSync(fakeStatePath)) { + return false; + } + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + return state.helperPids?.length === 1 && state.appServerPids?.length === 1; + }); + + const firstState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + const appServerPid = firstState.appServerPids[0]; + const helperPid = firstState.helperPids[0]; + await waitFor(() => !isLive(appServerPid)); + await waitFor(() => !isLive(helperPid)); + + const replacementResponse = await waitFor(async () => { + try { + const replacementClient = await CodexAppServerClient.connect(repo, { env }); + try { + return await replacementClient.request("thread/list", { cwd: repo }); + } finally { + await replacementClient.close(); + } + } catch (error) { + if (error?.rpcCode === -32002) { + return false; + } + throw error; + } + }); + + assert.ok(Array.isArray(replacementResponse.data)); + assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 2); +}); + +test("shared broker observes a helper spawned after a streaming response before forwarding notifications", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for helper ownership cleanup."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "streaming-helper-after-response"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" + }, "streaming-helper-observation"); + const brokerSession = await ensureBrokerSession(repo, { env }); + if (!brokerSession) { + t.skip("broker socket unavailable in this sandbox"); + return; + } + + let helperPid = null; + let appServerPid = null; + let registeredChild = null; + const client = await CodexAppServerClient.connect(repo, { env }); + t.after(async () => { + await client.close().catch(() => {}); + await sendBrokerShutdown(brokerSession.endpoint).catch(() => {}); + for (const pid of [helperPid, appServerPid, brokerSession.pid]) { + if (!Number.isFinite(pid)) { + continue; + } + try { + process.kill(pid, "SIGKILL"); + } catch { + // Ignore processes already reclaimed by the ownership cleanup path. + } + } + clearBrokerSession(repo); + }); + + const started = await client.request("thread/start", { cwd: repo, ephemeral: true }); + await waitFor(() => { + registeredChild = loadBrokerChildren(brokerSession.registry).children[0] ?? null; + appServerPid = registeredChild?.pid ?? null; + return Number.isFinite(appServerPid) && Boolean(registeredChild?.pidIdentity); + }); + + await client.request("turn/start", { + threadId: started.thread.id, + input: [{ type: "text", text: "spawn a helper after returning this response" }] + }); + await waitFor(() => { + if (!fs.existsSync(fakeStatePath)) { + return false; + } + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + helperPid = state.helperPids?.[0] ?? null; + return Number.isFinite(helperPid) && Number.isFinite(appServerPid); + }); + await waitFor(() => !hasLiveProcessIdentity(appServerPid, registeredChild.pidIdentity)); + await waitFor(() => { + try { + process.kill(helperPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }, { timeoutMs: 3000 }); +}); + +test("shared broker tears down a post-activation helper before reporting observation lock contention", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for helper ownership cleanup."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "post-activation-helper-on-thread-list"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" + }, "observation-contention"); + const brokerSession = await ensureBrokerSession(repo, { env }); + if (!brokerSession) { + t.skip("broker socket unavailable in this sandbox"); + return; + } + + let helperPid = null; + let appServerPid = null; + let registryLock = null; + const client = await CodexAppServerClient.connect(repo, { env }); + t.after(async () => { + await client.close().catch(() => {}); + if (registryLock?.acquired === true) { + const currentRegistration = loadBrokerRegistration({ + endpoint: brokerSession.endpoint, + brokerIdentity: brokerSession.pidIdentity, + env + }); + releaseBrokerRegistryLock(currentRegistration, registryLock); + registryLock = null; + } + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + for (const pid of [helperPid, appServerPid]) { + if (!pid) { + continue; + } + try { + process.kill(pid, "SIGKILL"); + } catch { + // Ignore fixture processes already reclaimed by the broker. + } + } + }); + + function isLive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } + } + + await client.request("account/read", {}); + const registration = loadBrokerRegistration({ + endpoint: brokerSession.endpoint, + brokerIdentity: brokerSession.pidIdentity, + env + }); + assert.equal(registration.registered, true); + registryLock = acquireBrokerRegistryLock(registration); + assert.equal(registryLock.acquired, true); + + try { + await assert.rejects( + client.request("thread/list", { cwd: repo }), + (error) => { + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + helperPid = state.helperPids?.[0] ?? null; + appServerPid = state.appServerPids?.[0] ?? null; + return ( + error?.rpcCode === -32005 && + /registry-busy/.test(error.message) && + helperPid !== null && + appServerPid !== null && + !isLive(helperPid) && + !isLive(appServerPid) + ); + } + ); + } finally { + assert.equal(releaseBrokerRegistryLock(registration, registryLock).released, true); + registryLock = null; + } + + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + assert.equal(state.appServerStarts, 1); + assert.equal(isLive(helperPid), false); + assert.equal(isLive(appServerPid), false); +}); + +test("automatic broker registration preserves a shared broker until its final owner releases", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + let ownerIdentity; + try { + ownerIdentity = getProcessIdentity(process.pid); + } catch (error) { + if (error?.code === "PROCESS_TABLE_UNAVAILABLE") { + t.skip(`process table unavailable: ${error.message}`); + return; + } + throw error; + } + assert.ok(ownerIdentity); + + const baseEnv = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: ownerIdentity + }; + const envA = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "registry-owner-a" }; + const envB = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "registry-owner-b" }; + const first = await ensureBrokerSession(repo, { env: envA }); + if (!first) { + t.skip("broker socket unavailable in this sandbox"); + return; + } + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envB, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "registry-owner-b" }) + }); + }); + + assert.equal(first.registry?.registered, true); + assert.equal(fs.statSync(path.join(first.registry.registryDir, "broker.json")).mode & 0o777, 0o600); + const second = await ensureBrokerSession(repo, { env: envB }); + assert.equal(second.pid, first.pid); + assert.equal(fs.readdirSync(path.join(first.registry.registryDir, "owners")).filter((name) => name.endsWith(".json")).length, 2); + + const client = await CodexAppServerClient.connect(repo, { env: envB }); + await client.request("thread/list", { cwd: repo }); + await client.close(); + await waitFor(() => { + const childrenDir = path.join(first.registry.registryDir, "children"); + return fs.existsSync(childrenDir) && fs.readdirSync(childrenDir).some((name) => name.endsWith(".json")); + }); + + const firstEnd = run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envA, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "registry-owner-a" }) + }); + assert.equal(firstEnd.status, 0, firstEnd.stderr); + assert.ok(loadBrokerSession(repo)); + assert.doesNotThrow(() => process.kill(first.pid, 0)); + + const finalEnd = run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envB, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "registry-owner-b" }) + }); + assert.equal(finalEnd.status, 0, finalEnd.stderr); + assert.equal(loadBrokerSession(repo), null); + await waitFor(() => { + try { + process.kill(first.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); +}); + +test("existing broker reuse holds the registry lock through owner publication", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const baseEnv = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: getProcessIdentity(process.pid) + }; + const envA = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "locked-reuse-a" }; + const envB = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "locked-reuse-b" }; + const first = await ensureBrokerSession(repo, { env: envA }); + let heldLock = null; + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envB, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "locked-reuse-b" }) + }); + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envA, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "locked-reuse-a" }) + }); + }); + + const second = await ensureBrokerSession(repo, { + env: envB, + acquireBrokerRegistryLockImpl(registration) { + heldLock = acquireBrokerRegistryLock(registration); + return heldLock; + }, + loadBrokerRegistrationImpl(args) { + assert.equal(heldLock?.acquired, true); + return loadBrokerRegistration(args); + }, + registerBrokerOwnerImpl(registration, options) { + assert.equal(options.registryLock, heldLock); + return registerBrokerOwner(registration, options); + }, + releaseBrokerRegistryLockImpl(registration, lock) { + assert.equal(lock, heldLock); + const released = releaseBrokerRegistryLock(registration, lock); + heldLock = null; + return released; + } + }); + + assert.equal(second.pid, first.pid); + assert.equal(heldLock, null); +}); + +test("existing broker reuse refuses an owner that dies at the publication boundary", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const baseEnv = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: getProcessIdentity(process.pid) + }; + const envA = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "publication-owner-a" }; + const envB = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "publication-owner-b" }; + const first = await ensureBrokerSession(repo, { env: envA }); + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envA, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "publication-owner-a" }) + }); + }); + + let publicationChecks = 0; + const reused = await ensureBrokerSession(repo, { + env: envB, + registerBrokerOwnerImpl(registration, options) { + return registerBrokerOwner(registration, { + ...options, + hasLiveProcessIdentityImpl(pid, identity) { + publicationChecks += 1; + assert.equal(pid, process.pid); + assert.equal(identity, envB[SESSION_OWNER_IDENTITY_ENV]); + return false; + } + }); + } + }); + + assert.equal(reused, null); + assert.equal(publicationChecks, 1); + assert.equal(loadBrokerSession(repo)?.pid, first.pid); + const registration = loadBrokerRegistration({ + endpoint: first.endpoint, + brokerIdentity: first.pidIdentity, + env: envB + }); + const owners = fs.readdirSync(path.join(registration.registryDir, "owners")) + .map((name) => JSON.parse(fs.readFileSync(path.join(registration.registryDir, "owners", name), "utf8"))); + assert.equal(owners.some((owner) => owner.sessionId === "publication-owner-b"), false); +}); + +test("deterministic broker launch lock never splits across fallback resources", async (t) => { + const repo = makeTempDir(); + const port = brokerLaunchLockPort(repo); + const foreignServer = net.createServer((socket) => socket.end("foreign-service\n")); + await new Promise((resolve, reject) => { + foreignServer.once("error", reject); + foreignServer.listen({ host: "127.0.0.1", port, exclusive: true }, resolve); + }); + t.after(async () => { + if (foreignServer.listening) { + await new Promise((resolve) => foreignServer.close(resolve)); + } + }); + + await assert.rejects( + acquireBrokerLaunchLock(repo, { timeoutMs: 250 }), + (error) => error?.code === "BROKER_LAUNCH_LOCK_UNAVAILABLE" + ); + await new Promise((resolve) => foreignServer.close(resolve)); + + const first = await acquireBrokerLaunchLock(repo, { timeoutMs: 250 }); + assert.equal(first.port, port); + try { + await assert.rejects( + acquireBrokerLaunchLock(repo, { timeoutMs: 75 }), + (error) => error?.code === "BROKER_LAUNCH_LOCK_TIMEOUT" + ); + } finally { + await first.release(); + } + const next = await acquireBrokerLaunchLock(repo, { timeoutMs: 250 }); + assert.equal(next.port, port); + await next.release(); +}); + +test("automatic broker falls back to direct mode when its launch lock cannot bind", async () => { + const repo = makeTempDir(); + const env = withBrokerOwner(process.env, "launch-lock-bind-denied"); + let endpointCreated = false; + + const session = await ensureBrokerSession(repo, { + env, + async acquireBrokerLaunchLockImpl() { + const error = new Error("loopback bind denied"); + error.code = "EACCES"; + throw error; + }, + createBrokerEndpoint() { + endpointCreated = true; + return "unix:/unused.sock"; + } + }); + + assert.equal(session, null); + assert.equal(endpointCreated, false); +}); + +test("automatic and explicit reuse paths refuse a live unregistered endpoint", async (t) => { + if (process.platform === "win32") { + t.skip("Unix broker sockets are required for this contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const socketPath = path.join("/private/tmp", `cxc-unregistered-${process.pid}-${Date.now()}.sock`); + const endpoint = `unix:${socketPath}`; + let brokerConnections = 0; + const server = net.createServer((socket) => { + brokerConnections += 1; + socket.end(); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, resolve); + }); + saveBrokerSession(repo, { endpoint, registry: null }); + t.after(async () => { + await new Promise((resolve) => server.close(resolve)); + clearBrokerSession(repo); + if (fs.existsSync(socketPath)) { + fs.unlinkSync(socketPath); + } + }); + + const directClient = await CodexAppServerClient.connect(repo, { + reuseExistingBroker: true, + env: buildEnv(binDir) + }); + await directClient.close(); + assert.equal(brokerConnections, 0); + + assert.equal( + await ensureBrokerSession(repo, { + env: withBrokerOwner(process.env, "unregistered-refusal") + }), + null + ); + assert.equal(loadBrokerSession(repo)?.endpoint, endpoint); + assert.equal(brokerConnections, 0); +}); + +test("invalid legacy broker state falls back to direct mode without deleting evidence", async () => { + const repo = makeTempDir(); + const stateDir = resolveStateDir(repo); + fs.mkdirSync(stateDir, { recursive: true }); + const statePath = path.join(stateDir, "broker.json"); + fs.writeFileSync(statePath, "{truncated\n", "utf8"); + + const session = await ensureBrokerSession(repo, { + env: withBrokerOwner(process.env, "invalid-legacy-state") + }); + + assert.equal(session, null); + assert.equal(fs.readFileSync(statePath, "utf8"), "{truncated\n"); + fs.unlinkSync(statePath); +}); + +test("automatic broker rolls back when the broker registration cannot be published", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + const socketPath = path.join("/private/tmp", `cxc-publish-failure-${process.pid}-${Date.now()}.sock`); + installFakeCodex(binDir, "review-ok"); + const ownerIdentity = getProcessIdentity(process.pid); + const env = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + CODEX_COMPANION_SESSION_ID: "publish-failure-owner", + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: ownerIdentity + }; + let brokerPid = null; + + await assert.rejects( + ensureBrokerSession(repo, { + env, + createBrokerEndpoint: () => `unix:${socketPath}`, + captureProcessOwnershipImpl(pid, options) { + brokerPid = pid; + return captureProcessOwnership(pid, options); + }, + publishRegisteredBrokerImpl() { + throw new Error("injected broker registration failure"); + } + }), + (error) => error?.code === "BROKER_REGISTRATION_FAILED" + ); + + assert.ok(Number.isFinite(brokerPid)); + await waitFor(() => { + try { + process.kill(brokerPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + assert.equal(loadBrokerSession(repo), null); + assert.equal(fs.existsSync(socketPath), false); +}); + +test("automatic broker rolls back when owner, state, or activation publication fails", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const ownerIdentity = getProcessIdentity(process.pid); + for (const failure of ["owner", "state", "activation"]) { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const socketPath = path.join("/private/tmp", `cxc-${failure}-failure-${process.pid}-${Date.now()}.sock`); + installFakeCodex(binDir, "review-ok"); + const env = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + CODEX_COMPANION_SESSION_ID: `${failure}-failure-owner`, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: ownerIdentity + }; + let brokerPid = null; + + await assert.rejects( + ensureBrokerSession(repo, { + env, + createBrokerEndpoint: () => `unix:${socketPath}`, + captureProcessOwnershipImpl(pid, options) { + brokerPid = pid; + return captureProcessOwnership(pid, options); + }, + publishRegisteredBrokerImpl: failure === "owner" + ? () => ({ registered: false, reason: "injected-owner-failure" }) + : undefined, + saveBrokerSessionImpl: failure === "state" + ? () => { + throw new Error("injected state publication failure"); + } + : undefined, + activateBrokerProcessImpl: failure === "activation" + ? async () => { + throw new Error("injected activation failure"); + } + : undefined + }), + (error) => error?.code === "BROKER_REGISTRATION_FAILED" + ); + + assert.ok(Number.isFinite(brokerPid)); + await waitFor(() => { + try { + process.kill(brokerPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + assert.equal(loadBrokerSession(repo), null); + assert.equal(fs.existsSync(socketPath), false); + } +}); + +test("automatic broker rollback remains locked until exact cleanup converges", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir + }, "locked-rollback"); + let registration = null; + let brokerPid = null; + let releaseObserved = false; + + await assert.rejects( + ensureBrokerSession(repo, { + env, + rollbackExitTimeoutMs: 0, + captureProcessOwnershipImpl(pid, options) { + brokerPid = pid; + return captureProcessOwnership(pid, options); + }, + publishRegisteredBrokerImpl(options) { + registration = publishRegisteredBroker(options); + return registration; + }, + saveBrokerSessionImpl() { + throw new Error("injected state publication failure"); + }, + releaseBrokerOwnerImpl(lockedRegistration, options) { + assert.equal(options.registryLock?.acquired, true); + assert.equal(fs.existsSync(options.registryLock.path), true); + assert.equal(acquireBrokerRegistryLock(lockedRegistration).acquired, false); + assert.equal(hasLiveProcessIdentity(brokerPid, registration.broker.pidIdentity), false); + releaseObserved = true; + return releaseBrokerOwner(lockedRegistration, options); + } + }), + (error) => error?.code === "BROKER_REGISTRATION_FAILED" + ); + + assert.equal(releaseObserved, true); + assert.equal(fs.existsSync(registration.registryLock.path), false); +}); + +test("a failed activation recovery marker is never reusable", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir + }, "failed-activation-marker"); + const unverifiedCleanup = async () => ({ + attempted: true, + delivered: false, + verified: false, + degraded: true, + survivors: [] + }); + + await assert.rejects( + ensureBrokerSession(repo, { + env, + rollbackExitTimeoutMs: 0, + async activateBrokerProcessImpl(child) { + await activateBrokerProcess(child); + throw new Error("injected post-activation publication failure"); + }, + killProcess: unverifiedCleanup + }), + (error) => error?.code === "BROKER_CLEANUP_UNVERIFIED" + ); + + const failedSession = loadBrokerSession(repo); + t.after(async () => { + await sendBrokerShutdown(failedSession?.endpoint).catch(() => {}); + if (Number.isFinite(failedSession?.pid)) { + await terminateProcessTree(failedSession.pid, { + expectedRootIdentity: failedSession.pidIdentity, + ownershipSnapshot: failedSession.ownershipSnapshot + }).catch(() => {}); + } + clearBrokerSession(repo); + }); + assert.equal(failedSession?.activationFailed, true); + assert.equal(loadReusableBrokerSession(repo, env), null); + assert.doesNotThrow(() => process.kill(failedSession.pid, 0)); + + assert.equal(await ensureBrokerSession(repo, { env, killProcess: unverifiedCleanup }), null); + assert.equal(loadBrokerSession(repo)?.pid, failedSession.pid); + assert.equal(loadBrokerSession(repo)?.activationFailed, true); +}); + +test("a broker remains non-reusable until activation is acknowledged", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir + }, "pending-activation"); + let releaseActivation; + const activationGate = new Promise((resolve) => { + releaseActivation = resolve; + }); + let session; + const launch = ensureBrokerSession(repo, { + env, + async activateBrokerProcessImpl(child) { + await activationGate; + await activateBrokerProcess(child); + } + }); + t.after(async () => { + releaseActivation?.(); + session ??= await launch.catch(() => null); + await sendBrokerShutdown(session?.endpoint).catch(() => {}); + if (Number.isFinite(session?.pid)) { + await terminateProcessTree(session.pid, { + expectedRootIdentity: session.pidIdentity, + ownershipSnapshot: session.ownershipSnapshot + }).catch(() => {}); + } + clearBrokerSession(repo); + }); + + await waitFor(() => loadBrokerSession(repo)?.activationPending === true); + assert.equal(loadReusableBrokerSession(repo, env), null); + + releaseActivation(); + session = await launch; + assert.equal(loadBrokerSession(repo)?.activationPending, false); + assert.equal(loadReusableBrokerSession(repo, env)?.pid, session.pid); +}); + +test("a registered broker with a missing socket is not reusable", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir + }, "missing-socket"); + const session = await ensureBrokerSession(repo, { env }); + t.after(async () => { + if (Number.isFinite(session?.pid)) { + await terminateProcessTree(session.pid, { + expectedRootIdentity: session.pidIdentity, + ownershipSnapshot: session.ownershipSnapshot + }).catch(() => {}); + } + clearBrokerSession(repo); + }); + + assert.equal(loadReusableBrokerSession(repo, env)?.pid, session.pid); + fs.unlinkSync(session.endpoint.slice("unix:".length)); + assert.equal(loadReusableBrokerSession(repo, env), null); +}); + +test("a crashed broker reclaims its registered child before starting a replacement", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir + }, "crashed-broker-child-reclaim"); + const session = await ensureBrokerSession(repo, { env }); + const helper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { + cwd: repo, + env, + detached: true, + stdio: "ignore" + }); + helper.unref(); + const helperOwnership = captureProcessOwnership(helper.pid, { cwd: repo, env }); + const childRegistration = publishBrokerChild(session.registry, { ownershipSnapshot: helperOwnership }); + assert.equal(childRegistration.registered, true); + + let replacement = null; + t.after(async () => { + await sendBrokerShutdown(replacement?.endpoint).catch(() => {}); + for (const target of [helper, { pid: replacement?.pid }]) { + if (!Number.isFinite(target?.pid)) { + continue; + } + try { + process.kill(target.pid, "SIGKILL"); + } catch { + // Ignore processes already reclaimed by the recovery path. + } + } + clearBrokerSession(repo); + }); + + process.kill(session.pid, "SIGKILL"); + await waitFor(() => !hasLiveProcessIdentity(session.pid, session.pidIdentity)); + replacement = await ensureBrokerSession(repo, { env }); + + assert.notEqual(replacement?.pid, session.pid); + await waitFor(() => !hasLiveProcessIdentity(helper.pid, helperOwnership.rootIdentity)); + assert.equal(loadBrokerChildren(session.registry).children.length, 0); +}); + +test("automatic broker falls back without leaving a process when session ownership is unavailable", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + const socketPath = path.join("/private/tmp", `cxc-owner-unavailable-${process.pid}-${Date.now()}.sock`); + installFakeCodex(binDir, "review-ok"); + let endpointFactoryCalled = false; + let brokerPid = null; + + const session = await ensureBrokerSession(repo, { + env: buildEnv(binDir), + createBrokerEndpoint: () => { + endpointFactoryCalled = true; + return `unix:${socketPath}`; + }, + captureProcessOwnershipImpl(pid, options) { + brokerPid = pid; + return captureProcessOwnership(pid, options); + } + }); + + assert.equal(session, null); + assert.equal(endpointFactoryCalled, false); + assert.equal(brokerPid, null); + assert.equal(loadBrokerSession(repo), null); + assert.equal(fs.existsSync(socketPath), false); +}); + +test("concurrent automatic broker launches converge on one registered process", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const ownerIdentity = getProcessIdentity(process.pid); + const baseEnv = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: ownerIdentity + }; + const snapshots = []; + const capture = (pid, options) => { + const snapshot = captureProcessOwnership(pid, options); + snapshots.push(snapshot); + return snapshot; + }; + t.after(async () => { + for (const snapshot of snapshots) { + await terminateProcessTree(snapshot.rootPid, { + expectedRootIdentity: snapshot.rootIdentity, + ownershipSnapshot: snapshot + }).catch(() => {}); + } + }); + + const [first, second] = await Promise.all([ + ensureBrokerSession(repo, { + env: { ...baseEnv, CODEX_COMPANION_SESSION_ID: "concurrent-owner-a" }, + captureProcessOwnershipImpl: capture + }), + ensureBrokerSession(repo, { + env: { ...baseEnv, CODEX_COMPANION_SESSION_ID: "concurrent-owner-b" }, + captureProcessOwnershipImpl: capture + }) + ]); + + assert.equal(first.pid, second.pid); + assert.equal(snapshots.length, 1); + assert.equal(loadBrokerSession(repo)?.pid, first.pid); + assert.equal(fs.readdirSync(path.join(first.registry.registryDir, "owners")).filter((name) => name.endsWith(".json")).length, 2); +}); + +test("an unreachable registered broker is never terminated while an owner is live", async (t) => { + if (process.platform === "win32") { + t.skip("Unix broker sockets are required for this contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const ownerIdentity = getProcessIdentity(process.pid); + const baseEnv = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: ownerIdentity + }; + const first = await ensureBrokerSession(repo, { + env: { ...baseEnv, CODEX_COMPANION_SESSION_ID: "unreachable-live-owner-a" } + }); + t.after(async () => { + await terminateProcessTree(first.pid, { + expectedRootIdentity: first.pidIdentity, + ownershipSnapshot: first.ownershipSnapshot + }).catch(() => {}); + clearBrokerSession(repo); + }); + + const socketPath = first.endpoint.slice("unix:".length); + fs.unlinkSync(socketPath); + const second = await ensureBrokerSession(repo, { + env: { ...baseEnv, CODEX_COMPANION_SESSION_ID: "unreachable-live-owner-b" } + }); + + assert.equal(second, null); + assert.doesNotThrow(() => process.kill(first.pid, 0)); + assert.equal(loadBrokerSession(repo)?.pid, first.pid); +}); + +test("a detached test broker self-expires when its test runner cannot clean it", async (t) => { + if (process.platform === "win32") { + t.skip("Unix broker sockets are required for this contract."); + return; + } + + const repo = makeTempDir(); + const socketPath = path.join("/private/tmp", `cxc-test-ttl-${process.pid}-${Date.now()}.sock`); + const endpoint = `unix:${socketPath}`; + const broker = spawn(process.execPath, [BROKER_SCRIPT, "serve", "--endpoint", endpoint, "--cwd", repo], { + cwd: repo, + env: { + ...process.env, + CODEX_COMPANION_TEST_BROKER_TTL_MS: "250" + }, + detached: true, + stdio: "ignore" + }); + broker.unref(); + const ownership = captureProcessOwnership(broker.pid, { cwd: repo }); + t.after(async () => { + await terminateProcessTree(broker.pid, { + expectedRootIdentity: ownership.rootIdentity, + ownershipSnapshot: ownership + }).catch(() => {}); + }); + + assert.equal(await waitForBrokerEndpoint(endpoint, 2000), true); + await waitFor(() => !hasLiveProcessIdentity(broker.pid, ownership.rootIdentity), { timeoutMs: 2000 }); + assert.equal(fs.existsSync(socketPath), false); +}); + +test("pre-activation broker exits when its launcher pipe closes", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process groups are required for this contract."); + return; + } + + const repo = makeTempDir(); + const sessionDir = makeTempDir("cxc-launcher-exit-"); + const socketPath = path.join("/private/tmp", `cxc-launcher-exit-${process.pid}-${Date.now()}.sock`); + const endpoint = `unix:${socketPath}`; + const pidFile = path.join(sessionDir, "broker.pid"); + const child = spawn( + process.execPath, + [BROKER_SCRIPT, "serve", "--endpoint", endpoint, "--cwd", repo, "--pid-file", pidFile, "--require-activation-stdin"], + { + cwd: repo, + env: { ...process.env, CODEX_COMPANION_TEST_BROKER_TTL_MS: "30000" }, + detached: true, + stdio: ["pipe", "ignore", "ignore"], + timeout: 30000, + killSignal: "SIGKILL" + } + ); + const ownershipSnapshot = captureProcessOwnership(child.pid, { cwd: repo }); + t.after(async () => { + child.stdin?.destroy(); + await terminateProcessTree(child.pid, { + expectedRootIdentity: ownershipSnapshot.rootIdentity, + ownershipSnapshot + }).catch(() => {}); + }); + + assert.equal(await waitForBrokerEndpoint(endpoint, 2000), true); + const preActivationResponse = await new Promise((resolve, reject) => { + const socket = net.createConnection({ path: socketPath }); + let buffer = ""; + socket.setEncoding("utf8"); + socket.on("connect", () => { + socket.write(`${JSON.stringify({ id: 1, method: "initialize", params: {} })}\n`); + }); + socket.on("data", (chunk) => { + buffer += chunk; + }); + socket.on("end", () => resolve(JSON.parse(buffer.trim()))); + socket.on("error", reject); + }); + assert.equal(preActivationResponse.error?.code, -32004); + child.stdin.end(); + await waitFor(() => { + try { + process.kill(child.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }, { timeoutMs: 2000 }); + assert.equal(fs.existsSync(socketPath), false); + assert.equal(fs.existsSync(pidFile), false); +}); + +test("an unregistered broker refuses to activate a detached app-server child", async (t) => { + if (process.platform === "win32") { + t.skip("Unix broker sockets are required for this contract."); + return; + } + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + const socketPath = path.join("/private/tmp", `cxc-unregistered-child-${process.pid}-${Date.now()}.sock`); + const endpoint = `unix:${socketPath}`; + installFakeCodex(binDir, "with-helper-child"); + const env = buildEnv(binDir); + const broker = spawn(process.execPath, [BROKER_SCRIPT, "serve", "--endpoint", endpoint, "--cwd", repo], { + cwd: repo, + env, + detached: false, + stdio: ["ignore", "ignore", "ignore"], + timeout: 30000, + killSignal: "SIGKILL" + }); + const brokerOwnership = captureProcessOwnership(broker.pid, { cwd: repo, env }); + t.after(async () => { + await sendBrokerShutdown(endpoint).catch(() => {}); + await terminateProcessTree(broker.pid, { + expectedRootIdentity: brokerOwnership.rootIdentity, + ownershipSnapshot: brokerOwnership + }).catch(() => {}); + await waitFor(() => !hasLiveProcessIdentity(broker.pid, brokerOwnership.rootIdentity)); + }); + assert.equal(await waitForBrokerEndpoint(endpoint, 2000), true); + + const client = await CodexAppServerClient.connect(repo, { brokerEndpoint: endpoint, env }); + await assert.rejects( + client.request("thread/start", { cwd: repo, ephemeral: true }), + (error) => error?.rpcCode === -32005 + ); + await client.close(); + assert.equal(fs.existsSync(fakeStatePath), false); +}); + +test("automatic broker creation stays disabled where registered ownership is unsupported", async () => { + const repo = makeTempDir(); + let endpointFactoryCalled = false; + const session = await ensureBrokerSession(repo, { + platform: "win32", + createBrokerEndpoint() { + endpointFactoryCalled = true; + throw new Error("Windows automatic broker creation must stop before endpoint creation."); + } + }); + + assert.equal(session, null); + assert.equal(endpointFactoryCalled, false); + assert.equal(loadBrokerSession(repo), null); +}); + +test("direct app-server reclaims a post-snapshot helper after its child crashes", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process groups are not available on Windows."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "crash-with-post-snapshot-helper"); + const env = buildEnv(binDir); + const client = await CodexAppServerClient.connect(repo, { env, disableBroker: true }); + let helperPid = null; + t.after(async () => { + await client.close().catch(() => {}); + if (Number.isFinite(helperPid)) { + try { + process.kill(helperPid, "SIGKILL"); + } catch { + // Ignore a helper already reclaimed after the app-server crash. + } + } + }); + + await waitFor(() => { + if (!fs.existsSync(fakeStatePath)) { + return false; + } + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + if (state.helperPids?.length !== 1 || state.appServerPids?.length !== 1) { + return false; + } + helperPid = state.helperPids[0]; + return true; + }); + await client.waitForExit(); + await waitFor(() => { + try { + process.kill(helperPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }, { timeoutMs: 3000 }); +}); + +test("broker shutdown completes without starting a second app-server", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "with-resistant-helper"); + const env = withBrokerOwner(buildEnv(binDir), "broker-shutdown"); + const brokerSocketPath = path.join("/private/tmp", `cxc-p1c-${process.pid}-${Date.now()}.sock`); + const brokerSession = await ensureBrokerSession(repo, { + env, + createBrokerEndpoint: () => `unix:${brokerSocketPath}` + }); + if (!brokerSession) { + t.skip("broker socket unavailable in this sandbox"); + return; + } + assert.ok(brokerSession.pid); + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + }); + + const client = await CodexAppServerClient.connect(repo, { env }); + await client.request("thread/list", { cwd: repo }); + const appServerStartsBeforeShutdown = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts; + assert.equal(appServerStartsBeforeShutdown, 1); + await sendBrokerShutdown(brokerSession.endpoint); + await waitFor(() => { + try { + process.kill(brokerSession.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, appServerStartsBeforeShutdown); +}); async function waitFor(predicate, { timeoutMs = 5000, intervalMs = 50 } = {}) { const start = Date.now(); @@ -28,6 +1773,90 @@ async function waitFor(predicate, { timeoutMs = 5000, intervalMs = 50 } = {}) { throw new Error("Timed out waiting for condition."); } +function installSlowRejectFakeCodex(binDir) { + installFakeCodex(binDir, "slow-reject"); + const scriptPath = path.join(binDir, "codex"); + const marker = "const turnId = nextTurnId(state);"; + const source = fs.readFileSync(scriptPath, "utf8"); + // The marker also appears in the review/start handler; anchor the patch to + // the turn/start case so the injection lands in the right handler. + const caseStart = source.indexOf('case "turn/start"'); + assert.ok(caseStart !== -1); + const markerAt = source.indexOf(marker, caseStart); + assert.ok(markerAt !== -1); + fs.writeFileSync( + scriptPath, + source.slice(0, markerAt) + + source.slice(markerAt).replace( + marker, + `if (BEHAVIOR === "slow-reject") { + state.turnRejectPending = true; + saveState(state); + setTimeout(() => { + const current = loadState(); + send({ id: message.id, error: { code: -32000, message: "slow fake turn rejected" } }); + current.turnRejectPending = false; + current.turnRejectSent = true; + saveState(current); + }, 400); + break; + } + + ${marker}` + ), + "utf8" + ); +} + +function installInitializeErrorFakeCodex(binDir) { + installFakeCodex(binDir, "initialize-rpc-error"); + const scriptPath = path.join(binDir, "codex"); + let source = fs.readFileSync(scriptPath, "utf8"); + const startsMarker = "bootState.appServerStarts = (bootState.appServerStarts || 0) + 1;"; + assert.ok(source.includes(startsMarker)); + source = source.replace( + startsMarker, + `${startsMarker} +bootState.appServerPids = [...(bootState.appServerPids || []), process.pid];` + ); + const initializeMarker = ` case "initialize": + state.capabilities = message.params.capabilities || null;`; + assert.ok(source.includes(initializeMarker)); + source = source.replace( + initializeMarker, + ` case "initialize": + if (BEHAVIOR === "initialize-rpc-error") { + send({ id: message.id, error: { code: -32010, message: "initialize failed" } }); + break; + } + state.capabilities = message.params.capabilities || null;` + ); + fs.writeFileSync(scriptPath, source, "utf8"); +} + +function instrumentSlowFakeTurnState(binDir) { + const scriptPath = path.join(binDir, "codex"); + const source = fs.readFileSync(scriptPath, "utf8"); + const delayedTurn = "emitTurnCompletedLater(thread.id, turnId, items, 400);"; + assert.ok(source.includes(delayedTurn)); + fs.writeFileSync( + scriptPath, + source.replace( + delayedTurn, + `state.turnInFlight = true; + saveState(state); + ${delayedTurn}` + ).replace( + "emitTurnCompleted(threadId, turnId, item);", + `emitTurnCompleted(threadId, turnId, item); + const currentState = loadState(); + currentState.turnInFlight = false; + saveState(currentState);` + ), + "utf8" + ); +} + test("setup reports ready when fake codex is installed and authenticated", () => { const binDir = makeTempDir(); installFakeCodex(binDir); @@ -669,7 +2498,7 @@ test("task --resume-last ignores running tasks from other Claude sessions", () = assert.match(resume.stderr, /No previous Codex task thread was found for this repository\./); }); -test("session start hook exports the Claude session id, transcript path, and plugin data dir", () => { +test("session start hook exports session context and a stable owner when available", () => { const repo = makeTempDir(); const envFile = path.join(makeTempDir(), "claude-env.sh"); fs.writeFileSync(envFile, "", "utf8"); @@ -692,10 +2521,14 @@ test("session start hook exports the Claude session id, transcript path, and plu }); assert.equal(result.status, 0, result.stderr); - assert.equal( - fs.readFileSync(envFile, "utf8"), - `export CODEX_COMPANION_SESSION_ID='sess-current'\nexport CODEX_COMPANION_TRANSCRIPT_PATH='${transcriptPath}'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` - ); + const exported = fs.readFileSync(envFile, "utf8"); + assert.match(exported, /export CODEX_COMPANION_SESSION_ID='sess-current'/); + assert.match(exported, new RegExp(`export CODEX_COMPANION_TRANSCRIPT_PATH='${transcriptPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}'`)); + assert.match(exported, new RegExp(`export CLAUDE_PLUGIN_DATA='${pluginDataDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}'`)); + if (process.platform !== "win32") { + assert.match(exported, /export CODEX_COMPANION_SESSION_OWNER_PID='\d+'/); + assert.match(exported, /export CODEX_COMPANION_SESSION_OWNER_IDENTITY='\d+@[^']+'/); + } }); test("write task output focuses on the Codex result without generic follow-up hints", () => { @@ -900,7 +2733,7 @@ test("task using the shared broker still completes when Codex spawns subagents", run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); - const env = buildEnv(binDir); + const env = withBrokerOwner(buildEnv(binDir), "subagent-task"); const review = run("node", [SCRIPT, "review"], { cwd: repo, env @@ -920,6 +2753,43 @@ test("task using the shared broker still completes when Codex spawns subagents", assert.equal(result.stdout, "Handled the requested task.\nTask prompt accepted.\n"); }); +test("inferred shared turn completion releases broker stream state and its idle child", async (t) => { + if (process.platform === "win32") { + return; + } + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "with-subagent-no-main-turn-completed"); + initGitRepo(repo); + fs.writeFileSync(path.join(repo, "README.md"), "hello\n"); + run("git", ["add", "README.md"], { cwd: repo }); + run("git", ["commit", "-m", "init"], { cwd: repo }); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" + }, "inferred-completion"); + + const result = run("node", [SCRIPT, "task", "challenge the current design"], { + cwd: repo, + env + }); + assert.equal(result.status, 0, result.stderr); + const brokerSession = loadBrokerSession(repo); + assert.equal(brokerSession?.registry?.registered, true); + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + }); + + await waitFor(() => { + const children = loadBrokerChildren(brokerSession.registry); + return children.valid === true && children.children.length === 0 && children.releasedChildren.length === 1; + }, { timeoutMs: 2500, intervalMs: 25 }); +}); + test("task --background enqueues a detached worker and exposes per-job status", async () => { const repo = makeTempDir(); const binDir = makeTempDir(); @@ -1539,99 +3409,647 @@ test("result for a finished write-capable task returns the raw Codex final respo assert.match(result.stdout, /Resume in Codex: codex resume thr_[a-z0-9]+/i); }); +test("task worker persists its own process identity and cancel verifies cleanup", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identity is not available on Windows."); + return; + } + + const workspace = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "interruptible-slow-task"); + initGitRepo(workspace); + fs.writeFileSync(path.join(workspace, "README.md"), "hello\n", "utf8"); + + const job = { + id: "task-worker-self-identity", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Verify worker self-identity", + createdAt: "2026-07-28T08:03:00.000Z" + }; + const request = { + cwd: workspace, + model: null, + effort: null, + prompt: "Wait while cancellation verifies worker ownership.", + write: false, + resumeLast: false, + jobId: job.id + }; + let worker = null; + + t.after(() => { + if (!worker?.pid) { + return; + } + try { + process.kill(-worker.pid, "SIGKILL"); + } catch { + try { + process.kill(worker.pid, "SIGKILL"); + } catch { + // Ignore a worker already reclaimed by cancellation. + } + } + }); + + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl(cwd, jobId) { + worker = spawn(process.execPath, [SCRIPT, "task-worker", "--cwd", cwd, "--job-id", jobId], { + cwd, + env: buildEnv(binDir), + detached: true, + stdio: "ignore" + }); + worker.unref(); + return worker; + } + }); + + const runningJob = await waitFor(() => { + const stored = readStoredJob(workspace, job.id); + return stored?.status === "running" && Number.isFinite(stored.pid) ? stored : null; + }); + + if (runningJob.ownershipCaptureFailed === true) { + t.skip("Process table unavailable for worker self-identity verification."); + return; + } + + let liveIdentity; + try { + liveIdentity = getProcessIdentity(runningJob.pid); + } catch (error) { + if (error?.code === "PROCESS_TABLE_UNAVAILABLE") { + t.skip(`process table unavailable: ${error.message}`); + return; + } + throw error; + } + + assert.equal(runningJob.pid, worker.pid); + assert.equal(runningJob.processIdentity, liveIdentity); + assert.equal(Object.hasOwn(runningJob, "ownershipSnapshot"), false); + assert.equal(Object.hasOwn(runningJob, "ownershipCaptureFailed"), false); + + await handleCancel([job.id, "--cwd", workspace, "--json"], { + interruptAppServerTurnImpl: async () => ({ attempted: false, interrupted: false }) + }); + + await waitFor(() => { + try { + process.kill(worker.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + + const cancelledJob = readStoredJob(workspace, job.id); + assert.equal(cancelledJob.status, "cancelled"); + assert.equal(cancelledJob.pid, null); + assert.equal(cancelledJob.processIdentity, liveIdentity); +}); + test("cancel stops an active background job and marks it cancelled", async (t) => { const workspace = makeTempDir(); const stateDir = resolveStateDir(workspace); const jobsDir = path.join(stateDir, "jobs"); fs.mkdirSync(jobsDir, { recursive: true }); - const sleeper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + const sleeper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { cwd: workspace, detached: true, stdio: "ignore" }); - sleeper.unref(); + sleeper.unref(); + const ownershipSnapshot = captureProcessOwnership(sleeper.pid, { cwd: workspace }); + + t.after(() => { + try { + process.kill(-sleeper.pid, "SIGTERM"); + } catch { + try { + process.kill(sleeper.pid, "SIGTERM"); + } catch { + // Ignore missing process. + } + } + }); + + const logFile = path.join(jobsDir, "task-live.log"); + const jobFile = path.join(jobsDir, "task-live.json"); + fs.writeFileSync(logFile, "[2026-03-18T15:30:00.000Z] Starting Codex Task.\n", "utf8"); + fs.writeFileSync( + jobFile, + JSON.stringify( + { + id: "task-live", + status: "running", + title: "Codex Task", + pid: sleeper.pid, + processIdentity: ownershipSnapshot.rootIdentity, + ownershipSnapshot, + logFile + }, + null, + 2 + ), + "utf8" + ); + fs.writeFileSync( + path.join(stateDir, "state.json"), + `${JSON.stringify( + { + version: 1, + config: { stopReviewGate: false }, + jobs: [ + { + id: "task-live", + status: "running", + title: "Codex Task", + jobClass: "task", + summary: "Investigate flaky test", + pid: sleeper.pid, + processIdentity: ownershipSnapshot.rootIdentity, + ownershipSnapshot, + logFile, + createdAt: "2026-03-18T15:30:00.000Z", + startedAt: "2026-03-18T15:30:01.000Z", + updatedAt: "2026-03-18T15:30:02.000Z" + } + ] + }, + null, + 2 + )}\n`, + "utf8" + ); + + const cancelResult = run("node", [SCRIPT, "cancel", "task-live", "--json"], { + cwd: workspace + }); + + assert.equal(cancelResult.status, 0, cancelResult.stderr); + assert.equal(JSON.parse(cancelResult.stdout).status, "cancelled"); + + await waitFor(() => { + try { + process.kill(sleeper.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + + const state = JSON.parse(fs.readFileSync(path.join(stateDir, "state.json"), "utf8")); + const cancelled = state.jobs.find((job) => job.id === "task-live"); + assert.equal(cancelled.status, "cancelled"); + assert.equal(cancelled.pid, null); + + const stored = JSON.parse(fs.readFileSync(jobFile, "utf8")); + assert.equal(stored.status, "cancelled"); + assert.match(fs.readFileSync(logFile, "utf8"), /Cancelled by user/); +}); + +test("cancelling a queued pid-less job prevents its worker from performing work", async () => { + const workspace = makeTempDir(); + const job = { + id: "task-queued-cancel", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + status: "queued", + phase: "queued", + pid: null, + createdAt: "2026-07-28T08:02:00.000Z", + updatedAt: "2026-07-28T08:02:00.000Z" + }; + writeJobFile(workspace, job.id, job); + upsertJob(workspace, job); + + let terminateCalled = false; + await handleCancel([job.id, "--cwd", workspace, "--json"], { + interruptAppServerTurnImpl: async () => ({ attempted: false, interrupted: false }), + terminateProcessTreeImpl: async () => { + terminateCalled = true; + return { + attempted: false, + delivered: false, + verified: true, + degraded: false + }; + } + }); + + assert.equal(terminateCalled, false); + assert.equal(hasCancelFlag(workspace, job.id), true); + + const workMarker = path.join(workspace, "work-performed"); + await assert.rejects( + runTrackedJob(job, async () => { + fs.writeFileSync(workMarker, "ran\n", "utf8"); + return { + exitStatus: 0, + payload: { ok: true }, + rendered: "Work ran.", + summary: "Work ran." + }; + }), + (error) => error?.code === "JOB_CANCELLED" + ); + + assert.equal(fs.existsSync(workMarker), false); +}); + +test("session cleanup preserves a queued cancel between worker read and pid publication", async () => { + const workspace = makeTempDir(); + const sessionId = "sess-queued-worker-race"; + const job = { + id: "task-session-end-race", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Exercise the SessionEnd startup race", + sessionId, + createdAt: "2026-07-28T08:03:00.000Z" + }; + const request = { + cwd: workspace, + prompt: "Do not run after SessionEnd.", + jobId: job.id + }; + const workMarker = path.join(workspace, "work-performed"); + let firstCleanupPromise = null; + let cancelFlagSurvivedCleanup = false; + + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl() {} + }); + const queuedState = loadState(workspace); + const queuedJob = queuedState.jobs.find((candidate) => candidate.id === job.id); + const newerJobs = Array.from({ length: 50 }, (_, index) => ({ + id: `newer-job-${index}`, + status: "completed", + phase: "done", + pid: null, + createdAt: new Date(Date.UTC(2026, 6, 29, 9, index, 0)).toISOString(), + updatedAt: new Date(Date.UTC(2026, 6, 29, 9, index, 0)).toISOString() + })); + fs.writeFileSync( + resolveStateFile(workspace), + `${JSON.stringify({ ...queuedState, jobs: [...newerJobs, { ...queuedJob, updatedAt: "2026-07-28T08:03:00.000Z" }] }, null, 2)}\n`, + "utf8" + ); + + await assert.rejects( + handleTaskWorker(["--cwd", workspace, "--job-id", job.id], { + getProcessIdentityImpl(pid) { + // handleTaskWorker has already read the queued record, but runTrackedJob + // has not yet published this worker's pid. + firstCleanupPromise = cleanupSessionJobs(workspace, sessionId); + cancelFlagSurvivedCleanup = hasCancelFlag(workspace, job.id); + return `${pid}@Mon Jul 28 08:03:01 2026`; + }, + runTrackedJobImpl(candidate, _runner, options) { + return runTrackedJob( + candidate, + async () => { + fs.writeFileSync(workMarker, "ran\n", "utf8"); + return { + exitStatus: 0, + payload: { ok: true }, + rendered: "Work ran.", + summary: "Work ran." + }; + }, + options + ); + } + }), + (error) => error?.code === "JOB_CANCELLED" + ); + + const firstCleanup = await firstCleanupPromise; + assert.equal(firstCleanup.verified, false); + assert.equal(cancelFlagSurvivedCleanup, true); + assert.equal(fs.existsSync(workMarker), false); + assert.equal(readStoredJob(workspace, job.id).status, "cancelled"); + + const finalCleanup = await cleanupSessionJobs(workspace, sessionId); + assert.equal(finalCleanup.verified, true); + assert.equal(readStoredJob(workspace, job.id), null); + assert.equal(hasCancelFlag(workspace, job.id), false); +}); + +test("worker converges a flagged running record to cancelled", async () => { + const workspace = makeTempDir(); + const job = { + id: "task-flag-convergence", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + status: "queued", + phase: "queued", + pid: null, + createdAt: "2026-07-28T08:04:00.000Z" + }; + writeJobFile(workspace, job.id, job); + upsertJob(workspace, job); + writeCancelFlag(workspace, job.id); + + let workPerformed = false; + await assert.rejects( + runTrackedJob(job, async () => { + workPerformed = true; + return { + exitStatus: 0, + payload: { ok: true }, + rendered: "Work ran.", + summary: "Work ran." + }; + }), + (error) => error?.name === "JobCancelledError" && error?.code === "JOB_CANCELLED" + ); + + assert.equal(workPerformed, false); + const storedJob = readStoredJob(workspace, job.id); + assert.equal(storedJob.status, "cancelled"); + assert.equal(storedJob.phase, "cancelled"); + assert.equal(storedJob.pid, null); + assert.match(storedJob.startedAt, /^\d{4}-\d{2}-\d{2}T/); + + const indexedJob = listJobs(workspace).find((candidate) => candidate.id === job.id); + assert.equal(indexedJob.status, "cancelled"); + assert.equal(indexedJob.phase, "cancelled"); + assert.equal(indexedJob.pid, null); + + saveState(workspace, { + ...loadState(workspace), + jobs: [] + }); + assert.equal(readStoredJob(workspace, job.id), null); + assert.equal(hasCancelFlag(workspace, job.id), false); +}); + +test("cancel reclaims a helper spawned after worker identity capture without an ownership snapshot", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process groups are not available on Windows."); + return; + } + + const workspace = makeTempDir(); + const helperPidFile = path.join(workspace, "late-helper.pid"); + const leader = spawn( + process.execPath, + [ + "-e", + ` + const fs = require("node:fs"); + const { spawn } = require("node:child_process"); + setTimeout(() => { + const helper = spawn(process.execPath, ["-e", ${JSON.stringify(SELF_EXPIRING_KEEPALIVE)}], { + stdio: "ignore" + }); + helper.unref(); + fs.writeFileSync(${JSON.stringify(helperPidFile)}, String(helper.pid)); + }, 750); + ${SELF_EXPIRING_KEEPALIVE}; + ` + ], + { + cwd: workspace, + detached: true, + stdio: "ignore" + } + ); + leader.unref(); + let helperPid = null; + + t.after(() => { + try { + process.kill(-leader.pid, "SIGKILL"); + } catch { + try { + process.kill(leader.pid, "SIGKILL"); + } catch { + // Ignore a leader already reclaimed by cancellation. + } + } + if (Number.isFinite(helperPid)) { + try { + process.kill(helperPid, "SIGKILL"); + } catch { + // Ignore a helper already reclaimed with the worker group. + } + } + }); + + let leaderIdentity; + try { + leaderIdentity = getProcessIdentity(leader.pid); + } catch (error) { + if (error?.code === "PROCESS_TABLE_UNAVAILABLE") { + t.skip(`process table unavailable: ${error.message}`); + return; + } + throw error; + } + assert.ok(leaderIdentity); + assert.equal(fs.existsSync(helperPidFile), false); + + const logFile = path.join(workspace, "late-helper.log"); + fs.writeFileSync(logFile, "", "utf8"); + const job = { + id: "task-late-helper", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + status: "running", + phase: "running", + pid: leader.pid, + processIdentity: leaderIdentity, + logFile, + createdAt: "2026-07-28T08:05:00.000Z", + updatedAt: "2026-07-28T08:05:00.000Z" + }; + writeJobFile(workspace, job.id, job); + upsertJob(workspace, job); + + helperPid = await waitFor(() => { + if (!fs.existsSync(helperPidFile)) { + return null; + } + return Number(fs.readFileSync(helperPidFile, "utf8")); + }); + assert.ok(Number.isFinite(helperPid)); + + await handleCancel([job.id, "--cwd", workspace, "--json"], { + interruptAppServerTurnImpl: async () => ({ attempted: false, interrupted: false }) + }); + + function isRunning(pid) { + const result = run("/bin/ps", ["-o", "stat=", "-p", String(pid)]); + return result.status === 0 && !result.stdout.trim().startsWith("Z"); + } + + await waitFor(() => !isRunning(leader.pid)); + await waitFor(() => !isRunning(helperPid)); + assert.equal(readStoredJob(workspace, job.id).status, "cancelled"); + assert.equal(Object.hasOwn(readStoredJob(workspace, job.id), "ownershipSnapshot"), false); +}); + +test("cancel and session cleanup converge after an identity-tracked worker exits abnormally", async () => { + const workspace = makeTempDir(); + const sessionId = "sess-abnormal-worker-exit"; + const job = { + id: "task-abnormal-worker-exit", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Clean up an exited worker", + sessionId, + status: "running", + phase: "running", + pid: 43210, + processIdentity: "43210@Mon Jul 28 08:05:00 2026", + createdAt: "2026-07-28T08:05:00.000Z" + }; + let processTableReads = 0; + let terminateCalledDuringSessionCleanup = false; + + writeJobFile(workspace, job.id, job); + upsertJob(workspace, job); + + await handleCancel([job.id, "--cwd", workspace, "--json"], { + interruptAppServerTurnImpl: async () => ({ attempted: false, interrupted: false }), + terminateProcessTreeImpl(pid, options) { + return terminateProcessTree(pid, { + ...options, + platform: "darwin", + runCommandImpl(command, args) { + processTableReads += 1; + return { + command, + args, + status: 0, + signal: null, + stdout: "", + stderr: "", + error: null + }; + }, + killImpl() { + throw new Error("an absent worker must not be signaled"); + } + }); + } + }); - t.after(() => { - try { - process.kill(-sleeper.pid, "SIGTERM"); - } catch { - try { - process.kill(sleeper.pid, "SIGTERM"); - } catch { - // Ignore missing process. - } + const cancelledJob = readStoredJob(workspace, job.id); + assert.equal(processTableReads, 1); + assert.equal(cancelledJob.status, "cancelled"); + assert.equal(cancelledJob.pid, null); + assert.equal(cancelledJob.processIdentity, job.processIdentity); + assert.equal(Object.hasOwn(cancelledJob, "ownershipSnapshot"), false); + + const sessionCleanup = await cleanupSessionJobs(workspace, sessionId, { + terminateProcessTreeImpl() { + terminateCalledDuringSessionCleanup = true; + throw new Error("terminal jobs must not be terminated again"); } }); + assert.equal(sessionCleanup.verified, true); + assert.equal(terminateCalledDuringSessionCleanup, false); + assert.equal(readStoredJob(workspace, job.id), null); +}); +test("unverified cleanup preserves cancel, session, and broker ownership records", async () => { + const workspace = makeTempDir(); + const stateDir = resolveStateDir(workspace); + const jobsDir = path.join(stateDir, "jobs"); + fs.mkdirSync(jobsDir, { recursive: true }); const logFile = path.join(jobsDir, "task-live.log"); const jobFile = path.join(jobsDir, "task-live.json"); - fs.writeFileSync(logFile, "[2026-03-18T15:30:00.000Z] Starting Codex Task.\n", "utf8"); - fs.writeFileSync( - jobFile, - JSON.stringify( - { - id: "task-live", - status: "running", - title: "Codex Task", - logFile - }, - null, - 2 - ), - "utf8" - ); + const job = { + id: "task-live", + status: "running", + phase: "running", + title: "Codex Task", + sessionId: "sess-current", + pid: 123, + processIdentity: "123@old", + logFile + }; + fs.writeFileSync(logFile, "starting\n", "utf8"); + fs.writeFileSync(jobFile, `${JSON.stringify(job, null, 2)}\n`, "utf8"); fs.writeFileSync( path.join(stateDir, "state.json"), - `${JSON.stringify( - { - version: 1, - config: { stopReviewGate: false }, - jobs: [ - { - id: "task-live", - status: "running", - title: "Codex Task", - jobClass: "task", - summary: "Investigate flaky test", - pid: sleeper.pid, - logFile, - createdAt: "2026-03-18T15:30:00.000Z", - startedAt: "2026-03-18T15:30:01.000Z", - updatedAt: "2026-03-18T15:30:02.000Z" - } - ] - }, - null, - 2 - )}\n`, + `${JSON.stringify({ version: 1, config: { stopReviewGate: false }, jobs: [job] }, null, 2)}\n`, "utf8" ); - const cancelResult = run("node", [SCRIPT, "cancel", "task-live", "--json"], { - cwd: workspace - }); - - assert.equal(cancelResult.status, 0, cancelResult.stderr); - assert.equal(JSON.parse(cancelResult.stdout).status, "cancelled"); - - await waitFor(() => { - try { - process.kill(sleeper.pid, 0); - return false; - } catch (error) { - return error?.code === "ESRCH"; + const cleanupOutcome = { + attempted: true, + delivered: true, + verified: false, + degraded: true, + survivors: [123], + survivorIdentities: ["123@old"] + }; + await assert.rejects( + handleCancel(["task-live", "--cwd", workspace, "--json"], { + interruptAppServerTurnImpl: async () => ({ attempted: false, interrupted: false }), + terminateProcessTreeImpl: async () => cleanupOutcome + }), + /ownership records were preserved for retry/ + ); + let state = JSON.parse(fs.readFileSync(path.join(stateDir, "state.json"), "utf8")); + assert.equal(state.jobs[0].status, "running"); + assert.equal(state.jobs[0].pid, 123); + assert.deepEqual(state.jobs[0].cleanupOutcome.survivorIdentities, ["123@old"]); + + let sessionCleanupOptions; + const sessionCleanup = await cleanupSessionJobs(workspace, "sess-current", { + terminateProcessTreeImpl: async (_pid, options) => { + sessionCleanupOptions = options; + return cleanupOutcome; } }); - - const state = JSON.parse(fs.readFileSync(path.join(stateDir, "state.json"), "utf8")); - const cancelled = state.jobs.find((job) => job.id === "task-live"); - assert.equal(cancelled.status, "cancelled"); - assert.equal(cancelled.pid, null); - - const stored = JSON.parse(fs.readFileSync(jobFile, "utf8")); - assert.equal(stored.status, "cancelled"); - assert.match(fs.readFileSync(logFile, "utf8"), /Cancelled by user/); + assert.equal(sessionCleanup.verified, false); + assert.equal(sessionCleanupOptions.priorCleanupDegraded, true); + state = JSON.parse(fs.readFileSync(path.join(stateDir, "state.json"), "utf8")); + assert.equal(state.jobs[0].pid, 123); + assert.equal(fs.existsSync(jobFile), true); + + const sessionDir = path.join(workspace, "broker-session"); + fs.mkdirSync(sessionDir, { recursive: true }); + const pidFile = path.join(sessionDir, "broker.pid"); + const brokerLog = path.join(sessionDir, "broker.log"); + const endpointPath = path.join(sessionDir, "broker.sock"); + fs.writeFileSync(pidFile, "456\n", "utf8"); + fs.writeFileSync(brokerLog, "broker\n", "utf8"); + fs.writeFileSync(endpointPath, "socket-marker\n", "utf8"); + const brokerCleanup = await teardownBrokerSession({ + endpoint: `unix:${endpointPath}`, + pidFile, + logFile: brokerLog, + sessionDir, + pid: 456, + killProcess: async () => cleanupOutcome + }); + assert.equal(brokerCleanup.verified, false); + assert.equal(fs.existsSync(pidFile), true); + assert.equal(fs.existsSync(brokerLog), true); + assert.equal(fs.existsSync(endpointPath), true); }); test("cancel without a job id ignores active jobs from other Claude sessions", () => { @@ -1747,7 +4165,7 @@ test("cancel sends turn interrupt to the shared app-server before killing a brok run("git", ["add", "README.md"], { cwd: repo }); run("git", ["commit", "-m", "init"], { cwd: repo }); - const env = buildEnv(binDir); + const env = withBrokerOwner(buildEnv(binDir), "cancel-interrupt"); const launched = run("node", [SCRIPT, "task", "--background", "--json", "investigate the flaky worker timeout"], { cwd: repo, env @@ -1824,13 +4242,27 @@ test("session end fully cleans up jobs for the ending session", async (t) => { fs.writeFileSync(completedJobFile, JSON.stringify({ id: "review-completed" }, null, 2), "utf8"); fs.writeFileSync(otherJobFile, JSON.stringify({ id: "review-other" }, null, 2), "utf8"); - const sleeper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + const sleeper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { cwd: repo, detached: true, stdio: "ignore" }); sleeper.unref(); - fs.writeFileSync(runningJobFile, JSON.stringify({ id: "review-running" }, null, 2), "utf8"); + const ownershipSnapshot = captureProcessOwnership(sleeper.pid, { cwd: repo }); + fs.writeFileSync( + runningJobFile, + JSON.stringify( + { + id: "review-running", + pid: sleeper.pid, + processIdentity: ownershipSnapshot.rootIdentity, + ownershipSnapshot + }, + null, + 2 + ), + "utf8" + ); t.after(() => { try { @@ -1866,6 +4298,8 @@ test("session end fully cleans up jobs for the ending session", async (t) => { title: "Codex Review", sessionId: "sess-current", pid: sleeper.pid, + processIdentity: ownershipSnapshot.rootIdentity, + ownershipSnapshot, logFile: runningLog, createdAt: "2026-03-18T15:32:00.000Z", updatedAt: "2026-03-18T15:33:00.000Z" @@ -2128,7 +4562,7 @@ test("commands lazily start and reuse one shared app-server after first use", as run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); - const env = buildEnv(binDir); + const env = withBrokerOwner(buildEnv(binDir), "lazy-reuse"); const review = run("node", [SCRIPT, "review"], { cwd: repo, @@ -2161,6 +4595,316 @@ test("commands lazily start and reuse one shared app-server after first use", as assert.equal(cleanup.status, 0, cleanup.stderr); }); +test("shared broker clears a disconnected stream after its request rejects", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installSlowRejectFakeCodex(binDir); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" + }, "disconnected-stream"); + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + }); + + const clientA = await CodexAppServerClient.connect(repo, { env }); + const started = await clientA.request("thread/start", { cwd: repo, ephemeral: true }); + const pendingTurn = clientA + .request("turn/start", { + threadId: started.thread.id, + input: [{ type: "text", text: "reject this turn after the client disconnects" }] + }) + .catch(() => null); + + await waitFor(() => { + if (!fs.existsSync(fakeStatePath)) { + return false; + } + return JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).turnRejectPending === true; + }); + await clientA.close(); + await pendingTurn; + + await waitFor(() => { + if (!fs.existsSync(fakeStatePath)) { + return false; + } + return JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).turnRejectSent === true; + }); + + const clientB = await CodexAppServerClient.connect(repo, { env }); + t.after(() => clientB.close().catch(() => {})); + const response = await waitFor(async () => { + try { + return await clientB.request("thread/list", { cwd: repo }); + } catch (error) { + if (error.rpcCode === -32001) { + return false; + } + throw error; + } + }, { timeoutMs: 1500, intervalMs: 20 }); + assert.ok(Array.isArray(response.data)); +}); + +test("shared broker cleans up an app-server whose initialization returns an RPC error", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installInitializeErrorFakeCodex(binDir); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" + }, "initialize-error"); + const brokerSession = await ensureBrokerSession(repo, { env }); + t.after(() => { + if (brokerSession) { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + } + if (fs.existsSync(fakeStatePath)) { + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + for (const pid of state.appServerPids || []) { + try { + process.kill(pid, "SIGKILL"); + } catch { + // Ignore children already terminated by the cleanup path. + } + } + } + }); + + async function requestUntilInitializationFails() { + const request = brokerSession + ? (() => { + const client = CodexAppServerClient.connect(repo, { env }); + return client.then((connectedClient) => connectedClient.request("thread/list", { cwd: repo })); + })() + : CodexAppServerClient.connect(repo, { env, disableBroker: true }); + await assert.rejects( + request, + (error) => + (error.rpcCode === -32010 && /initialize failed/.test(error.message)) || + (error.rpcCode === -32002 && /refusing to spawn a replacement child/i.test(error.message)) + ); + } + + function loadFakeState() { + return JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + } + + function isLive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } + } + + await requestUntilInitializationFails(); + await waitFor(() => { + const state = loadFakeState(); + return state.appServerStarts === 1 && state.appServerPids?.length === 1 && !isLive(state.appServerPids[0]); + }); + + await requestUntilInitializationFails(); + await waitFor(() => { + const state = loadFakeState(); + return state.appServerStarts <= 2 && state.appServerPids?.length <= 2 && state.appServerPids.every((pid) => !isLive(pid)); + }); +}); + +test("shared broker releases its idle app-server child and restarts it on demand", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installFakeCodex(binDir, "with-helper-child"); + initGitRepo(repo); + fs.writeFileSync(path.join(repo, "README.md"), "hello\n"); + run("git", ["add", "README.md"], { cwd: repo }); + run("git", ["commit", "-m", "init"], { cwd: repo }); + fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); + + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" + }, "idle-child"); + + const review = run("node", [SCRIPT, "review"], { + cwd: repo, + env + }); + assert.equal(review.status, 0, review.stderr); + + const firstSession = loadBrokerSession(repo); + assert.ok(firstSession?.pid); + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ + hook_event_name: "SessionEnd", + cwd: repo + }) + }); + if (fs.existsSync(fakeStatePath)) { + const fakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + for (const helperPid of fakeState.helperPids || []) { + try { + process.kill(helperPid, "SIGTERM"); + } catch { + // Ignore helpers already terminated with their app-server group. + } + } + } + }); + + const firstFakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + const firstHelperPid = firstFakeState.helperPids?.[0]; + assert.ok(firstHelperPid); + await waitFor(() => { + try { + process.kill(firstHelperPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + + const adversarial = run("node", [SCRIPT, "adversarial-review"], { + cwd: repo, + env + }); + assert.equal(adversarial.status, 0, adversarial.stderr); + + const secondSession = loadBrokerSession(repo); + assert.equal(secondSession?.pid, firstSession.pid); + const fakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + assert.equal(fakeState.appServerStarts, 2); + assert.equal(fakeState.helperPids.length, 2); +}); + +test("identity capture failure prevents app-server activation and reports unverified cleanup", async () => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installFakeCodex(binDir, "with-helper-child"); + const env = buildEnv(binDir); + await assert.rejects( + CodexAppServerClient.connect(repo, { + disableBroker: true, + env, + captureProcessOwnershipImpl() { + throw new Error("identity lookup injected failure"); + } + }), + (error) => error.cleanupOutcome?.verified === false && error.cleanupOutcome?.degraded === true + ); + + assert.equal(fs.existsSync(fakeStatePath), false); +}); + +test("a broker-owned app-server cannot activate before durable child publication", async () => { + if (process.platform === "win32") { + return; + } + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "with-helper-child"); + const env = buildEnv(binDir); + let wrapperPid = null; + + await assert.rejects( + CodexAppServerClient.connect(repo, { + disableBroker: true, + gatedBrokerChild: true, + env, + beforeAppServerActivation(ownershipSnapshot) { + wrapperPid = ownershipSnapshot.rootPid; + assert.equal(fs.existsSync(fakeStatePath), false); + throw new Error("injected durable child publication failure"); + } + }), + /durable child publication failure/ + ); + + assert.ok(Number.isFinite(wrapperPid)); + assert.equal(fs.existsSync(fakeStatePath), false); + await waitFor(() => getProcessIdentity(wrapperPid) === null); +}); + +test("shared broker keeps active work alive after its client disconnects", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installFakeCodex(binDir, "slow-task-with-helper-child"); + instrumentSlowFakeTurnState(binDir); + initGitRepo(repo); + fs.writeFileSync(path.join(repo, "README.md"), "hello\n"); + run("git", ["add", "README.md"], { cwd: repo }); + run("git", ["commit", "-m", "init"], { cwd: repo }); + + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" + }, "active-work"); + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + }); + + const client = await CodexAppServerClient.connect(repo, { env }); + const started = await client.request("thread/start", { cwd: repo, ephemeral: true }); + await client.request("turn/start", { + threadId: started.thread.id, + input: [{ type: "text", text: "finish after the client disconnects" }] + }); + await client.close(); + + const initialState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + const helperPid = initialState.helperPids?.[0]; + assert.ok(helperPid); + + await waitFor(() => { + if (JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).turnInFlight !== true) { + return false; + } + try { + process.kill(helperPid, 0); + return true; + } catch { + return false; + } + }); + + await waitFor(() => { + try { + process.kill(helperPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); +}); + test("setup reuses an existing shared app-server without starting another one", () => { const repo = makeTempDir(); const binDir = makeTempDir(); @@ -2173,7 +4917,7 @@ test("setup reuses an existing shared app-server without starting another one", run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); - const env = buildEnv(binDir); + const env = withBrokerOwner(buildEnv(binDir), "setup-reuse"); const review = run("node", [SCRIPT, "review"], { cwd: repo, @@ -2216,9 +4960,10 @@ test("status reports shared session runtime when a lazy broker is active", () => run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); + const env = withBrokerOwner(buildEnv(binDir), "status-runtime"); const review = run("node", [SCRIPT, "review"], { cwd: repo, - env: buildEnv(binDir) + env }); assert.equal(review.status, 0, review.stderr); @@ -2228,14 +4973,21 @@ test("status reports shared session runtime when a lazy broker is active", () => const result = run("node", [SCRIPT, "status"], { cwd: repo, - env: buildEnv(binDir) + env }); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /Session runtime: shared session/); + + const unownedResult = run("node", [SCRIPT, "status"], { + cwd: repo, + env: buildEnv(binDir) + }); + assert.equal(unownedResult.status, 0, unownedResult.stderr); + assert.match(unownedResult.stdout, /Session runtime: direct startup/); }); -test("setup and status honor --cwd when reading shared session runtime", () => { +test("setup and status ignore an unregistered saved runtime", () => { const targetWorkspace = makeTempDir(); const invocationWorkspace = makeTempDir(); @@ -2247,13 +4999,13 @@ test("setup and status honor --cwd when reading shared session runtime", () => { cwd: invocationWorkspace }); assert.equal(status.status, 0, status.stderr); - assert.match(status.stdout, /Session runtime: shared session/); + assert.match(status.stdout, /Session runtime: direct startup/); const setup = run("node", [SCRIPT, "setup", "--cwd", targetWorkspace, "--json"], { cwd: invocationWorkspace }); assert.equal(setup.status, 0, setup.stderr); const payload = JSON.parse(setup.stdout); - assert.equal(payload.sessionRuntime.mode, "shared"); - assert.equal(payload.sessionRuntime.endpoint, "unix:/tmp/fake-broker.sock"); + assert.equal(payload.sessionRuntime.mode, "direct"); + assert.equal(payload.sessionRuntime.endpoint, null); });