From be5edeac1f0694c87615b6d6f80d82c737e26d3d Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Sun, 26 Jul 2026 23:12:17 -0700 Subject: [PATCH 01/29] [CC-5219] release idle Codex app-server children Attributed-To: Codex interactive --- plugins/codex/scripts/app-server-broker.mjs | 133 ++++++++++++++++++-- tests/runtime.test.mjs | 50 ++++++++ 2 files changed, 170 insertions(+), 13 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 1954274fe..333e5c071 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -9,8 +9,18 @@ import { parseArgs } from "./lib/args.mjs"; import { BROKER_BUSY_RPC_CODE, CodexAppServerClient } from "./lib/app-server.mjs"; import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs"; +const DEFAULT_CHILD_IDLE_MS = 5 * 60 * 1000; const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]); +function resolveChildIdleMs(env = process.env) { + const raw = env.CODEX_COMPANION_BROKER_CHILD_IDLE_MS; + if (raw == null || raw === "") { + return DEFAULT_CHILD_IDLE_MS; + } + const value = Number(raw); + return Number.isFinite(value) && value >= 0 ? value : DEFAULT_CHILD_IDLE_MS; +} + function buildStreamThreadIds(method, params, result) { const threadIds = new Set(); if (params?.threadId) { @@ -65,12 +75,69 @@ async function main() { const pidFile = options["pid-file"] ? path.resolve(options["pid-file"]) : null; writePidFile(pidFile); - const appClient = await CodexAppServerClient.connect(cwd, { disableBroker: true }); + const childIdleMs = resolveChildIdleMs(); + let appClient = null; + let appClientStartPromise = null; + let appClientClosePromise = null; + let childIdleTimer = null; + let inFlightRequests = 0; + let shutdownPromise = null; let activeRequestSocket = null; let activeStreamSocket = null; let activeStreamThreadIds = null; const sockets = new Set(); + function cancelChildIdleClose() { + if (childIdleTimer) { + clearTimeout(childIdleTimer); + childIdleTimer = null; + } + } + + function hasActiveWork() { + return ( + sockets.size > 0 || + inFlightRequests > 0 || + activeRequestSocket !== null || + activeStreamSocket !== null + ); + } + + async function closeAppClient() { + cancelChildIdleClose(); + if (appClientStartPromise) { + await appClientStartPromise.catch(() => {}); + } + if (appClientClosePromise) { + await appClientClosePromise; + return; + } + const client = appClient; + appClient = null; + if (!client) { + return; + } + appClientClosePromise = client.close().catch(() => {}).finally(() => { + appClientClosePromise = null; + }); + await appClientClosePromise; + } + + function scheduleChildIdleClose() { + cancelChildIdleClose(); + if (!appClient || hasActiveWork()) { + return; + } + childIdleTimer = setTimeout(() => { + childIdleTimer = null; + if (hasActiveWork()) { + return; + } + void closeAppClient(); + }, childIdleMs); + childIdleTimer.unref?.(); + } + function clearSocketOwnership(socket) { if (activeRequestSocket === socket) { activeRequestSocket = null; @@ -99,23 +166,51 @@ async function main() { } } - async function shutdown(server) { - for (const socket of sockets) { - socket.end(); + async function getAppClient() { + cancelChildIdleClose(); + if (appClient) { + return appClient; } - await appClient.close().catch(() => {}); - await new Promise((resolve) => server.close(resolve)); - if (listenTarget.kind === "unix" && fs.existsSync(listenTarget.path)) { - fs.unlinkSync(listenTarget.path); + if (appClientClosePromise) { + await appClientClosePromise; } - if (pidFile && fs.existsSync(pidFile)) { - fs.unlinkSync(pidFile); + if (!appClientStartPromise) { + appClientStartPromise = CodexAppServerClient.connect(cwd, { disableBroker: true }) + .then((client) => { + appClient = client; + client.setNotificationHandler(routeNotification); + return client; + }) + .finally(() => { + appClientStartPromise = null; + }); } + return appClientStartPromise; } - appClient.setNotificationHandler(routeNotification); + async function shutdown(server) { + if (shutdownPromise) { + return shutdownPromise; + } + shutdownPromise = (async () => { + cancelChildIdleClose(); + for (const socket of sockets) { + socket.end(); + } + await closeAppClient(); + await new Promise((resolve) => server.close(resolve)); + if (listenTarget.kind === "unix" && fs.existsSync(listenTarget.path)) { + fs.unlinkSync(listenTarget.path); + } + if (pidFile && fs.existsSync(pidFile)) { + fs.unlinkSync(pidFile); + } + })(); + return shutdownPromise; + } const server = net.createServer((socket) => { + cancelChildIdleClose(); sockets.add(socket); socket.setEncoding("utf8"); let buffer = ""; @@ -182,23 +277,30 @@ async function main() { } if (allowInterruptDuringActiveStream) { + inFlightRequests += 1; try { - const result = await appClient.request(message.method, message.params ?? {}); + const client = await getAppClient(); + const result = await client.request(message.method, message.params ?? {}); send(socket, { id: message.id, result }); } catch (error) { send(socket, { id: message.id, error: buildJsonRpcError(error.rpcCode ?? -32000, error.message) }); + } finally { + inFlightRequests -= 1; + scheduleChildIdleClose(); } continue; } const isStreaming = STREAMING_METHODS.has(message.method); activeRequestSocket = socket; + inFlightRequests += 1; try { - const result = await appClient.request(message.method, message.params ?? {}); + const client = await getAppClient(); + const result = await client.request(message.method, message.params ?? {}); send(socket, { id: message.id, result }); if (isStreaming) { activeStreamSocket = socket; @@ -218,6 +320,9 @@ async function main() { if (activeStreamSocket === socket && !isStreaming) { activeStreamSocket = null; } + } finally { + inFlightRequests -= 1; + scheduleChildIdleClose(); } } }); @@ -225,11 +330,13 @@ async function main() { socket.on("close", () => { sockets.delete(socket); clearSocketOwnership(socket); + scheduleChildIdleClose(); }); socket.on("error", () => { sockets.delete(socket); clearSocketOwnership(socket); + scheduleChildIdleClose(); }); }); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 8f276835b..cfe5211b3 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -2161,6 +2161,56 @@ test("commands lazily start and reuse one shared app-server after first use", as assert.equal(cleanup.status, 0, cleanup.stderr); }); +test("shared broker releases its idle app-server child and restarts it on demand", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installFakeCodex(binDir); + initGitRepo(repo); + fs.writeFileSync(path.join(repo, "README.md"), "hello\n"); + run("git", ["add", "README.md"], { cwd: repo }); + run("git", ["commit", "-m", "init"], { cwd: repo }); + fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); + + const env = { + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" + }; + + const review = run("node", [SCRIPT, "review"], { + cwd: repo, + env + }); + assert.equal(review.status, 0, review.stderr); + + const firstSession = loadBrokerSession(repo); + assert.ok(firstSession?.pid); + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ + hook_event_name: "SessionEnd", + cwd: repo + }) + }); + }); + + await new Promise((resolve) => setTimeout(resolve, 300)); + + const adversarial = run("node", [SCRIPT, "adversarial-review"], { + cwd: repo, + env + }); + assert.equal(adversarial.status, 0, adversarial.stderr); + + const secondSession = loadBrokerSession(repo); + assert.equal(secondSession?.pid, firstSession.pid); + const fakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + assert.equal(fakeState.appServerStarts, 2); +}); + test("setup reuses an existing shared app-server without starting another one", () => { const repo = makeTempDir(); const binDir = makeTempDir(); From eb6b3a6edf17d32ff0987a5e2a0e946a72f66203 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Sun, 26 Jul 2026 23:17:42 -0700 Subject: [PATCH 02/29] [CC-5219] terminate idle app-server helper groups Attributed-To: Codex interactive --- plugins/codex/scripts/lib/app-server.mjs | 20 ++++++++++---------- tests/fake-codex-fixture.mjs | 8 ++++++++ tests/runtime.test.mjs | 24 +++++++++++++++++++++++- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index 72b30a764..ef7201f6c 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -190,6 +190,7 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { this.proc = spawn("codex", ["app-server"], { cwd: this.cwd, env: this.options.env ?? process.env, + detached: process.platform !== "win32", stdio: ["pipe", "pipe", "pipe"], shell: process.platform === "win32" ? (process.env.SHELL || true) : false, windowsHide: true @@ -243,23 +244,22 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { if (this.proc && !this.proc.killed) { this.proc.stdin.end(); - setTimeout(() => { - if (this.proc && !this.proc.killed && this.proc.exitCode === null) { - // On Windows with shell: true, the direct child is cmd.exe. - // Use terminateProcessTree to kill the entire tree including - // the grandchild node process. - if (process.platform === "win32") { + if (process.platform === "win32") { + setTimeout(() => { + if (this.proc && !this.proc.killed && this.proc.exitCode === null) { try { terminateProcessTree(this.proc.pid); } catch { // Best-effort cleanup inside an unref'd timer — swallow errors // to avoid crashing the host process during shutdown. } - } else { - this.proc.kill("SIGTERM"); } - } - }, 50).unref?.(); + }, 50).unref?.(); + } else { + // The app-server is its own process-group leader on Unix. Terminate + // the group so MCP helpers cannot outlive the app-server parent. + terminateProcessTree(this.proc.pid); + } } await this.exitPromise; diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index f83c96a0d..fea20414f 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -16,6 +16,7 @@ const readline = require("node:readline"); const STATE_PATH = ${JSON.stringify(statePath)}; const BEHAVIOR = ${JSON.stringify(behavior)}; const interruptibleTurns = new Map(); + const { spawn } = require("node:child_process"); function loadState() { if (!fs.existsSync(STATE_PATH)) { @@ -272,6 +273,13 @@ if (args[0] !== "app-server") { } const bootState = loadState(); bootState.appServerStarts = (bootState.appServerStarts || 0) + 1; +if (BEHAVIOR === "with-helper-child") { + const helper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + stdio: "ignore" + }); + helper.unref(); + bootState.helperPids = [...(bootState.helperPids || []), helper.pid]; +} saveState(bootState); const rl = readline.createInterface({ input: process.stdin }); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index cfe5211b3..1706a856b 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -2166,7 +2166,7 @@ test("shared broker releases its idle app-server child and restarts it on demand const binDir = makeTempDir(); const fakeStatePath = path.join(binDir, "fake-codex-state.json"); - installFakeCodex(binDir); + installFakeCodex(binDir, "with-helper-child"); initGitRepo(repo); fs.writeFileSync(path.join(repo, "README.md"), "hello\n"); run("git", ["add", "README.md"], { cwd: repo }); @@ -2195,9 +2195,30 @@ test("shared broker releases its idle app-server child and restarts it on demand cwd: repo }) }); + if (fs.existsSync(fakeStatePath)) { + const fakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + for (const helperPid of fakeState.helperPids || []) { + try { + process.kill(helperPid, "SIGTERM"); + } catch { + // Ignore helpers already terminated with their app-server group. + } + } + } }); + const firstFakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + const firstHelperPid = firstFakeState.helperPids?.[0]; + assert.ok(firstHelperPid); await new Promise((resolve) => setTimeout(resolve, 300)); + await waitFor(() => { + try { + process.kill(firstHelperPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); const adversarial = run("node", [SCRIPT, "adversarial-review"], { cwd: repo, @@ -2209,6 +2230,7 @@ test("shared broker releases its idle app-server child and restarts it on demand assert.equal(secondSession?.pid, firstSession.pid); const fakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); assert.equal(fakeState.appServerStarts, 2); + assert.equal(fakeState.helperPids.length, 2); }); test("setup reuses an existing shared app-server without starting another one", () => { From 289f39d0c9faa670bebfb7aae4fec3536ed974e6 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Sun, 26 Jul 2026 23:24:05 -0700 Subject: [PATCH 03/29] [CC-5219] clean detached app-server descendants Attributed-To: Codex interactive --- plugins/codex/scripts/lib/process.mjs | 69 ++++++++++++++++++++------- tests/fake-codex-fixture.mjs | 1 + tests/process.test.mjs | 63 ++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 17 deletions(-) diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index af28d1cf5..5215e7845 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -54,6 +54,53 @@ function looksLikeMissingProcessMessage(text) { return /not found|no running instance|cannot find|does not exist|no such process/i.test(text); } +function listDescendantPids(pid, runCommandImpl) { + const result = runCommandImpl("ps", ["-axo", "pid=,ppid="]); + if (result.error || result.status !== 0) { + return []; + } + + const childrenByParent = new Map(); + for (const line of result.stdout.split("\n")) { + const match = line.trim().match(/^(\d+)\s+(\d+)$/); + if (!match) { + continue; + } + const childPid = Number(match[1]); + const parentPid = Number(match[2]); + const children = childrenByParent.get(parentPid) ?? []; + children.push(childPid); + childrenByParent.set(parentPid, children); + } + + const descendants = []; + const visit = (parentPid) => { + for (const childPid of childrenByParent.get(parentPid) ?? []) { + visit(childPid); + descendants.push(childPid); + } + }; + visit(pid); + return descendants; +} + +function terminateUnixProcessOrGroup(pid, killImpl) { + try { + killImpl(-pid, "SIGTERM"); + return true; + } catch (groupError) { + try { + killImpl(pid, "SIGTERM"); + return true; + } catch (processError) { + if (processError?.code === "ESRCH") { + return false; + } + throw processError?.code ? processError : groupError; + } + } +} + export function terminateProcessTree(pid, options = {}) { if (!Number.isFinite(pid)) { return { attempted: false, delivered: false, method: null }; @@ -97,24 +144,12 @@ export function terminateProcessTree(pid, options = {}) { throw new Error(formatCommandFailure(result)); } - try { - killImpl(-pid, "SIGTERM"); - return { attempted: true, delivered: true, method: "process-group" }; - } catch (error) { - if (error?.code !== "ESRCH") { - try { - killImpl(pid, "SIGTERM"); - return { attempted: true, delivered: true, method: "process" }; - } catch (innerError) { - if (innerError?.code === "ESRCH") { - return { attempted: true, delivered: false, method: "process" }; - } - throw innerError; - } - } - - return { attempted: true, delivered: false, method: "process-group" }; + const targets = [...listDescendantPids(pid, runCommandImpl), pid]; + let delivered = false; + for (const targetPid of targets) { + delivered = terminateUnixProcessOrGroup(targetPid, killImpl) || delivered; } + return { attempted: true, delivered, method: "process-tree", targets }; } export function formatCommandFailure(result) { diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index fea20414f..d56f73465 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -275,6 +275,7 @@ const bootState = loadState(); bootState.appServerStarts = (bootState.appServerStarts || 0) + 1; if (BEHAVIOR === "with-helper-child") { const helper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + detached: process.platform !== "win32", stdio: "ignore" }); helper.unref(); diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 80e0715b0..438258222 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -53,3 +53,66 @@ test("terminateProcessTree treats missing Windows processes as already stopped", assert.equal(outcome.result.status, 128); assert.match(outcome.result.stdout, /not found/i); }); + +test("terminateProcessTree terminates Unix descendant groups deepest-first", () => { + const signals = []; + const outcome = terminateProcessTree(1234, { + platform: "darwin", + runCommandImpl(command, args) { + assert.equal(command, "ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid="]); + return { + command, + args, + status: 0, + signal: null, + stdout: "1234 1\n1235 1234\n1236 1235\n1237 1234\n", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, [ + [-1236, "SIGTERM"], + [-1235, "SIGTERM"], + [-1237, "SIGTERM"], + [-1234, "SIGTERM"] + ]); + assert.equal(outcome.delivered, true); + assert.equal(outcome.method, "process-tree"); + assert.deepEqual(outcome.targets, [1236, 1235, 1237, 1234]); +}); + +test("terminateProcessTree falls back to a Unix PID when it is not a group leader", () => { + const signals = []; + const outcome = terminateProcessTree(1234, { + platform: "darwin", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "1234 1\n", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + if (pid < 0) { + const error = new Error("no such process group"); + error.code = "ESRCH"; + throw error; + } + } + }); + + assert.deepEqual(signals, [[-1234, "SIGTERM"], [1234, "SIGTERM"]]); + assert.equal(outcome.delivered, true); + assert.equal(outcome.method, "process-tree"); +}); From 41fe9c75be4068e0475cc4a0e7aa407e692d4769 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Mon, 27 Jul 2026 00:38:44 -0700 Subject: [PATCH 04/29] [CC-5219] make helper cleanup identity safe Attributed-To: Codex interactive --- plugins/codex/scripts/app-server-broker.mjs | 33 ++- plugins/codex/scripts/lib/app-server.mjs | 34 ++- plugins/codex/scripts/lib/process.mjs | 264 +++++++++++++++++--- tests/fake-codex-fixture.mjs | 4 +- tests/process.test.mjs | 159 +++++++++++- tests/runtime.test.mjs | 49 ++++ 6 files changed, 483 insertions(+), 60 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 333e5c071..59f978882 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -85,6 +85,7 @@ async function main() { let activeRequestSocket = null; let activeStreamSocket = null; let activeStreamThreadIds = null; + let activeStreamRunning = false; const sockets = new Set(); function cancelChildIdleClose() { @@ -99,7 +100,7 @@ async function main() { sockets.size > 0 || inFlightRequests > 0 || activeRequestSocket !== null || - activeStreamSocket !== null + activeStreamRunning ); } @@ -144,24 +145,25 @@ async function main() { } if (activeStreamSocket === socket) { activeStreamSocket = null; - activeStreamThreadIds = null; } } function routeNotification(message) { const target = activeRequestSocket ?? activeStreamSocket; - if (!target) { - return; + if (target) { + send(target, message); } - send(target, message); - if (message.method === "turn/completed" && activeStreamSocket === target) { + if (message.method === "turn/completed" && activeStreamRunning) { const threadId = message.params?.threadId ?? null; if (!threadId || !activeStreamThreadIds || activeStreamThreadIds.has(threadId)) { + const streamSocket = activeStreamSocket; + activeStreamRunning = false; activeStreamSocket = null; activeStreamThreadIds = null; - if (activeRequestSocket === target) { + if (activeRequestSocket === streamSocket) { activeRequestSocket = null; } + scheduleChildIdleClose(); } } } @@ -263,10 +265,11 @@ async function main() { } const allowInterruptDuringActiveStream = - isInterruptRequest(message) && activeStreamSocket && activeStreamSocket !== socket && !activeRequestSocket; + isInterruptRequest(message) && activeStreamRunning && activeStreamSocket !== socket && !activeRequestSocket; if ( - ((activeRequestSocket && activeRequestSocket !== socket) || (activeStreamSocket && activeStreamSocket !== socket)) && + ((activeRequestSocket && activeRequestSocket !== socket) || + (activeStreamRunning && activeStreamSocket !== socket)) && !allowInterruptDuringActiveStream ) { send(socket, { @@ -296,14 +299,18 @@ async function main() { const isStreaming = STREAMING_METHODS.has(message.method); activeRequestSocket = socket; + if (isStreaming) { + activeStreamRunning = true; + activeStreamSocket = socket; + activeStreamThreadIds = buildStreamThreadIds(message.method, message.params ?? {}, null); + } inFlightRequests += 1; try { const client = await getAppClient(); const result = await client.request(message.method, message.params ?? {}); send(socket, { id: message.id, result }); - if (isStreaming) { - activeStreamSocket = socket; + if (isStreaming && activeStreamRunning && activeStreamSocket === socket) { activeStreamThreadIds = buildStreamThreadIds(message.method, message.params ?? {}, result); } if (activeRequestSocket === socket) { @@ -317,8 +324,10 @@ async function main() { if (activeRequestSocket === socket) { activeRequestSocket = null; } - if (activeStreamSocket === socket && !isStreaming) { + if (isStreaming && activeStreamSocket === socket) { + activeStreamRunning = false; activeStreamSocket = null; + activeStreamThreadIds = null; } } finally { inFlightRequests -= 1; diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index ef7201f6c..a9f51b747 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -14,7 +14,7 @@ import { spawn } from "node:child_process"; import readline from "node:readline"; import { parseBrokerEndpoint } from "./broker-endpoint.mjs"; import { ensureBrokerSession, loadBrokerSession } from "./broker-lifecycle.mjs"; -import { terminateProcessTree } from "./process.mjs"; +import { getProcessIdentity, terminateProcessTree } from "./process.mjs"; const PLUGIN_MANIFEST_URL = new URL("../../.claude-plugin/plugin.json", import.meta.url); const PLUGIN_MANIFEST = JSON.parse(fs.readFileSync(PLUGIN_MANIFEST_URL, "utf8")); @@ -195,7 +195,6 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { shell: process.platform === "win32" ? (process.env.SHELL || true) : false, windowsHide: true }); - this.proc.stdout.setEncoding("utf8"); this.proc.stderr.setEncoding("utf8"); @@ -227,6 +226,30 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { clientInfo: this.options.clientInfo ?? DEFAULT_CLIENT_INFO, capabilities: this.options.capabilities ?? DEFAULT_CAPABILITIES }); + try { + this.procIdentity = + process.platform === "win32" + ? null + : getProcessIdentity(this.proc.pid, { + cwd: this.cwd, + env: this.options.env ?? process.env + }); + } catch (error) { + try { + process.kill(-this.proc.pid, "SIGKILL"); + } catch { + // The child may have exited before identity capture completed. + } + throw error; + } + if (process.platform !== "win32" && !this.procIdentity) { + try { + process.kill(-this.proc.pid, "SIGKILL"); + } catch { + // The child may have exited before identity capture completed. + } + throw new Error("Unable to capture codex app-server process identity."); + } this.notify("initialized", {}); } @@ -258,7 +281,12 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { } else { // The app-server is its own process-group leader on Unix. Terminate // the group so MCP helpers cannot outlive the app-server parent. - terminateProcessTree(this.proc.pid); + const outcome = terminateProcessTree(this.proc.pid, { + expectedRootIdentity: this.procIdentity + }); + if (!outcome.verified) { + throw new Error(`Unable to verify codex app-server cleanup; surviving PIDs: ${outcome.survivors.join(", ")}.`); + } } } diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 5215e7845..9c41078e5 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -54,51 +54,199 @@ function looksLikeMissingProcessMessage(text) { return /not found|no running instance|cannot find|does not exist|no such process/i.test(text); } -function listDescendantPids(pid, runCommandImpl) { - const result = runCommandImpl("ps", ["-axo", "pid=,ppid="]); +const UNIX_PROCESS_TABLE_ARGS = ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]; +const UNIX_PS_COMMAND = "/bin/ps"; + +function readUnixProcessTable(runCommandImpl, options = {}) { + const result = runCommandImpl(UNIX_PS_COMMAND, UNIX_PROCESS_TABLE_ARGS, { + cwd: options.cwd, + env: options.env + }); if (result.error || result.status !== 0) { - return []; + const detail = result.error?.message ?? result.stderr.trim() ?? result.stdout.trim() ?? `exit ${result.status}`; + throw new Error(`Unable to enumerate Unix processes: ${detail || `exit ${result.status}`}`); } - const childrenByParent = new Map(); + const processes = new Map(); for (const line of result.stdout.split("\n")) { - const match = line.trim().match(/^(\d+)\s+(\d+)$/); + const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/); if (!match) { continue; } - const childPid = Number(match[1]); + const pid = Number(match[1]); const parentPid = Number(match[2]); - const children = childrenByParent.get(parentPid) ?? []; - children.push(childPid); - childrenByParent.set(parentPid, children); + const processGroupId = Number(match[3]); + const state = match[4]; + const startedAt = match[5].trim(); + processes.set(pid, { + pid, + parentPid, + processGroupId, + state, + startedAt, + identity: `${pid}@${startedAt}` + }); } + return processes; +} + +function isRunningProcess(record) { + return record && !record.state.startsWith("Z"); +} - const descendants = []; - const visit = (parentPid) => { - for (const childPid of childrenByParent.get(parentPid) ?? []) { - visit(childPid); - descendants.push(childPid); +function collectProcessTree(rootPid, processes, rootDepth = 0) { + const childrenByParent = new Map(); + for (const record of processes.values()) { + const children = childrenByParent.get(record.parentPid) ?? []; + children.push(record); + childrenByParent.set(record.parentPid, children); + } + + const records = []; + const visited = new Set(); + const visit = (parentPid, depth) => { + if (visited.has(parentPid)) { + return; + } + visited.add(parentPid); + for (const child of childrenByParent.get(parentPid) ?? []) { + visit(child.pid, depth + 1); + records.push({ ...child, depth: depth + 1 }); } }; - visit(pid); - return descendants; + visit(rootPid, rootDepth); + const root = processes.get(rootPid); + if (root) { + records.push({ ...root, depth: rootDepth }); + } + return records; +} + +function sleepSync(milliseconds) { + if (milliseconds <= 0) { + return; + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds); +} + +export function getProcessIdentity(pid, options = {}) { + if (!Number.isFinite(pid) || (options.platform ?? process.platform) === "win32") { + return null; + } + return readUnixProcessTable(options.runCommandImpl ?? runCommand, options).get(pid)?.identity ?? null; } -function terminateUnixProcessOrGroup(pid, killImpl) { +function mergeTrackedDescendants(tracked, processes, rootPid, rootIdentity) { + const trackedByPid = new Map([...tracked.values()].map((record) => [record.pid, record.identity])); + const roots = []; + const currentRoot = processes.get(rootPid); + if (currentRoot?.identity === rootIdentity) { + roots.push({ pid: rootPid, depth: 0 }); + } + for (const trackedRecord of tracked.values()) { + const current = processes.get(trackedRecord.pid); + if (current?.identity === trackedRecord.identity) { + roots.push({ pid: current.pid, depth: trackedRecord.depth }); + } + } + for (const root of roots) { + const records = collectProcessTree(root.pid, processes, root.depth).sort((left, right) => left.depth - right.depth); + for (const record of records) { + const priorIdentity = trackedByPid.get(record.pid); + if (priorIdentity && priorIdentity !== record.identity) { + continue; + } + if (record.pid !== root.pid) { + const parentIdentity = trackedByPid.get(record.parentPid); + if (!parentIdentity || processes.get(record.parentPid)?.identity !== parentIdentity) { + continue; + } + } + if (!tracked.has(record.identity)) { + tracked.set(record.identity, record); + trackedByPid.set(record.pid, record.identity); + } + } + } +} + +function listLiveTracked(tracked, processes) { + return [...tracked.values()].filter((record) => { + const current = processes.get(record.pid); + return current?.identity === record.identity && isRunningProcess(current); + }); +} + +function buildSignalUnits(records) { + const groupLeaders = new Map(); + for (const record of records) { + if (record.pid === record.processGroupId) { + groupLeaders.set(record.processGroupId, record); + } + } + + const units = []; + for (const record of records) { + const groupLeader = groupLeaders.get(record.processGroupId); + if (groupLeader && groupLeader.identity !== record.identity) { + continue; + } + units.push({ + record, + group: record.pid === record.processGroupId + }); + } + units.sort((left, right) => right.record.depth - left.record.depth); + return units; +} + +function signalVerifiedUnit(unit, signal, { runCommandImpl, killImpl, cwd, env }) { + const processes = readUnixProcessTable(runCommandImpl, { cwd, env }); + const current = processes.get(unit.record.pid); + if (current?.identity !== unit.record.identity || !isRunningProcess(current)) { + return false; + } + if (unit.group && current.processGroupId !== current.pid) { + return false; + } + + const target = unit.group ? -current.pid : current.pid; try { - killImpl(-pid, "SIGTERM"); + killImpl(target, signal); return true; - } catch (groupError) { - try { - killImpl(pid, "SIGTERM"); - return true; - } catch (processError) { - if (processError?.code === "ESRCH") { - return false; - } - throw processError?.code ? processError : groupError; + } catch (error) { + if (error?.code === "ESRCH") { + return false; } + throw error; + } +} + +function signalTracked(tracked, signal, options) { + const processes = readUnixProcessTable(options.runCommandImpl, options); + mergeTrackedDescendants(tracked, processes, options.rootPid, options.rootIdentity); + const units = buildSignalUnits(listLiveTracked(tracked, processes)); + let delivered = false; + for (const unit of units) { + delivered = signalVerifiedUnit(unit, signal, options) || delivered; } + return delivered; +} + +function pollTracked(tracked, options, attempts) { + let live = []; + for (let attempt = 0; attempt < attempts; attempt += 1) { + const processes = readUnixProcessTable(options.runCommandImpl, options); + mergeTrackedDescendants(tracked, processes, options.rootPid, options.rootIdentity); + live = listLiveTracked(tracked, processes); + if (live.length === 0) { + break; + } + if (attempt + 1 < attempts) { + options.sleepImpl(options.pollIntervalMs); + } + } + return live; } export function terminateProcessTree(pid, options = {}) { @@ -144,12 +292,64 @@ export function terminateProcessTree(pid, options = {}) { throw new Error(formatCommandFailure(result)); } - const targets = [...listDescendantPids(pid, runCommandImpl), pid]; - let delivered = false; - for (const targetPid of targets) { - delivered = terminateUnixProcessOrGroup(targetPid, killImpl) || delivered; + const initialProcesses = readUnixProcessTable(runCommandImpl, options); + const root = initialProcesses.get(pid); + const expectedRootIdentity = options.expectedRootIdentity ?? root?.identity ?? null; + if (!root) { + return { + attempted: true, + delivered: false, + verified: true, + escalated: false, + method: "process-tree", + targets: [] + }; + } + if (root.identity !== expectedRootIdentity) { + return { + attempted: true, + delivered: false, + verified: true, + escalated: false, + identityMismatch: true, + method: "process-tree", + targets: [] + }; + } + + const tracked = new Map(); + for (const record of collectProcessTree(pid, initialProcesses)) { + tracked.set(record.identity, record); } - return { attempted: true, delivered, method: "process-tree", targets }; + const unixOptions = { + rootPid: pid, + rootIdentity: expectedRootIdentity, + runCommandImpl, + killImpl, + cwd: options.cwd, + env: options.env, + sleepImpl: options.sleepImpl ?? sleepSync, + pollIntervalMs: options.pollIntervalMs ?? 25 + }; + + let delivered = signalTracked(tracked, "SIGTERM", unixOptions); + let live = pollTracked(tracked, unixOptions, options.termPollAttempts ?? 11); + let escalated = false; + if (live.length > 0) { + escalated = true; + delivered = signalTracked(tracked, "SIGKILL", unixOptions) || delivered; + live = pollTracked(tracked, unixOptions, options.killPollAttempts ?? 11); + } + + return { + attempted: true, + delivered, + verified: live.length === 0, + escalated, + method: "process-tree", + targets: [...tracked.values()].sort((left, right) => right.depth - left.depth).map((record) => record.pid), + survivors: live.map((record) => record.pid) + }; } export function formatCommandFailure(result) { diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index d56f73465..9bb1da5d2 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -273,7 +273,7 @@ if (args[0] !== "app-server") { } const bootState = loadState(); bootState.appServerStarts = (bootState.appServerStarts || 0) + 1; -if (BEHAVIOR === "with-helper-child") { +if (BEHAVIOR === "with-helper-child" || BEHAVIOR === "slow-task-with-helper-child") { const helper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: process.platform !== "win32", stdio: "ignore" @@ -609,7 +609,7 @@ rl.on("line", (line) => { send({ method: "turn/completed", params: { threadId: thread.id, turn: buildTurn(turnId, "completed") } }); }, 5000); interruptibleTurns.set(turnId, { threadId: thread.id, timer }); - } else if (BEHAVIOR === "slow-task") { + } else if (BEHAVIOR === "slow-task" || BEHAVIOR === "slow-task-with-helper-child") { emitTurnCompletedLater(thread.id, turnId, items, 400); } else { emitTurnCompleted(thread.id, turnId, items); diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 438258222..c1976b522 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -56,23 +56,29 @@ test("terminateProcessTree treats missing Windows processes as already stopped", test("terminateProcessTree terminates Unix descendant groups deepest-first", () => { const signals = []; + const alive = new Set([1234, 1235, 1236, 1237]); + const parents = new Map([[1234, 1], [1235, 1234], [1236, 1235], [1237, 1234]]); const outcome = terminateProcessTree(1234, { platform: "darwin", runCommandImpl(command, args) { - assert.equal(command, "ps"); - assert.deepEqual(args, ["-axo", "pid=,ppid="]); + assert.equal(command, "/bin/ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]); + const stdout = [...alive] + .map((pid) => `${pid} ${parents.get(pid)} ${pid} S Mon Jul 27 00:00:0${pid - 1234} 2026`) + .join("\n"); return { command, args, status: 0, signal: null, - stdout: "1234 1\n1235 1234\n1236 1235\n1237 1234\n", + stdout: stdout ? `${stdout}\n` : "", stderr: "", error: null }; }, killImpl(pid, signal) { signals.push([pid, signal]); + alive.delete(Math.abs(pid)); } }); @@ -87,8 +93,9 @@ test("terminateProcessTree terminates Unix descendant groups deepest-first", () assert.deepEqual(outcome.targets, [1236, 1235, 1237, 1234]); }); -test("terminateProcessTree falls back to a Unix PID when it is not a group leader", () => { +test("terminateProcessTree signals a Unix PID directly when it is not a group leader", () => { const signals = []; + let alive = true; const outcome = terminateProcessTree(1234, { platform: "darwin", runCommandImpl(command, args) { @@ -97,22 +104,152 @@ test("terminateProcessTree falls back to a Unix PID when it is not a group leade args, status: 0, signal: null, - stdout: "1234 1\n", + stdout: alive ? "1234 1 999 S Mon Jul 27 00:00:00 2026\n" : "", stderr: "", error: null }; }, killImpl(pid, signal) { signals.push([pid, signal]); - if (pid < 0) { - const error = new Error("no such process group"); - error.code = "ESRCH"; - throw error; - } + alive = false; } }); - assert.deepEqual(signals, [[-1234, "SIGTERM"], [1234, "SIGTERM"]]); + assert.deepEqual(signals, [[1234, "SIGTERM"]]); assert.equal(outcome.delivered, true); + assert.equal(outcome.verified, true); assert.equal(outcome.method, "process-tree"); }); + +test("terminateProcessTree fails closed when Unix process enumeration fails", () => { + assert.throws( + () => + terminateProcessTree(1234, { + platform: "darwin", + runCommandImpl(command, args) { + return { + command, + args, + status: 1, + signal: null, + stdout: "", + stderr: "ps denied", + error: null + }; + } + }), + /Unable to enumerate Unix processes.*ps denied/i + ); +}); + +test("terminateProcessTree refuses a reused root PID", () => { + const signals = []; + const outcome = terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Sun Jul 26 00:00:00 2026", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "1234 1 1234 S Mon Jul 27 00:00:00 2026\n", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.verified, true); + assert.equal(outcome.identityMismatch, true); +}); + +test("terminateProcessTree revalidates descendant identities before signaling", () => { + const signals = []; + let rootAlive = true; + let snapshots = 0; + const outcome = terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", + termPollAttempts: 1, + killPollAttempts: 1, + sleepImpl() {}, + runCommandImpl(command, args) { + snapshots += 1; + const root = rootAlive ? "1234 1 1234 S Mon Jul 27 00:00:00 2026\n" : ""; + const childStart = snapshots === 1 ? "Mon Jul 27 00:00:01 2026" : "Mon Jul 27 00:01:01 2026"; + return { + command, + args, + status: 0, + signal: null, + stdout: `${root}1235 1234 1235 S ${childStart}\n`, + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + if (pid === -1234) { + rootAlive = false; + } + } + }); + + assert.deepEqual(signals, [[-1234, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.equal(outcome.escalated, false); +}); + +test("terminateProcessTree escalates resistant descendants and verifies exit", () => { + const signals = []; + const alive = new Set([1234, 1235]); + const pgids = new Map([[1234, 1234], [1235, 1235]]); + const outcome = terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", + termPollAttempts: 1, + killPollAttempts: 1, + sleepImpl() {}, + runCommandImpl(command, args) { + const stdout = [ + alive.has(1234) ? "1234 1 1234 S Mon Jul 27 00:00:00 2026" : null, + alive.has(1235) ? "1235 1234 1235 S Mon Jul 27 00:00:01 2026" : null + ].filter(Boolean).join("\n"); + return { + command, + args, + status: 0, + signal: null, + stdout: stdout ? `${stdout}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + if (signal !== "SIGKILL") { + return; + } + const targetPgid = pid < 0 ? -pid : pgids.get(pid); + for (const candidate of [...alive]) { + if (candidate === pid || pgids.get(candidate) === targetPgid) { + alive.delete(candidate); + } + } + } + }); + + assert.deepEqual(signals, [ + [-1235, "SIGTERM"], + [-1234, "SIGTERM"], + [-1235, "SIGKILL"], + [-1234, "SIGKILL"] + ]); + assert.equal(outcome.verified, true); + assert.equal(outcome.escalated, true); +}); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 1706a856b..814ea0502 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -7,6 +7,7 @@ import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; import { initGitRepo, makeTempDir, run } from "./helpers.mjs"; +import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; import { loadBrokerSession, saveBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { resolveStateDir } from "../plugins/codex/scripts/lib/state.mjs"; @@ -2233,6 +2234,54 @@ test("shared broker releases its idle app-server child and restarts it on demand assert.equal(fakeState.helperPids.length, 2); }); +test("shared broker keeps active work alive after its client disconnects", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installFakeCodex(binDir, "slow-task-with-helper-child"); + initGitRepo(repo); + fs.writeFileSync(path.join(repo, "README.md"), "hello\n"); + run("git", ["add", "README.md"], { cwd: repo }); + run("git", ["commit", "-m", "init"], { cwd: repo }); + + const env = { + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" + }; + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + }); + + const client = await CodexAppServerClient.connect(repo, { env }); + const started = await client.request("thread/start", { cwd: repo, ephemeral: true }); + await client.request("turn/start", { + threadId: started.thread.id, + input: [{ type: "text", text: "finish after the client disconnects" }] + }); + await client.close(); + + const initialState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + const helperPid = initialState.helperPids?.[0]; + assert.ok(helperPid); + + await new Promise((resolve) => setTimeout(resolve, 250)); + assert.doesNotThrow(() => process.kill(helperPid, 0)); + + await waitFor(() => { + try { + process.kill(helperPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); +}); + test("setup reuses an existing shared app-server without starting another one", () => { const repo = makeTempDir(); const binDir = makeTempDir(); From 0f235a0f86a7014b9acd29614fda3b895a42afff Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Mon, 27 Jul 2026 00:55:55 -0700 Subject: [PATCH 05/29] [CC-5219] isolate runtime broker fixtures Attributed-To: Codex interactive --- tests/runtime.test.mjs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 814ea0502..4c0ff907d 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -6,7 +6,7 @@ import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; -import { initGitRepo, makeTempDir, run } from "./helpers.mjs"; +import { initGitRepo, makeTempDir as createTempDir, run } from "./helpers.mjs"; import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; import { loadBrokerSession, saveBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { resolveStateDir } from "../plugins/codex/scripts/lib/state.mjs"; @@ -16,6 +16,35 @@ const PLUGIN_ROOT = path.join(ROOT, "plugins", "codex"); const SCRIPT = path.join(PLUGIN_ROOT, "scripts", "codex-companion.mjs"); const STOP_HOOK = path.join(PLUGIN_ROOT, "scripts", "stop-review-gate-hook.mjs"); const SESSION_HOOK = path.join(PLUGIN_ROOT, "scripts", "session-lifecycle-hook.mjs"); +const runtimeTempDirs = new Set(); +const runtimePluginDataDir = createTempDir("codex-plugin-runtime-state-"); +process.env.CLAUDE_PLUGIN_DATA = runtimePluginDataDir; + +function makeTempDir(prefix) { + const tempDir = createTempDir(prefix); + runtimeTempDirs.add(tempDir); + return tempDir; +} + +test.after(() => { + const cleanupFailures = []; + + for (const cwd of [ROOT, ...runtimeTempDirs]) { + if (!loadBrokerSession(cwd)) { + continue; + } + + const cleanup = run(process.execPath, [SESSION_HOOK, "SessionEnd"], { + cwd, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd }) + }); + if (cleanup.status !== 0 || loadBrokerSession(cwd)) { + cleanupFailures.push({ cwd, status: cleanup.status, stderr: cleanup.stderr }); + } + } + + assert.deepEqual(cleanupFailures, []); +}); async function waitFor(predicate, { timeoutMs = 5000, intervalMs = 50 } = {}) { const start = Date.now(); From 38529f436ef15fbbe7710b6b81bfd30fea4f0717 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Mon, 27 Jul 2026 10:47:41 -0700 Subject: [PATCH 06/29] [CC-5219] address review findings: unwedge disconnected-stream state, non-fatal cleanup verification, async non-blocking process cleanup, ps portability with degraded fallback, de-flaked idle-release tests Attributed-To: Codex executor lane (implementation), Claude interactive (verification, test-anchor fix) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MrRkob3GtGnbYqq6aeE5co --- plugins/codex/scripts/app-server-broker.mjs | 77 +++++++++- plugins/codex/scripts/lib/app-server.mjs | 41 +++-- plugins/codex/scripts/lib/process.mjs | 161 ++++++++++++++++---- tests/process.test.mjs | 116 ++++++++++---- tests/runtime.test.mjs | 131 +++++++++++++++- 5 files changed, 438 insertions(+), 88 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 59f978882..2f0e751e2 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -8,9 +8,11 @@ import process from "node:process"; import { parseArgs } from "./lib/args.mjs"; import { BROKER_BUSY_RPC_CODE, CodexAppServerClient } from "./lib/app-server.mjs"; import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs"; +import { getLiveProcessPids } from "./lib/process.mjs"; const DEFAULT_CHILD_IDLE_MS = 5 * 60 * 1000; const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]); +const BROKER_CLEANUP_UNVERIFIED_RPC_CODE = -32002; function resolveChildIdleMs(env = process.env) { const raw = env.CODEX_COMPANION_BROKER_CHILD_IDLE_MS; @@ -86,6 +88,7 @@ async function main() { let activeStreamSocket = null; let activeStreamThreadIds = null; let activeStreamRunning = false; + let blockedCleanup = null; const sockets = new Set(); function cancelChildIdleClose() { @@ -104,8 +107,30 @@ async function main() { ); } + function clearStreamState() { + activeStreamRunning = false; + activeStreamSocket = null; + activeStreamThreadIds = null; + } + + function recordUnverifiedCleanup(client) { + const outcome = client.cleanupOutcome; + if (!outcome || outcome.verified !== false) { + return; + } + const survivors = outcome.survivors ?? []; + blockedCleanup = { + degraded: Boolean(outcome.degraded) || survivors.length === 0, + survivors + }; + process.stderr.write( + `Warning: shared Codex broker will not spawn a replacement after unverified cleanup; surviving PIDs: ${survivors.join(", ") || "none known"}.\n` + ); + } + async function closeAppClient() { cancelChildIdleClose(); + clearStreamState(); if (appClientStartPromise) { await appClientStartPromise.catch(() => {}); } @@ -118,9 +143,18 @@ async function main() { if (!client) { return; } - appClientClosePromise = client.close().catch(() => {}).finally(() => { - appClientClosePromise = null; - }); + appClientClosePromise = client + .close() + .then(() => { + recordUnverifiedCleanup(client); + }) + .catch((error) => { + blockedCleanup = { degraded: true, survivors: [] }; + process.stderr.write(`Warning: shared Codex broker cleanup failed: ${error.message}. No replacement child will be spawned.\n`); + }) + .finally(() => { + appClientClosePromise = null; + }); await appClientClosePromise; } @@ -144,7 +178,7 @@ async function main() { activeRequestSocket = null; } if (activeStreamSocket === socket) { - activeStreamSocket = null; + clearStreamState(); } } @@ -168,19 +202,48 @@ async function main() { } } + function ensureCleanupSafeToSpawn() { + if (blockedCleanup) { + if (blockedCleanup.survivors.length > 0) { + const liveSurvivors = getLiveProcessPids(blockedCleanup.survivors); + if (liveSurvivors.length > 0) { + const error = new Error(`Shared Codex broker cleanup is unverified; surviving PIDs: ${liveSurvivors.join(", ")}.`); + error.rpcCode = BROKER_CLEANUP_UNVERIFIED_RPC_CODE; + throw error; + } + blockedCleanup = null; + } else if (blockedCleanup.degraded) { + const error = new Error("Shared Codex broker cleanup is unverified; refusing to spawn a replacement child."); + error.rpcCode = BROKER_CLEANUP_UNVERIFIED_RPC_CODE; + throw error; + } else { + blockedCleanup = null; + } + } + } + async function getAppClient() { cancelChildIdleClose(); + ensureCleanupSafeToSpawn(); if (appClient) { return appClient; } if (appClientClosePromise) { await appClientClosePromise; + ensureCleanupSafeToSpawn(); } if (!appClientStartPromise) { appClientStartPromise = CodexAppServerClient.connect(cwd, { disableBroker: true }) .then((client) => { appClient = client; client.setNotificationHandler(routeNotification); + client.setExitHandler(() => { + clearStreamState(); + if (appClient === client) { + appClient = null; + } + scheduleChildIdleClose(); + }); return client; }) .finally(() => { @@ -324,10 +387,8 @@ async function main() { if (activeRequestSocket === socket) { activeRequestSocket = null; } - if (isStreaming && activeStreamSocket === socket) { - activeStreamRunning = false; - activeStreamSocket = null; - activeStreamThreadIds = null; + if (isStreaming) { + clearStreamState(); } } finally { inFlightRequests -= 1; diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index a9f51b747..00261cdc5 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -63,6 +63,7 @@ class AppServerClientBase { this.stderr = ""; this.closed = false; this.exitError = null; + this.exitHandler = null; /** @type {AppServerNotificationHandler | null} */ this.notificationHandler = null; this.lineBuffer = ""; @@ -77,6 +78,10 @@ class AppServerClientBase { this.notificationHandler = handler; } + setExitHandler(handler) { + this.exitHandler = handler; + } + /** * @template {AppServerMethod} M * @param {M} method @@ -173,6 +178,7 @@ class AppServerClientBase { } this.pending.clear(); this.resolveExit(undefined); + this.exitHandler?.(this.exitError); } sendMessage(_message) { @@ -236,7 +242,7 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { }); } catch (error) { try { - process.kill(-this.proc.pid, "SIGKILL"); + this.proc.kill("SIGKILL"); } catch { // The child may have exited before identity capture completed. } @@ -244,7 +250,7 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { } if (process.platform !== "win32" && !this.procIdentity) { try { - process.kill(-this.proc.pid, "SIGKILL"); + this.proc.kill("SIGKILL"); } catch { // The child may have exited before identity capture completed. } @@ -270,22 +276,35 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { if (process.platform === "win32") { setTimeout(() => { if (this.proc && !this.proc.killed && this.proc.exitCode === null) { - try { - terminateProcessTree(this.proc.pid); - } catch { - // Best-effort cleanup inside an unref'd timer — swallow errors - // to avoid crashing the host process during shutdown. - } + void terminateProcessTree(this.proc.pid) + .then((outcome) => { + this.cleanupOutcome = { + verified: outcome.verified ?? null, + survivors: outcome.survivors ?? [] + }; + }) + .catch(() => { + // Best-effort cleanup inside an unref'd timer — swallow errors + // to avoid crashing the host process during shutdown. + }); } }, 50).unref?.(); } else { // The app-server is its own process-group leader on Unix. Terminate // the group so MCP helpers cannot outlive the app-server parent. - const outcome = terminateProcessTree(this.proc.pid, { - expectedRootIdentity: this.procIdentity + const outcome = await terminateProcessTree(this.proc.pid, { + expectedRootIdentity: this.procIdentity, + directKillImpl: (signal) => this.proc.kill(signal), + warnImpl: () => {} }); + this.cleanupOutcome = { + verified: outcome.verified ?? false, + survivors: outcome.survivors ?? [] + }; if (!outcome.verified) { - throw new Error(`Unable to verify codex app-server cleanup; surviving PIDs: ${outcome.survivors.join(", ")}.`); + process.stderr.write( + `Warning: unable to verify codex app-server cleanup; surviving PIDs: ${outcome.survivors?.join(", ") || "none known"}.\n` + ); } } } diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 0ed0cf5df..1c61157d9 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -56,28 +56,48 @@ function looksLikeMissingProcessMessage(text) { const UNIX_PROCESS_TABLE_ARGS = ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]; const UNIX_PS_COMMAND = "/bin/ps"; +const UNIX_PS_PATH_COMMAND = "ps"; + +function createProcessTableError(message) { + const error = new Error(message); + error.code = "PROCESS_TABLE_UNAVAILABLE"; + return error; +} function readUnixProcessTable(runCommandImpl, options = {}) { - const result = runCommandImpl(UNIX_PS_COMMAND, UNIX_PROCESS_TABLE_ARGS, { + let result = runCommandImpl(UNIX_PS_COMMAND, UNIX_PROCESS_TABLE_ARGS, { cwd: options.cwd, env: options.env }); + if (result.error?.code === "ENOENT") { + result = runCommandImpl(UNIX_PS_PATH_COMMAND, UNIX_PROCESS_TABLE_ARGS, { + cwd: options.cwd, + env: options.env + }); + } if (result.error || result.status !== 0) { - const detail = result.error?.message ?? result.stderr.trim() ?? result.stdout.trim() ?? `exit ${result.status}`; - throw new Error(`Unable to enumerate Unix processes: ${detail || `exit ${result.status}`}`); + const detail = result.error?.message || result.stderr.trim() || result.stdout.trim() || `exit ${result.status}`; + throw createProcessTableError(`Unable to enumerate Unix processes: ${detail || `exit ${result.status}`}`); } const processes = new Map(); for (const line of result.stdout.split("\n")) { - const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/); - if (!match) { + const trimmedLine = line.trim(); + if (!trimmedLine) { continue; } + const match = trimmedLine.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/); + if (!match) { + throw createProcessTableError(`Unable to parse Unix process table line: ${trimmedLine}`); + } const pid = Number(match[1]); const parentPid = Number(match[2]); const processGroupId = Number(match[3]); const state = match[4]; const startedAt = match[5].trim(); + if (!Number.isSafeInteger(pid) || !Number.isSafeInteger(parentPid) || !Number.isSafeInteger(processGroupId) || !startedAt) { + throw createProcessTableError(`Unable to parse Unix process table line: ${trimmedLine}`); + } processes.set(pid, { pid, parentPid, @@ -122,11 +142,11 @@ function collectProcessTree(rootPid, processes, rootDepth = 0) { return records; } -function sleepSync(milliseconds) { +function sleep(milliseconds) { if (milliseconds <= 0) { - return; + return Promise.resolve(); } - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds); + return new Promise((resolve) => setTimeout(resolve, milliseconds)); } export function getProcessIdentity(pid, options = {}) { @@ -136,6 +156,32 @@ export function getProcessIdentity(pid, options = {}) { return readUnixProcessTable(options.runCommandImpl ?? runCommand, options).get(pid)?.identity ?? null; } +export function getLiveProcessPids(pids, options = {}) { + const candidates = [...new Set(pids.filter((pid) => Number.isFinite(pid)))]; + if (candidates.length === 0) { + return []; + } + if ((options.platform ?? process.platform) === "win32") { + const killImpl = options.killImpl ?? process.kill.bind(process); + return candidates.filter((pid) => { + try { + killImpl(pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } + }); + } + + try { + const processes = readUnixProcessTable(options.runCommandImpl ?? runCommand, options); + return candidates.filter((pid) => isRunningProcess(processes.get(pid))); + } catch { + // An unverified cleanup remains blocked when liveness cannot be checked. + return candidates; + } +} + function mergeTrackedDescendants(tracked, processes, rootPid, rootIdentity) { const trackedByPid = new Map([...tracked.values()].map((record) => [record.pid, record.identity])); const roots = []; @@ -200,8 +246,7 @@ function buildSignalUnits(records) { return units; } -function signalVerifiedUnit(unit, signal, { runCommandImpl, killImpl, cwd, env }) { - const processes = readUnixProcessTable(runCommandImpl, { cwd, env }); +function signalVerifiedUnit(unit, signal, processes, killImpl) { const current = processes.get(unit.record.pid); if (current?.identity !== unit.record.identity || !isRunningProcess(current)) { return false; @@ -228,12 +273,12 @@ function signalTracked(tracked, signal, options) { const units = buildSignalUnits(listLiveTracked(tracked, processes)); let delivered = false; for (const unit of units) { - delivered = signalVerifiedUnit(unit, signal, options) || delivered; + delivered = signalVerifiedUnit(unit, signal, processes, options.killImpl) || delivered; } return delivered; } -function pollTracked(tracked, options, attempts) { +async function pollTracked(tracked, options, attempts) { let live = []; for (let attempt = 0; attempt < attempts; attempt += 1) { const processes = readUnixProcessTable(options.runCommandImpl, options); @@ -243,13 +288,46 @@ function pollTracked(tracked, options, attempts) { break; } if (attempt + 1 < attempts) { - options.sleepImpl(options.pollIntervalMs); + await options.sleepImpl(options.pollIntervalMs); } } return live; } -export function terminateProcessTree(pid, options = {}) { +function warnProcessCleanup(message, options) { + const warnImpl = options.warnImpl ?? ((warning) => process.stderr.write(`${warning}\n`)); + try { + warnImpl(message); + } catch { + // Cleanup warnings must not turn a best-effort kill into a host failure. + } +} + +function degradedDirectChildKill(pid, options, killImpl, reason) { + const directKillImpl = options.directKillImpl ?? ((signal) => killImpl(pid, signal)); + let delivered = false; + try { + delivered = directKillImpl("SIGKILL") !== false; + } catch { + // The direct child may already have exited. + } + warnProcessCleanup( + `Unable to verify Unix process cleanup for PID ${pid}; used direct-child kill fallback (${String(reason).replace(/\s+/g, " ").trim()}). Surviving PIDs: none known.`, + options + ); + return { + attempted: true, + delivered, + verified: false, + escalated: false, + degraded: true, + method: "direct-child", + targets: [pid], + survivors: [] + }; +} + +export async function terminateProcessTree(pid, options = {}) { if (!Number.isFinite(pid)) { return { attempted: false, delivered: false, method: null }; } @@ -292,7 +370,15 @@ export function terminateProcessTree(pid, options = {}) { throw new Error(formatCommandFailure(result)); } - const initialProcesses = readUnixProcessTable(runCommandImpl, options); + let initialProcesses; + try { + initialProcesses = readUnixProcessTable(runCommandImpl, options); + } catch (error) { + if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { + throw error; + } + return degradedDirectChildKill(pid, options, killImpl, error.message); + } const root = initialProcesses.get(pid); const expectedRootIdentity = options.expectedRootIdentity ?? root?.identity ?? null; if (!root) { @@ -328,28 +414,37 @@ export function terminateProcessTree(pid, options = {}) { killImpl, cwd: options.cwd, env: options.env, - sleepImpl: options.sleepImpl ?? sleepSync, + sleepImpl: options.sleepImpl ?? sleep, pollIntervalMs: options.pollIntervalMs ?? 25 }; - let delivered = signalTracked(tracked, "SIGTERM", unixOptions); - let live = pollTracked(tracked, unixOptions, options.termPollAttempts ?? 11); - let escalated = false; - if (live.length > 0) { - escalated = true; - delivered = signalTracked(tracked, "SIGKILL", unixOptions) || delivered; - live = pollTracked(tracked, unixOptions, options.killPollAttempts ?? 11); - } + try { + let delivered = signalTracked(tracked, "SIGTERM", unixOptions); + let live = await pollTracked(tracked, unixOptions, options.termPollAttempts ?? 11); + let escalated = false; + if (live.length > 0) { + escalated = true; + delivered = signalTracked(tracked, "SIGKILL", unixOptions) || delivered; + live = await pollTracked(tracked, unixOptions, options.killPollAttempts ?? 11); + } - return { - attempted: true, - delivered, - verified: live.length === 0, - escalated, - method: "process-tree", - targets: [...tracked.values()].sort((left, right) => right.depth - left.depth).map((record) => record.pid), - survivors: live.map((record) => record.pid) - }; + return { + attempted: true, + delivered, + verified: live.length === 0, + escalated, + method: "process-tree", + // The algorithm covers same-process-group descendants plus those observed at scan time. + // A post-scan setsid descendant can escape the tracked process tree. + targets: [...tracked.values()].sort((left, right) => right.depth - left.depth).map((record) => record.pid), + survivors: live.map((record) => record.pid) + }; + } catch (error) { + if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { + throw error; + } + return degradedDirectChildKill(pid, options, killImpl, error.message); + } } export function formatCommandFailure(result) { diff --git a/tests/process.test.mjs b/tests/process.test.mjs index c1976b522..7f3b65d98 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -3,9 +3,9 @@ import assert from "node:assert/strict"; import { terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; -test("terminateProcessTree uses taskkill on Windows", () => { +test("terminateProcessTree uses taskkill on Windows", async () => { let captured = null; - const outcome = terminateProcessTree(1234, { + const outcome = await terminateProcessTree(1234, { platform: "win32", runCommandImpl(command, args) { captured = { command, args }; @@ -32,8 +32,8 @@ test("terminateProcessTree uses taskkill on Windows", () => { assert.equal(outcome.method, "taskkill"); }); -test("terminateProcessTree treats missing Windows processes as already stopped", () => { - const outcome = terminateProcessTree(1234, { +test("terminateProcessTree treats missing Windows processes as already stopped", async () => { + const outcome = await terminateProcessTree(1234, { platform: "win32", runCommandImpl(command, args) { return { @@ -54,11 +54,11 @@ test("terminateProcessTree treats missing Windows processes as already stopped", assert.match(outcome.result.stdout, /not found/i); }); -test("terminateProcessTree terminates Unix descendant groups deepest-first", () => { +test("terminateProcessTree terminates Unix descendant groups deepest-first", async () => { const signals = []; const alive = new Set([1234, 1235, 1236, 1237]); const parents = new Map([[1234, 1], [1235, 1234], [1236, 1235], [1237, 1234]]); - const outcome = terminateProcessTree(1234, { + const outcome = await terminateProcessTree(1234, { platform: "darwin", runCommandImpl(command, args) { assert.equal(command, "/bin/ps"); @@ -93,10 +93,10 @@ test("terminateProcessTree terminates Unix descendant groups deepest-first", () assert.deepEqual(outcome.targets, [1236, 1235, 1237, 1234]); }); -test("terminateProcessTree signals a Unix PID directly when it is not a group leader", () => { +test("terminateProcessTree signals a Unix PID directly when it is not a group leader", async () => { const signals = []; let alive = true; - const outcome = terminateProcessTree(1234, { + const outcome = await terminateProcessTree(1234, { platform: "darwin", runCommandImpl(command, args) { return { @@ -121,30 +121,80 @@ test("terminateProcessTree signals a Unix PID directly when it is not a group le assert.equal(outcome.method, "process-tree"); }); -test("terminateProcessTree fails closed when Unix process enumeration fails", () => { - assert.throws( - () => - terminateProcessTree(1234, { - platform: "darwin", - runCommandImpl(command, args) { - return { - command, - args, - status: 1, - signal: null, - stdout: "", - stderr: "ps denied", - error: null - }; - } - }), - /Unable to enumerate Unix processes.*ps denied/i - ); +test("terminateProcessTree parses a captured Linux procps process table", async () => { + const signals = []; + const alive = new Set([42001, 42002]); + const sample = [ + "42001 1 42001 Ss Mon Jul 27 12:34:56 2026", + "42002 42001 42002 S Mon Jul 27 12:34:57 2026" + ].join("\n"); + const outcome = await terminateProcessTree(42001, { + platform: "linux", + runCommandImpl(command, args) { + assert.equal(command, "/bin/ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]); + const stdout = [...alive] + .map((pid) => sample.split("\n").find((line) => line.startsWith(String(pid)))) + .filter(Boolean) + .join("\n"); + return { + command, + args, + status: 0, + signal: null, + stdout: stdout ? `${stdout}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive.delete(Math.abs(pid)); + } + }); + + assert.deepEqual(signals, [[-42002, "SIGTERM"], [-42001, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.deepEqual(outcome.targets, [42002, 42001]); +}); + +test("terminateProcessTree falls back to a direct child kill when Unix process enumeration fails", async () => { + const signals = []; + const warnings = []; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + warnImpl(message) { + warnings.push(message); + }, + runCommandImpl(command, args) { + assert.equal(command, "/bin/ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]); + return { + command, + args, + status: 1, + signal: null, + stdout: "", + stderr: "ps denied", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, [[1234, "SIGKILL"]]); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.deepEqual(outcome.survivors, []); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /direct-child kill fallback.*none known/i); }); -test("terminateProcessTree refuses a reused root PID", () => { +test("terminateProcessTree refuses a reused root PID", async () => { const signals = []; - const outcome = terminateProcessTree(1234, { + const outcome = await terminateProcessTree(1234, { platform: "darwin", expectedRootIdentity: "1234@Sun Jul 26 00:00:00 2026", runCommandImpl(command, args) { @@ -168,11 +218,11 @@ test("terminateProcessTree refuses a reused root PID", () => { assert.equal(outcome.identityMismatch, true); }); -test("terminateProcessTree revalidates descendant identities before signaling", () => { +test("terminateProcessTree revalidates descendant identities before signaling", async () => { const signals = []; let rootAlive = true; let snapshots = 0; - const outcome = terminateProcessTree(1234, { + const outcome = await terminateProcessTree(1234, { platform: "darwin", expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", termPollAttempts: 1, @@ -205,11 +255,11 @@ test("terminateProcessTree revalidates descendant identities before signaling", assert.equal(outcome.escalated, false); }); -test("terminateProcessTree escalates resistant descendants and verifies exit", () => { +test("terminateProcessTree escalates resistant descendants and verifies exit", async () => { const signals = []; const alive = new Set([1234, 1235]); const pgids = new Map([[1234, 1234], [1235, 1235]]); - const outcome = terminateProcessTree(1234, { + const outcome = await terminateProcessTree(1234, { platform: "darwin", expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", termPollAttempts: 1, diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 4c0ff907d..86975115a 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -58,6 +58,64 @@ async function waitFor(predicate, { timeoutMs = 5000, intervalMs = 50 } = {}) { throw new Error("Timed out waiting for condition."); } +function installSlowRejectFakeCodex(binDir) { + installFakeCodex(binDir, "slow-reject"); + const scriptPath = path.join(binDir, "codex"); + const marker = "const turnId = nextTurnId(state);"; + const source = fs.readFileSync(scriptPath, "utf8"); + // The marker also appears in the review/start handler; anchor the patch to + // the turn/start case so the injection lands in the right handler. + const caseStart = source.indexOf('case "turn/start"'); + assert.ok(caseStart !== -1); + const markerAt = source.indexOf(marker, caseStart); + assert.ok(markerAt !== -1); + fs.writeFileSync( + scriptPath, + source.slice(0, markerAt) + + source.slice(markerAt).replace( + marker, + `if (BEHAVIOR === "slow-reject") { + state.turnRejectPending = true; + saveState(state); + setTimeout(() => { + const current = loadState(); + send({ id: message.id, error: { code: -32000, message: "slow fake turn rejected" } }); + current.turnRejectPending = false; + current.turnRejectSent = true; + saveState(current); + }, 400); + break; + } + + ${marker}` + ), + "utf8" + ); +} + +function instrumentSlowFakeTurnState(binDir) { + const scriptPath = path.join(binDir, "codex"); + const source = fs.readFileSync(scriptPath, "utf8"); + const delayedTurn = "emitTurnCompletedLater(thread.id, turnId, items, 400);"; + assert.ok(source.includes(delayedTurn)); + fs.writeFileSync( + scriptPath, + source.replace( + delayedTurn, + `state.turnInFlight = true; + saveState(state); + ${delayedTurn}` + ).replace( + "emitTurnCompleted(threadId, turnId, item);", + `emitTurnCompleted(threadId, turnId, item); + const currentState = loadState(); + currentState.turnInFlight = false; + saveState(currentState);` + ), + "utf8" + ); +} + test("setup reports ready when fake codex is installed and authenticated", () => { const binDir = makeTempDir(); installFakeCodex(binDir); @@ -2191,6 +2249,64 @@ test("commands lazily start and reuse one shared app-server after first use", as assert.equal(cleanup.status, 0, cleanup.stderr); }); +test("shared broker clears a disconnected stream after its request rejects", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installSlowRejectFakeCodex(binDir); + const env = { + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" + }; + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + }); + + const clientA = await CodexAppServerClient.connect(repo, { env }); + const started = await clientA.request("thread/start", { cwd: repo, ephemeral: true }); + const pendingTurn = clientA + .request("turn/start", { + threadId: started.thread.id, + input: [{ type: "text", text: "reject this turn after the client disconnects" }] + }) + .catch(() => null); + + await waitFor(() => { + if (!fs.existsSync(fakeStatePath)) { + return false; + } + return JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).turnRejectPending === true; + }); + await clientA.close(); + await pendingTurn; + + await waitFor(() => { + if (!fs.existsSync(fakeStatePath)) { + return false; + } + return JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).turnRejectSent === true; + }); + + const clientB = await CodexAppServerClient.connect(repo, { env }); + t.after(() => clientB.close().catch(() => {})); + const response = await waitFor(async () => { + try { + return await clientB.request("thread/list", { cwd: repo }); + } catch (error) { + if (error.rpcCode === -32001) { + return false; + } + throw error; + } + }, { timeoutMs: 1500, intervalMs: 20 }); + assert.ok(Array.isArray(response.data)); +}); + test("shared broker releases its idle app-server child and restarts it on demand", async (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); @@ -2240,7 +2356,6 @@ test("shared broker releases its idle app-server child and restarts it on demand const firstFakeState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); const firstHelperPid = firstFakeState.helperPids?.[0]; assert.ok(firstHelperPid); - await new Promise((resolve) => setTimeout(resolve, 300)); await waitFor(() => { try { process.kill(firstHelperPid, 0); @@ -2269,6 +2384,7 @@ test("shared broker keeps active work alive after its client disconnects", async const fakeStatePath = path.join(binDir, "fake-codex-state.json"); installFakeCodex(binDir, "slow-task-with-helper-child"); + instrumentSlowFakeTurnState(binDir); initGitRepo(repo); fs.writeFileSync(path.join(repo, "README.md"), "hello\n"); run("git", ["add", "README.md"], { cwd: repo }); @@ -2298,8 +2414,17 @@ test("shared broker keeps active work alive after its client disconnects", async const helperPid = initialState.helperPids?.[0]; assert.ok(helperPid); - await new Promise((resolve) => setTimeout(resolve, 250)); - assert.doesNotThrow(() => process.kill(helperPid, 0)); + await waitFor(() => { + if (JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).turnInFlight !== true) { + return false; + } + try { + process.kill(helperPid, 0); + return true; + } catch { + return false; + } + }); await waitFor(() => { try { From 16547a0cc882a6f49ea007d1041e97e491da4dd5 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Mon, 27 Jul 2026 11:59:13 -0700 Subject: [PATCH 07/29] [CC-5219] address Codex review: await async teardown at all call sites, keep stream ownership on socket close, reap descendants before their parent Attributed-To: Claude interactive Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MrRkob3GtGnbYqq6aeE5co --- plugins/codex/scripts/app-server-broker.mjs | 5 +++- plugins/codex/scripts/codex-companion.mjs | 2 +- .../codex/scripts/lib/broker-lifecycle.mjs | 8 ++--- plugins/codex/scripts/lib/process.mjs | 29 +++++++++++++++---- .../codex/scripts/session-lifecycle-hook.mjs | 8 ++--- 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 2f0e751e2..33304883b 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -178,7 +178,10 @@ async function main() { activeRequestSocket = null; } if (activeStreamSocket === socket) { - clearStreamState(); + // Detach only the notification socket. The turn may still be running in + // the app-server; stream state is cleared on turn/completed, on a + // streaming request error, on child exit, or when the child is released. + activeStreamSocket = null; } } diff --git a/plugins/codex/scripts/codex-companion.mjs b/plugins/codex/scripts/codex-companion.mjs index 83df468ad..4ca3c9d0c 100644 --- a/plugins/codex/scripts/codex-companion.mjs +++ b/plugins/codex/scripts/codex-companion.mjs @@ -983,7 +983,7 @@ async function handleCancel(argv) { ); } - terminateProcessTree(job.pid ?? Number.NaN); + await terminateProcessTree(job.pid ?? Number.NaN); appendLogLine(job.logFile, "Cancelled by user."); const completedAt = nowIso(); diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index ef763819c..2822dac57 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -117,7 +117,7 @@ export async function ensureBrokerSession(cwd, options = {}) { } if (existing) { - teardownBrokerSession({ + await teardownBrokerSession({ endpoint: existing.endpoint ?? null, pidFile: existing.pidFile ?? null, logFile: existing.logFile ?? null, @@ -148,7 +148,7 @@ export async function ensureBrokerSession(cwd, options = {}) { const ready = await waitForBrokerEndpoint(endpoint, options.timeoutMs ?? 2000); if (!ready) { - teardownBrokerSession({ + await teardownBrokerSession({ endpoint, pidFile, logFile, @@ -170,10 +170,10 @@ export async function ensureBrokerSession(cwd, options = {}) { return session; } -export function teardownBrokerSession({ endpoint = null, pidFile, logFile, sessionDir = null, pid = null, killProcess = null }) { +export async function teardownBrokerSession({ endpoint = null, pidFile, logFile, sessionDir = null, pid = null, killProcess = null }) { if (Number.isFinite(pid) && killProcess) { try { - killProcess(pid); + await killProcess(pid); } catch { // Ignore missing or already-exited broker processes. } diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 1c61157d9..950e36435 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -418,14 +418,31 @@ export async function terminateProcessTree(pid, options = {}) { pollIntervalMs: options.pollIntervalMs ?? 25 }; + // Terminate descendants first and wait for them while their parent is still + // alive, so the parent can reap them. Killing the whole tree back-to-back + // leaves permanent zombies where PID 1 does not reap orphans (containers). + const descendants = new Map(); + const roots = new Map(); + for (const [identity, record] of tracked) { + (record.pid === pid ? roots : descendants).set(identity, record); + } + try { - let delivered = signalTracked(tracked, "SIGTERM", unixOptions); - let live = await pollTracked(tracked, unixOptions, options.termPollAttempts ?? 11); + let delivered = false; let escalated = false; - if (live.length > 0) { - escalated = true; - delivered = signalTracked(tracked, "SIGKILL", unixOptions) || delivered; - live = await pollTracked(tracked, unixOptions, options.killPollAttempts ?? 11); + let live = []; + for (const phase of [descendants, roots]) { + if (phase.size === 0) { + continue; + } + delivered = signalTracked(phase, "SIGTERM", unixOptions) || delivered; + let phaseLive = await pollTracked(phase, unixOptions, options.termPollAttempts ?? 11); + if (phaseLive.length > 0) { + escalated = true; + delivered = signalTracked(phase, "SIGKILL", unixOptions) || delivered; + phaseLive = await pollTracked(phase, unixOptions, options.killPollAttempts ?? 11); + } + live = live.concat(phaseLive); } return { diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 778571e6c..0f54a6b09 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -39,7 +39,7 @@ function appendEnvVar(name, value) { fs.appendFileSync(process.env.CLAUDE_ENV_FILE, `export ${name}=${shellEscape(value)}\n`, "utf8"); } -function cleanupSessionJobs(cwd, sessionId) { +async function cleanupSessionJobs(cwd, sessionId) { if (!cwd || !sessionId) { return; } @@ -62,7 +62,7 @@ function cleanupSessionJobs(cwd, sessionId) { continue; } try { - terminateProcessTree(job.pid ?? Number.NaN); + await terminateProcessTree(job.pid ?? Number.NaN); } catch { // Ignore teardown failures during session shutdown. } @@ -101,8 +101,8 @@ async function handleSessionEnd(input) { await sendBrokerShutdown(brokerEndpoint); } - cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); - teardownBrokerSession({ + await cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); + await teardownBrokerSession({ endpoint: brokerEndpoint, pidFile, logFile, From 58d84b1801d8c58f01a012a18fb690c65d7da12a Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Mon, 27 Jul 2026 12:13:47 -0700 Subject: [PATCH 08/29] [CC-5219] address Codex review round 2: exclude root from descendant kill phase, reclaim orphaned process group on unexpected child exit Attributed-To: Claude interactive Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MrRkob3GtGnbYqq6aeE5co --- plugins/codex/scripts/app-server-broker.mjs | 24 +++- plugins/codex/scripts/lib/process.mjs | 119 ++++++++++++++++---- tests/process.test.mjs | 77 ++++++++++++- 3 files changed, 197 insertions(+), 23 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 33304883b..13f1ad53d 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -8,7 +8,7 @@ import process from "node:process"; import { parseArgs } from "./lib/args.mjs"; import { BROKER_BUSY_RPC_CODE, CodexAppServerClient } from "./lib/app-server.mjs"; import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs"; -import { getLiveProcessPids } from "./lib/process.mjs"; +import { getLiveProcessPids, terminateProcessGroup } from "./lib/process.mjs"; const DEFAULT_CHILD_IDLE_MS = 5 * 60 * 1000; const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]); @@ -240,11 +240,33 @@ async function main() { .then((client) => { appClient = client; client.setNotificationHandler(routeNotification); + const childPid = client.proc?.pid ?? null; client.setExitHandler(() => { clearStreamState(); if (appClient === client) { appClient = null; } + if (!client.closed && childPid != null && process.platform !== "win32") { + // The child is a detached process-group leader; on an unexpected + // exit its surviving helpers reparent away from the broker, so + // reclaim the group before allowing a replacement to spawn. + appClientClosePromise = terminateProcessGroup(childPid) + .then((outcome) => { + client.cleanupOutcome = { + verified: outcome.verified ?? false, + survivors: outcome.survivors ?? [], + degraded: outcome.degraded ?? false + }; + recordUnverifiedCleanup(client); + }) + .catch((error) => { + blockedCleanup = { degraded: true, survivors: [] }; + process.stderr.write(`Warning: shared Codex broker could not reclaim exited app-server group: ${error.message}. No replacement child will be spawned.\n`); + }) + .finally(() => { + appClientClosePromise = null; + }); + } scheduleChildIdleClose(); }); return client; diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 950e36435..f64eb8112 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -267,10 +267,17 @@ function signalVerifiedUnit(unit, signal, processes, killImpl) { } } +function withoutExcluded(records, excludePids) { + if (!excludePids || excludePids.size === 0) { + return records; + } + return records.filter((record) => !excludePids.has(record.pid)); +} + function signalTracked(tracked, signal, options) { const processes = readUnixProcessTable(options.runCommandImpl, options); mergeTrackedDescendants(tracked, processes, options.rootPid, options.rootIdentity); - const units = buildSignalUnits(listLiveTracked(tracked, processes)); + const units = buildSignalUnits(withoutExcluded(listLiveTracked(tracked, processes), options.excludePids)); let delivered = false; for (const unit of units) { delivered = signalVerifiedUnit(unit, signal, processes, options.killImpl) || delivered; @@ -283,7 +290,7 @@ async function pollTracked(tracked, options, attempts) { for (let attempt = 0; attempt < attempts; attempt += 1) { const processes = readUnixProcessTable(options.runCommandImpl, options); mergeTrackedDescendants(tracked, processes, options.rootPid, options.rootIdentity); - live = listLiveTracked(tracked, processes); + live = withoutExcluded(listLiveTracked(tracked, processes), options.excludePids); if (live.length === 0) { break; } @@ -418,31 +425,29 @@ export async function terminateProcessTree(pid, options = {}) { pollIntervalMs: options.pollIntervalMs ?? 25 }; - // Terminate descendants first and wait for them while their parent is still - // alive, so the parent can reap them. Killing the whole tree back-to-back - // leaves permanent zombies where PID 1 does not reap orphans (containers). - const descendants = new Map(); - const roots = new Map(); - for (const [identity, record] of tracked) { - (record.pid === pid ? roots : descendants).set(identity, record); - } - try { let delivered = false; let escalated = false; let live = []; - for (const phase of [descendants, roots]) { - if (phase.size === 0) { - continue; - } - delivered = signalTracked(phase, "SIGTERM", unixOptions) || delivered; - let phaseLive = await pollTracked(phase, unixOptions, options.termPollAttempts ?? 11); + // Terminate descendants first and wait for them while their parent is + // still alive, so the parent can reap them; killing the whole tree + // back-to-back leaves permanent zombies where PID 1 does not reap orphans + // (containers). Both phases share the one tracked map so PID-reuse + // identity memory carries across phases; the descendant phase only + // excludes the root pid from signaling and liveness. + const hasDescendants = [...tracked.values()].some((record) => record.pid !== pid); + const phases = hasDescendants + ? [{ ...unixOptions, excludePids: new Set([pid]) }, unixOptions] + : [unixOptions]; + for (const phaseOptions of phases) { + delivered = signalTracked(tracked, "SIGTERM", phaseOptions) || delivered; + let phaseLive = await pollTracked(tracked, phaseOptions, options.termPollAttempts ?? 11); if (phaseLive.length > 0) { escalated = true; - delivered = signalTracked(phase, "SIGKILL", unixOptions) || delivered; - phaseLive = await pollTracked(phase, unixOptions, options.killPollAttempts ?? 11); + delivered = signalTracked(tracked, "SIGKILL", phaseOptions) || delivered; + phaseLive = await pollTracked(tracked, phaseOptions, options.killPollAttempts ?? 11); } - live = live.concat(phaseLive); + live = phaseLive; } return { @@ -464,6 +469,80 @@ export async function terminateProcessTree(pid, options = {}) { } } +export async function terminateProcessGroup(pgid, options = {}) { + if (!Number.isFinite(pgid)) { + return { attempted: false, delivered: false, verified: true, survivors: [] }; + } + if ((options.platform ?? process.platform) === "win32") { + return { attempted: false, delivered: false, verified: true, survivors: [] }; + } + + const runCommandImpl = options.runCommandImpl ?? runCommand; + const killImpl = options.killImpl ?? process.kill.bind(process); + + let processes; + try { + processes = readUnixProcessTable(runCommandImpl, options); + } catch (error) { + if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { + throw error; + } + warnProcessCleanup( + `Unable to enumerate Unix processes while reclaiming process group ${pgid}; surviving PIDs unknown.`, + options + ); + return { attempted: false, delivered: false, verified: false, degraded: true, survivors: [] }; + } + + const tracked = new Map(); + for (const record of processes.values()) { + if (record.processGroupId === pgid && isRunningProcess(record)) { + tracked.set(record.identity, record); + } + } + if (tracked.size === 0) { + return { attempted: false, delivered: false, verified: true, survivors: [] }; + } + + const unixOptions = { + runCommandImpl, + killImpl, + cwd: options.cwd, + env: options.env, + sleepImpl: options.sleepImpl ?? sleep, + pollIntervalMs: options.pollIntervalMs ?? 25 + }; + + try { + let delivered = signalTracked(tracked, "SIGTERM", unixOptions); + let live = await pollTracked(tracked, unixOptions, options.termPollAttempts ?? 11); + let escalated = false; + if (live.length > 0) { + escalated = true; + delivered = signalTracked(tracked, "SIGKILL", unixOptions) || delivered; + live = await pollTracked(tracked, unixOptions, options.killPollAttempts ?? 11); + } + return { + attempted: true, + delivered, + verified: live.length === 0, + escalated, + method: "process-group", + targets: [...tracked.values()].map((record) => record.pid), + survivors: live.map((record) => record.pid) + }; + } catch (error) { + if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { + throw error; + } + warnProcessCleanup( + `Unable to verify Unix process-group cleanup for pgid ${pgid}; surviving PIDs unknown.`, + options + ); + return { attempted: true, delivered: true, verified: false, degraded: true, survivors: [] }; + } +} + export function formatCommandFailure(result) { const parts = [`${result.command} ${result.args.join(" ")}`.trim()]; if (result.signal) { diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 7f3b65d98..9dbc71636 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -1,7 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; +import { terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; test("terminateProcessTree uses taskkill on Windows", async () => { let captured = null; @@ -296,10 +296,83 @@ test("terminateProcessTree escalates resistant descendants and verifies exit", a assert.deepEqual(signals, [ [-1235, "SIGTERM"], - [-1234, "SIGTERM"], [-1235, "SIGKILL"], + [-1234, "SIGTERM"], [-1234, "SIGKILL"] ]); assert.equal(outcome.verified, true); assert.equal(outcome.escalated, true); }); + +test("terminateProcessTree keeps the root alive until its descendants are reaped", async () => { + const signals = []; + const alive = new Set([100, 200]); + const outcome = await terminateProcessTree(100, { + platform: "darwin", + pollIntervalMs: 0, + runCommandImpl(command, args) { + const rows = []; + if (alive.has(100)) { + rows.push("100 1 100 S Mon Jul 27 00:00:00 2026"); + } + if (alive.has(200)) { + rows.push("200 100 200 S Mon Jul 27 00:00:01 2026"); + } + return { + command, + args, + status: 0, + signal: null, + stdout: rows.length ? `${rows.join("\n")}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + // The helper resists SIGTERM so the descendant phase must escalate + // before the root may be signaled at all. + if (Math.abs(pid) === 200 && signal === "SIGTERM") { + return; + } + alive.delete(Math.abs(pid)); + } + }); + + assert.deepEqual(signals, [ + [-200, "SIGTERM"], + [-200, "SIGKILL"], + [-100, "SIGTERM"] + ]); + assert.equal(outcome.verified, true); + assert.equal(outcome.escalated, true); +}); + +test("terminateProcessGroup reclaims orphaned members of a dead leader's group", async () => { + const signals = []; + const alive = new Set([202]); + const outcome = await terminateProcessGroup(200, { + platform: "darwin", + pollIntervalMs: 0, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: alive.has(202) ? "202 1 200 S Mon Jul 27 00:00:02 2026\n" : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive.delete(Math.abs(pid)); + } + }); + + assert.deepEqual(signals, [[202, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.equal(outcome.method, "process-group"); + assert.deepEqual(outcome.targets, [202]); +}); From 59a40a38dfead67ac09b2cd4919364e7e710097a Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Mon, 27 Jul 2026 13:03:19 -0700 Subject: [PATCH 09/29] [CC-5219] track every observed member of a signaled process group A TERM-resistant helper whose parent already exited is reparented out of the root's tree, so it was signaled by the group kill but never tracked: polling saw no live records, skipped SIGKILL, and reported verified while the helper was still alive. Group members observed at scan time now enter the shared tracked map, so they escalate and count as survivors. Attributed-To: Codex executor lane Co-Authored-By: Claude Opus 5 --- plugins/codex/scripts/lib/process.mjs | 23 +++++++++++++ tests/process.test.mjs | 47 +++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index f64eb8112..197b874a8 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -246,6 +246,28 @@ function buildSignalUnits(records) { return units; } +function mergeTrackedGroupMembers(tracked, processes, units) { + const trackedByPid = new Map([...tracked.values()].map((record) => [record.pid, record.identity])); + for (const unit of units) { + if (!unit.group) { + continue; + } + for (const record of processes.values()) { + if (record.processGroupId !== unit.record.processGroupId) { + continue; + } + const priorIdentity = trackedByPid.get(record.pid); + if (priorIdentity && priorIdentity !== record.identity) { + continue; + } + if (!tracked.has(record.identity)) { + tracked.set(record.identity, { ...record, depth: unit.record.depth + 1 }); + trackedByPid.set(record.pid, record.identity); + } + } + } +} + function signalVerifiedUnit(unit, signal, processes, killImpl) { const current = processes.get(unit.record.pid); if (current?.identity !== unit.record.identity || !isRunningProcess(current)) { @@ -277,6 +299,7 @@ function withoutExcluded(records, excludePids) { function signalTracked(tracked, signal, options) { const processes = readUnixProcessTable(options.runCommandImpl, options); mergeTrackedDescendants(tracked, processes, options.rootPid, options.rootIdentity); + mergeTrackedGroupMembers(tracked, processes, buildSignalUnits(listLiveTracked(tracked, processes))); const units = buildSignalUnits(withoutExcluded(listLiveTracked(tracked, processes), options.excludePids)); let delivered = false; for (const unit of units) { diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 9dbc71636..6e9e3ae09 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -348,6 +348,53 @@ test("terminateProcessTree keeps the root alive until its descendants are reaped assert.equal(outcome.escalated, true); }); +test("terminateProcessTree tracks reparented members of a signaled process group", async () => { + async function runScenario(helperSurvivesKill) { + const signals = []; + const alive = new Set([100, 200]); + const outcome = await terminateProcessTree(100, { + platform: "darwin", + termPollAttempts: 1, + killPollAttempts: 1, + sleepImpl() {}, + runCommandImpl(command, args) { + const rows = []; + if (alive.has(100)) { + rows.push("100 1 100 S Mon Jul 27 00:00:00 2026"); + } + if (alive.has(200)) { + rows.push("200 1 100 S Mon Jul 27 00:00:01 2026"); + } + return { + command, + args, + status: 0, + signal: null, + stdout: rows.length ? `${rows.join("\n")}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + if (signal === "SIGTERM") { + alive.delete(100); + } else if (!helperSurvivesKill) { + alive.delete(200); + } + } + }); + + assert.deepEqual(signals, [[-100, "SIGTERM"], [200, "SIGKILL"]]); + assert.equal(outcome.escalated, true); + assert.equal(outcome.verified, !helperSurvivesKill); + assert.deepEqual(outcome.survivors, helperSurvivesKill ? [200] : []); + } + + await runScenario(false); + await runScenario(true); +}); + test("terminateProcessGroup reclaims orphaned members of a dead leader's group", async () => { const signals = []; const alive = new Set([202]); From 386411c99fec825be8bc9ac3e066530955357536 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Mon, 27 Jul 2026 13:03:19 -0700 Subject: [PATCH 10/29] [CC-5219] terminate the spawned app-server when initialization fails connect() rejected before returning the client, so a detached app-server that answered initialize with an RPC error but stayed alive was never closed, and every later request could spawn another live helper tree. connect() now closes its own client on a failed initialize and relays an unverified cleanup outcome to the broker's existing refusal gate. Attributed-To: Codex executor lane Co-Authored-By: Claude Opus 5 --- plugins/codex/scripts/app-server-broker.mjs | 6 ++ plugins/codex/scripts/lib/app-server.mjs | 20 +++- tests/runtime.test.mjs | 100 +++++++++++++++++++- 3 files changed, 123 insertions(+), 3 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 13f1ad53d..9e741f21a 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -271,6 +271,12 @@ async function main() { }); return client; }) + .catch((error) => { + if (error.cleanupOutcome) { + recordUnverifiedCleanup({ cleanupOutcome: error.cleanupOutcome }); + } + throw error; + }) .finally(() => { appClientStartPromise = null; }); diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index 00261cdc5..231b06f5f 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -395,7 +395,23 @@ export class CodexAppServerClient { const client = brokerEndpoint ? new BrokerCodexAppServerClient(cwd, { ...options, brokerEndpoint }) : new SpawnedCodexAppServerClient(cwd, options); - await client.initialize(); - return client; + try { + await client.initialize(); + return client; + } catch (error) { + try { + await client.close(); + } catch { + client.cleanupOutcome = { + verified: false, + survivors: [], + degraded: true + }; + } + if (client.cleanupOutcome?.verified === false) { + error.cleanupOutcome = client.cleanupOutcome; + } + throw error; + } } } diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 86975115a..6feb59bd2 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -8,7 +8,7 @@ import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; import { initGitRepo, makeTempDir as createTempDir, run } from "./helpers.mjs"; import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; -import { loadBrokerSession, saveBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { ensureBrokerSession, loadBrokerSession, saveBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { resolveStateDir } from "../plugins/codex/scripts/lib/state.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -93,6 +93,32 @@ function installSlowRejectFakeCodex(binDir) { ); } +function installInitializeErrorFakeCodex(binDir) { + installFakeCodex(binDir, "initialize-rpc-error"); + const scriptPath = path.join(binDir, "codex"); + let source = fs.readFileSync(scriptPath, "utf8"); + const startsMarker = "bootState.appServerStarts = (bootState.appServerStarts || 0) + 1;"; + assert.ok(source.includes(startsMarker)); + source = source.replace( + startsMarker, + `${startsMarker} +bootState.appServerPids = [...(bootState.appServerPids || []), process.pid];` + ); + const initializeMarker = ` case "initialize": + state.capabilities = message.params.capabilities || null;`; + assert.ok(source.includes(initializeMarker)); + source = source.replace( + initializeMarker, + ` case "initialize": + if (BEHAVIOR === "initialize-rpc-error") { + send({ id: message.id, error: { code: -32010, message: "initialize failed" } }); + break; + } + state.capabilities = message.params.capabilities || null;` + ); + fs.writeFileSync(scriptPath, source, "utf8"); +} + function instrumentSlowFakeTurnState(binDir) { const scriptPath = path.join(binDir, "codex"); const source = fs.readFileSync(scriptPath, "utf8"); @@ -2307,6 +2333,78 @@ test("shared broker clears a disconnected stream after its request rejects", asy assert.ok(Array.isArray(response.data)); }); +test("shared broker cleans up an app-server whose initialization returns an RPC error", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installInitializeErrorFakeCodex(binDir); + const env = { + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" + }; + const brokerSession = await ensureBrokerSession(repo, { env }); + t.after(() => { + if (brokerSession) { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + } + if (fs.existsSync(fakeStatePath)) { + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + for (const pid of state.appServerPids || []) { + try { + process.kill(pid, "SIGKILL"); + } catch { + // Ignore children already terminated by the cleanup path. + } + } + } + }); + + async function requestUntilInitializationFails() { + const request = brokerSession + ? (() => { + const client = CodexAppServerClient.connect(repo, { env }); + return client.then((connectedClient) => connectedClient.request("thread/list", { cwd: repo })); + })() + : CodexAppServerClient.connect(repo, { env, disableBroker: true }); + await assert.rejects( + request, + (error) => + (error.rpcCode === -32010 && /initialize failed/.test(error.message)) || + (error.rpcCode === -32002 && /refusing to spawn a replacement child/i.test(error.message)) + ); + } + + function loadFakeState() { + return JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + } + + function isLive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } + } + + await requestUntilInitializationFails(); + await waitFor(() => { + const state = loadFakeState(); + return state.appServerStarts === 1 && state.appServerPids?.length === 1 && !isLive(state.appServerPids[0]); + }); + + await requestUntilInitializationFails(); + await waitFor(() => { + const state = loadFakeState(); + return state.appServerStarts <= 2 && state.appServerPids?.length <= 2 && state.appServerPids.every((pid) => !isLive(pid)); + }); +}); + test("shared broker releases its idle app-server child and restarts it on demand", async (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); From c663c57f41c5199218a09b3eb788ecb7f33b6592 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Mon, 27 Jul 2026 21:11:41 -0700 Subject: [PATCH 11/29] Never report cleanup as verified without evidence An adversarial audit found this branch's cleanup layer repeatedly claiming success it could not prove: an empty process-group selection, a missing root, and a dead root all produced verified: true while a detached helper could still be alive. This replaces those claims with one contract. Ownership is captured while the root is alive, so cleanup consults what it recorded owning rather than rediscovering a tree whose root has since exited. Absence of evidence is no longer proof of death: an unreadable process table, an unaccounted-for owned descendant, or a missing root all report verified: false. A root positively observed absent from a readable table, with nothing owned unaccounted for, is still a clean success, so cancelling an already-exited job does not surface a spurious error. Survivors are tracked as pid@lstart identities rather than bare pids, so pid reuse can neither mask a survivor nor wedge the broker permanently. Also: identity-capture failure no longer pre-kills the child in a way that made close() skip tree cleanup and suppress the unverified-cleanup gate; the broker closes its listener before awaiting child cleanup during shutdown, so a late client cannot start a replacement app-server it would then abandon; and callers no longer discard an unverified cleanup outcome and delete the ownership records needed to retry it. Attributed-To: Codex executor lane Co-Authored-By: Claude Opus 5 --- plugins/codex/scripts/app-server-broker.mjs | 84 +++++-- plugins/codex/scripts/codex-companion.mjs | 46 +++- plugins/codex/scripts/lib/app-server.mjs | 70 +++--- .../codex/scripts/lib/broker-lifecycle.mjs | 55 ++++- plugins/codex/scripts/lib/process.mjs | 191 ++++++++++++--- .../codex/scripts/session-lifecycle-hook.mjs | 82 +++++-- tests/fake-codex-fixture.mjs | 13 +- tests/process.test.mjs | 128 +++++++++- tests/runtime.test.mjs | 219 +++++++++++++++++- 9 files changed, 773 insertions(+), 115 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 9e741f21a..3cd1a4ebf 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -4,15 +4,21 @@ import fs from "node:fs"; import net from "node:net"; import path from "node:path"; import process from "node:process"; +import { fileURLToPath } from "node:url"; import { parseArgs } from "./lib/args.mjs"; import { BROKER_BUSY_RPC_CODE, CodexAppServerClient } from "./lib/app-server.mjs"; import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs"; -import { getLiveProcessPids, terminateProcessGroup } from "./lib/process.mjs"; +import { getLiveProcessPids, normalizeProcessCleanupOutcome, terminateProcessGroup } from "./lib/process.mjs"; const DEFAULT_CHILD_IDLE_MS = 5 * 60 * 1000; const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]); const BROKER_CLEANUP_UNVERIFIED_RPC_CODE = -32002; +const BROKER_SHUTDOWN_RPC_CODE = -32003; + +export function isBrokerRequestAllowedDuringShutdown(shuttingDown, message) { + return !shuttingDown || message?.method === "broker/shutdown"; +} function resolveChildIdleMs(env = process.env) { const raw = env.CODEX_COMPANION_BROKER_CHILD_IDLE_MS; @@ -45,6 +51,15 @@ function send(socket, message) { socket.write(`${JSON.stringify(message)}\n`); } +function flushSocket(socket) { + if (socket.destroyed) { + return Promise.resolve(false); + } + return new Promise((resolve) => { + socket.write("", () => resolve(true)); + }); +} + function isInterruptRequest(message) { return message?.method === "turn/interrupt"; } @@ -84,6 +99,7 @@ async function main() { let childIdleTimer = null; let inFlightRequests = 0; let shutdownPromise = null; + let shuttingDown = false; let activeRequestSocket = null; let activeStreamSocket = null; let activeStreamThreadIds = null; @@ -121,7 +137,8 @@ async function main() { const survivors = outcome.survivors ?? []; blockedCleanup = { degraded: Boolean(outcome.degraded) || survivors.length === 0, - survivors + survivors, + survivorIdentities: outcome.survivorIdentities ?? [] }; process.stderr.write( `Warning: shared Codex broker will not spawn a replacement after unverified cleanup; surviving PIDs: ${survivors.join(", ") || "none known"}.\n` @@ -208,7 +225,9 @@ async function main() { function ensureCleanupSafeToSpawn() { if (blockedCleanup) { if (blockedCleanup.survivors.length > 0) { - const liveSurvivors = getLiveProcessPids(blockedCleanup.survivors); + const liveSurvivors = getLiveProcessPids(blockedCleanup.survivors, { + identities: blockedCleanup.survivorIdentities + }); if (liveSurvivors.length > 0) { const error = new Error(`Shared Codex broker cleanup is unverified; surviving PIDs: ${liveSurvivors.join(", ")}.`); error.rpcCode = BROKER_CLEANUP_UNVERIFIED_RPC_CODE; @@ -226,6 +245,11 @@ async function main() { } async function getAppClient() { + if (shuttingDown) { + const error = new Error("Shared Codex broker is shutting down."); + error.rpcCode = BROKER_SHUTDOWN_RPC_CODE; + throw error; + } cancelChildIdleClose(); ensureCleanupSafeToSpawn(); if (appClient) { @@ -233,6 +257,11 @@ async function main() { } if (appClientClosePromise) { await appClientClosePromise; + if (shuttingDown) { + const error = new Error("Shared Codex broker is shutting down."); + error.rpcCode = BROKER_SHUTDOWN_RPC_CODE; + throw error; + } ensureCleanupSafeToSpawn(); } if (!appClientStartPromise) { @@ -250,13 +279,11 @@ async function main() { // The child is a detached process-group leader; on an unexpected // exit its surviving helpers reparent away from the broker, so // reclaim the group before allowing a replacement to spawn. - appClientClosePromise = terminateProcessGroup(childPid) + appClientClosePromise = terminateProcessGroup(childPid, { + ownershipSnapshot: client.ownershipSnapshot + }) .then((outcome) => { - client.cleanupOutcome = { - verified: outcome.verified ?? false, - survivors: outcome.survivors ?? [], - degraded: outcome.degraded ?? false - }; + client.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); recordUnverifiedCleanup(client); }) .catch((error) => { @@ -288,13 +315,17 @@ async function main() { if (shutdownPromise) { return shutdownPromise; } + shuttingDown = true; + const serverClosePromise = new Promise((resolve) => { + server.close(resolve); + }); shutdownPromise = (async () => { cancelChildIdleClose(); for (const socket of sockets) { - socket.end(); + socket.destroy(); } await closeAppClient(); - await new Promise((resolve) => server.close(resolve)); + await serverClosePromise; if (listenTarget.kind === "unix" && fs.existsSync(listenTarget.path)) { fs.unlinkSync(listenTarget.path); } @@ -306,12 +337,20 @@ async function main() { } const server = net.createServer((socket) => { + if (shuttingDown) { + socket.destroy(); + return; + } cancelChildIdleClose(); sockets.add(socket); socket.setEncoding("utf8"); let buffer = ""; socket.on("data", async (chunk) => { + if (shuttingDown) { + socket.destroy(); + return; + } buffer += chunk; let newlineIndex = buffer.indexOf("\n"); while (newlineIndex !== -1) { @@ -349,11 +388,24 @@ async function main() { } if (message.id !== undefined && message.method === "broker/shutdown") { + shuttingDown = true; send(socket, { id: message.id, result: {} }); + await flushSocket(socket); await shutdown(server); process.exit(0); } + if (!isBrokerRequestAllowedDuringShutdown(shuttingDown, message)) { + if (message.id !== undefined) { + send(socket, { + id: message.id, + error: buildJsonRpcError(BROKER_SHUTDOWN_RPC_CODE, "Shared Codex broker is shutting down.") + }); + } + socket.destroy(); + continue; + } + if (message.id === undefined) { continue; } @@ -454,7 +506,9 @@ async function main() { server.listen(listenTarget.path); } -main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - process.exit(1); -}); +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/plugins/codex/scripts/codex-companion.mjs b/plugins/codex/scripts/codex-companion.mjs index 4ca3c9d0c..12992ea4a 100644 --- a/plugins/codex/scripts/codex-companion.mjs +++ b/plugins/codex/scripts/codex-companion.mjs @@ -4,7 +4,7 @@ import { spawn } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import process from "node:process"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs, splitRawArgumentString } from "./lib/args.mjs"; import { @@ -960,7 +960,7 @@ function handleTaskResumeCandidate(argv) { outputCommandResult(payload, rendered, options.json); } -async function handleCancel(argv) { +export async function handleCancel(argv, dependencies = {}) { const { options, positionals } = parseCommandInput(argv, { valueOptions: ["cwd"], booleanOptions: ["json"] @@ -973,7 +973,7 @@ async function handleCancel(argv) { const threadId = existing.threadId ?? job.threadId ?? null; const turnId = existing.turnId ?? job.turnId ?? null; - const interrupt = await interruptAppServerTurn(cwd, { threadId, turnId }); + const interrupt = await (dependencies.interruptAppServerTurnImpl ?? interruptAppServerTurn)(cwd, { threadId, turnId }); if (interrupt.attempted) { appendLogLine( job.logFile, @@ -983,7 +983,33 @@ async function handleCancel(argv) { ); } - await terminateProcessTree(job.pid ?? Number.NaN); + const cleanupOutcome = await (dependencies.terminateProcessTreeImpl ?? terminateProcessTree)(job.pid ?? Number.NaN, { + expectedRootIdentity: existing.processIdentity ?? job.processIdentity ?? null, + ownershipSnapshot: existing.ownershipSnapshot ?? job.ownershipSnapshot ?? null + }); + if (cleanupOutcome?.verified !== true) { + const failureMessage = `Unable to verify cleanup for ${job.id}; ownership records were preserved for retry.`; + appendLogLine(job.logFile, failureMessage); + const recoveryRecord = { + ...existing, + ...job, + status: job.status, + phase: "cleanup-pending", + pid: job.pid ?? existing.pid ?? null, + cleanupOutcome, + cleanupFailure: failureMessage + }; + writeJobFile(workspaceRoot, job.id, recoveryRecord); + upsertJob(workspaceRoot, { + id: job.id, + status: job.status, + phase: "cleanup-pending", + pid: job.pid ?? existing.pid ?? null, + cleanupOutcome, + cleanupFailure: failureMessage + }); + throw new Error(failureMessage); + } appendLogLine(job.logFile, "Cancelled by user."); const completedAt = nowIso(); @@ -1066,8 +1092,10 @@ async function main() { } } -main().catch((error) => { - const message = error instanceof Error ? error.message : String(error); - process.stderr.write(`${message}\n`); - process.exitCode = 1; -}); +if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) { + main().catch((error) => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${message}\n`); + process.exitCode = 1; + }); +} diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index 231b06f5f..a33963589 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -14,10 +14,11 @@ import { spawn } from "node:child_process"; import readline from "node:readline"; import { parseBrokerEndpoint } from "./broker-endpoint.mjs"; import { ensureBrokerSession, loadBrokerSession } from "./broker-lifecycle.mjs"; -import { getProcessIdentity, terminateProcessTree } from "./process.mjs"; +import { captureProcessOwnership, normalizeProcessCleanupOutcome, terminateProcessTree } from "./process.mjs"; const PLUGIN_MANIFEST_URL = new URL("../../.claude-plugin/plugin.json", import.meta.url); const PLUGIN_MANIFEST = JSON.parse(fs.readFileSync(PLUGIN_MANIFEST_URL, "utf8")); +const DEFAULT_CLOSE_WAIT_MS = 2000; export const BROKER_ENDPOINT_ENV = "CODEX_COMPANION_APP_SERVER_ENDPOINT"; export const BROKER_BUSY_RPC_CODE = -32001; @@ -233,27 +234,21 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { capabilities: this.options.capabilities ?? DEFAULT_CAPABILITIES }); try { - this.procIdentity = + const captureOwnership = this.options.captureProcessOwnershipImpl ?? captureProcessOwnership; + this.ownershipSnapshot = process.platform === "win32" ? null - : getProcessIdentity(this.proc.pid, { + : captureOwnership(this.proc.pid, { cwd: this.cwd, env: this.options.env ?? process.env }); + this.procIdentity = this.ownershipSnapshot?.rootIdentity ?? null; } catch (error) { - try { - this.proc.kill("SIGKILL"); - } catch { - // The child may have exited before identity capture completed. - } + this.identityCaptureFailed = true; throw error; } if (process.platform !== "win32" && !this.procIdentity) { - try { - this.proc.kill("SIGKILL"); - } catch { - // The child may have exited before identity capture completed. - } + this.identityCaptureFailed = true; throw new Error("Unable to capture codex app-server process identity."); } this.notify("initialized", {}); @@ -261,7 +256,7 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { async close() { if (this.closed) { - await this.exitPromise; + await this.waitForExit(); return; } @@ -271,17 +266,22 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { this.readline.close(); } - if (this.proc && !this.proc.killed) { - this.proc.stdin.end(); + if (this.proc) { + try { + this.proc.stdin.end(); + } catch { + // The child may have closed its input before cleanup began. + } if (process.platform === "win32") { setTimeout(() => { if (this.proc && !this.proc.killed && this.proc.exitCode === null) { - void terminateProcessTree(this.proc.pid) + void terminateProcessTree(this.proc.pid, { + expectedRootIdentity: this.procIdentity, + ownershipSnapshot: this.ownershipSnapshot, + requireVerifiedOwnership: this.identityCaptureFailed + }) .then((outcome) => { - this.cleanupOutcome = { - verified: outcome.verified ?? null, - survivors: outcome.survivors ?? [] - }; + this.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); }) .catch(() => { // Best-effort cleanup inside an unref'd timer — swallow errors @@ -294,13 +294,12 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { // the group so MCP helpers cannot outlive the app-server parent. const outcome = await terminateProcessTree(this.proc.pid, { expectedRootIdentity: this.procIdentity, + ownershipSnapshot: this.ownershipSnapshot, + requireVerifiedOwnership: this.identityCaptureFailed, directKillImpl: (signal) => this.proc.kill(signal), warnImpl: () => {} }); - this.cleanupOutcome = { - verified: outcome.verified ?? false, - survivors: outcome.survivors ?? [] - }; + this.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); if (!outcome.verified) { process.stderr.write( `Warning: unable to verify codex app-server cleanup; surviving PIDs: ${outcome.survivors?.join(", ") || "none known"}.\n` @@ -309,7 +308,26 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { } } - await this.exitPromise; + const exited = await this.waitForExit(); + if (!exited) { + this.cleanupOutcome = normalizeProcessCleanupOutcome({ + ...(this.cleanupOutcome ?? {}), + attempted: true, + verified: false, + degraded: true + }); + } + } + + async waitForExit() { + const timeoutMs = Number.isFinite(this.options.closeWaitMs) ? this.options.closeWaitMs : DEFAULT_CLOSE_WAIT_MS; + let timeout; + const timedOut = new Promise((resolve) => { + timeout = setTimeout(() => resolve(false), timeoutMs); + }); + const exited = await Promise.race([this.exitPromise.then(() => true), timedOut]); + clearTimeout(timeout); + return exited; } sendMessage(message) { diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 2822dac57..98d63d1fb 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -117,7 +117,7 @@ export async function ensureBrokerSession(cwd, options = {}) { } if (existing) { - await teardownBrokerSession({ + const cleanup = await teardownBrokerSession({ endpoint: existing.endpoint ?? null, pidFile: existing.pidFile ?? null, logFile: existing.logFile ?? null, @@ -125,6 +125,11 @@ export async function ensureBrokerSession(cwd, options = {}) { pid: existing.pid ?? null, killProcess: options.killProcess ?? null }); + if (cleanup?.verified !== true) { + const error = new Error("Broker cleanup is unverified; refusing to start another broker session."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } clearBrokerSession(cwd); } @@ -148,7 +153,7 @@ export async function ensureBrokerSession(cwd, options = {}) { const ready = await waitForBrokerEndpoint(endpoint, options.timeoutMs ?? 2000); if (!ready) { - await teardownBrokerSession({ + const cleanup = await teardownBrokerSession({ endpoint, pidFile, logFile, @@ -156,6 +161,9 @@ export async function ensureBrokerSession(cwd, options = {}) { pid: child.pid ?? null, killProcess: options.killProcess ?? null }); + if (cleanup?.verified !== true) { + return null; + } return null; } @@ -170,12 +178,46 @@ export async function ensureBrokerSession(cwd, options = {}) { return session; } -export async function teardownBrokerSession({ endpoint = null, pidFile, logFile, sessionDir = null, pid = null, killProcess = null }) { +export async function teardownBrokerSession({ + endpoint = null, + pidFile, + logFile, + sessionDir = null, + pid = null, + pidIdentity = null, + ownershipSnapshot = null, + killProcess = null +}) { + let cleanupOutcome = { + attempted: false, + delivered: false, + verified: true, + degraded: false, + method: null, + targets: [], + targetIdentities: [], + survivors: [], + survivorIdentities: [] + }; if (Number.isFinite(pid) && killProcess) { try { - await killProcess(pid); - } catch { - // Ignore missing or already-exited broker processes. + const outcome = await killProcess(pid, { + expectedRootIdentity: pidIdentity, + ownershipSnapshot + }); + cleanupOutcome = outcome ?? { + ...cleanupOutcome, + attempted: true, + verified: false, + degraded: true + }; + } catch (error) { + if (error?.code !== "ESRCH" && error?.code !== "ENOENT") { + throw error; + } + } + if (cleanupOutcome.verified !== true) { + return cleanupOutcome; } } @@ -206,4 +248,5 @@ export async function teardownBrokerSession({ endpoint = null, pidFile, logFile, // Ignore non-empty or missing directories. } } + return cleanupOutcome; } diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 197b874a8..520030807 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -142,6 +142,58 @@ function collectProcessTree(rootPid, processes, rootDepth = 0) { return records; } +export function captureProcessOwnership(pid, options = {}) { + if (!Number.isFinite(pid) || (options.platform ?? process.platform) === "win32") { + return null; + } + + const processes = readUnixProcessTable(options.runCommandImpl ?? runCommand, options); + const root = processes.get(pid); + if (!root) { + return null; + } + + return { + rootPid: root.pid, + rootIdentity: root.identity, + processGroupId: root.processGroupId, + members: collectProcessTree(pid, processes).map((record) => ({ ...record })) + }; +} + +function recordsFromOwnershipSnapshot(snapshot) { + return (snapshot?.members ?? []).filter((record) => { + return Number.isFinite(record.pid) && typeof record.identity === "string" && record.identity.length > 0; + }); +} + +function identitiesByPid(identities) { + const result = new Map(); + for (const identity of identities ?? []) { + const match = String(identity).match(/^(\d+)@/); + if (match) { + result.set(Number(match[1]), String(identity)); + } + } + return result; +} + +export function normalizeProcessCleanupOutcome(outcome = {}) { + return { + attempted: Boolean(outcome.attempted), + delivered: Boolean(outcome.delivered), + verified: outcome.verified === true, + degraded: Boolean(outcome.degraded), + method: outcome.method ?? null, + escalated: Boolean(outcome.escalated), + targets: Array.isArray(outcome.targets) ? outcome.targets : [], + targetIdentities: Array.isArray(outcome.targetIdentities) ? outcome.targetIdentities : [], + survivors: Array.isArray(outcome.survivors) ? outcome.survivors : [], + survivorIdentities: Array.isArray(outcome.survivorIdentities) ? outcome.survivorIdentities : [], + ...(outcome.identityMismatch ? { identityMismatch: true } : {}) + }; +} + function sleep(milliseconds) { if (milliseconds <= 0) { return Promise.resolve(); @@ -157,7 +209,8 @@ export function getProcessIdentity(pid, options = {}) { } export function getLiveProcessPids(pids, options = {}) { - const candidates = [...new Set(pids.filter((pid) => Number.isFinite(pid)))]; + const candidates = [...new Set((pids ?? []).filter((pid) => Number.isFinite(pid)))]; + const expectedIdentities = identitiesByPid(options.identities); if (candidates.length === 0) { return []; } @@ -175,7 +228,10 @@ export function getLiveProcessPids(pids, options = {}) { try { const processes = readUnixProcessTable(options.runCommandImpl ?? runCommand, options); - return candidates.filter((pid) => isRunningProcess(processes.get(pid))); + return candidates.filter((pid) => { + const current = processes.get(pid); + return isRunningProcess(current) && (!expectedIdentities.has(pid) || expectedIdentities.get(pid) === current.identity); + }); } catch { // An unverified cleanup remains blocked when liveness cannot be checked. return candidates; @@ -345,7 +401,7 @@ function degradedDirectChildKill(pid, options, killImpl, reason) { `Unable to verify Unix process cleanup for PID ${pid}; used direct-child kill fallback (${String(reason).replace(/\s+/g, " ").trim()}). Surviving PIDs: none known.`, options ); - return { + return normalizeProcessCleanupOutcome({ attempted: true, delivered, verified: false, @@ -353,13 +409,26 @@ function degradedDirectChildKill(pid, options, killImpl, reason) { degraded: true, method: "direct-child", targets: [pid], - survivors: [] - }; + targetIdentities: options.expectedRootIdentity ? [options.expectedRootIdentity] : [], + survivors: [], + survivorIdentities: [] + }); } export async function terminateProcessTree(pid, options = {}) { if (!Number.isFinite(pid)) { - return { attempted: false, delivered: false, method: null }; + const ownershipSnapshot = options.ownershipSnapshot ?? null; + if (Number.isFinite(ownershipSnapshot?.rootPid)) { + return terminateProcessTree(ownershipSnapshot.rootPid, options); + } + const ownershipEstablished = Boolean(options.expectedRootIdentity || options.requireVerifiedOwnership); + return normalizeProcessCleanupOutcome({ + attempted: false, + delivered: false, + verified: !ownershipEstablished, + degraded: ownershipEstablished, + method: null + }); } const platform = options.platform ?? process.platform; @@ -373,21 +442,21 @@ export async function terminateProcessTree(pid, options = {}) { }); if (!result.error && result.status === 0) { - return { attempted: true, delivered: true, method: "taskkill", result }; + return { ...normalizeProcessCleanupOutcome({ attempted: true, delivered: true, verified: true, method: "taskkill" }), result }; } const combinedOutput = `${result.stderr}\n${result.stdout}`.trim(); if (!result.error && looksLikeMissingProcessMessage(combinedOutput)) { - return { attempted: true, delivered: false, method: "taskkill", result }; + return { ...normalizeProcessCleanupOutcome({ attempted: true, delivered: false, verified: true, method: "taskkill" }), result }; } if (result.error?.code === "ENOENT") { try { killImpl(pid); - return { attempted: true, delivered: true, method: "kill" }; + return normalizeProcessCleanupOutcome({ attempted: true, delivered: true, verified: true, method: "kill" }); } catch (error) { if (error?.code === "ESRCH") { - return { attempted: true, delivered: false, method: "kill" }; + return normalizeProcessCleanupOutcome({ attempted: true, delivered: false, verified: true, method: "kill" }); } throw error; } @@ -410,31 +479,38 @@ export async function terminateProcessTree(pid, options = {}) { return degradedDirectChildKill(pid, options, killImpl, error.message); } const root = initialProcesses.get(pid); - const expectedRootIdentity = options.expectedRootIdentity ?? root?.identity ?? null; + const ownershipSnapshot = options.ownershipSnapshot ?? null; + const ownershipEstablished = Boolean(ownershipSnapshot || options.expectedRootIdentity || options.requireVerifiedOwnership); + const expectedRootIdentity = options.expectedRootIdentity ?? ownershipSnapshot?.rootIdentity ?? root?.identity ?? null; if (!root) { - return { - attempted: true, - delivered: false, - verified: true, - escalated: false, - method: "process-tree", - targets: [] - }; + if (!ownershipSnapshot) { + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified: !ownershipEstablished, + degraded: ownershipEstablished, + method: "process-tree", + targets: [] + }); + } } - if (root.identity !== expectedRootIdentity) { - return { + if (root && root.identity !== expectedRootIdentity) { + return normalizeProcessCleanupOutcome({ attempted: true, delivered: false, - verified: true, - escalated: false, + verified: false, + degraded: ownershipEstablished, identityMismatch: true, method: "process-tree", targets: [] - }; + }); } const tracked = new Map(); - for (const record of collectProcessTree(pid, initialProcesses)) { + for (const record of recordsFromOwnershipSnapshot(ownershipSnapshot)) { + tracked.set(record.identity, record); + } + for (const record of root ? collectProcessTree(pid, initialProcesses) : []) { tracked.set(record.identity, record); } const unixOptions = { @@ -473,17 +549,24 @@ export async function terminateProcessTree(pid, options = {}) { live = phaseLive; } - return { + const verified = + live.length === 0 && + !options.requireVerifiedOwnership && + (root !== undefined || ownershipSnapshot !== null); + return normalizeProcessCleanupOutcome({ attempted: true, delivered, - verified: live.length === 0, + verified, + degraded: ownershipEstablished && !verified, escalated, method: "process-tree", // The algorithm covers same-process-group descendants plus those observed at scan time. // A post-scan setsid descendant can escape the tracked process tree. targets: [...tracked.values()].sort((left, right) => right.depth - left.depth).map((record) => record.pid), - survivors: live.map((record) => record.pid) - }; + targetIdentities: [...tracked.values()].sort((left, right) => right.depth - left.depth).map((record) => record.identity), + survivors: live.map((record) => record.pid), + survivorIdentities: live.map((record) => record.identity) + }); } catch (error) { if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { throw error; @@ -494,10 +577,10 @@ export async function terminateProcessTree(pid, options = {}) { export async function terminateProcessGroup(pgid, options = {}) { if (!Number.isFinite(pgid)) { - return { attempted: false, delivered: false, verified: true, survivors: [] }; + return normalizeProcessCleanupOutcome({ attempted: false, delivered: false, verified: false, degraded: true }); } if ((options.platform ?? process.platform) === "win32") { - return { attempted: false, delivered: false, verified: true, survivors: [] }; + return normalizeProcessCleanupOutcome({ attempted: false, delivered: false, verified: false, degraded: true }); } const runCommandImpl = options.runCommandImpl ?? runCommand; @@ -514,17 +597,45 @@ export async function terminateProcessGroup(pgid, options = {}) { `Unable to enumerate Unix processes while reclaiming process group ${pgid}; surviving PIDs unknown.`, options ); - return { attempted: false, delivered: false, verified: false, degraded: true, survivors: [] }; + return normalizeProcessCleanupOutcome({ + attempted: false, + delivered: false, + verified: false, + degraded: true, + survivors: recordsFromOwnershipSnapshot(options.ownershipSnapshot).map((record) => record.pid), + survivorIdentities: recordsFromOwnershipSnapshot(options.ownershipSnapshot).map((record) => record.identity) + }); } const tracked = new Map(); + const ownershipSnapshot = options.ownershipSnapshot ?? null; + const ownershipEstablished = Boolean(ownershipSnapshot); + for (const record of recordsFromOwnershipSnapshot(ownershipSnapshot)) { + tracked.set(record.identity, record); + } + let groupSelectionFound = false; for (const record of processes.values()) { + if (record.processGroupId !== pgid || !isRunningProcess(record)) { + continue; + } + const snapshotRecord = recordsFromOwnershipSnapshot(ownershipSnapshot).find((candidate) => candidate.pid === record.pid); + if (ownershipSnapshot && (!snapshotRecord || snapshotRecord.identity !== record.identity)) { + continue; + } + groupSelectionFound = true; if (record.processGroupId === pgid && isRunningProcess(record)) { tracked.set(record.identity, record); } } if (tracked.size === 0) { - return { attempted: false, delivered: false, verified: true, survivors: [] }; + return normalizeProcessCleanupOutcome({ + attempted: false, + delivered: false, + verified: !ownershipEstablished, + degraded: ownershipEstablished, + survivors: [], + survivorIdentities: [] + }); } const unixOptions = { @@ -545,15 +656,19 @@ export async function terminateProcessGroup(pgid, options = {}) { delivered = signalTracked(tracked, "SIGKILL", unixOptions) || delivered; live = await pollTracked(tracked, unixOptions, options.killPollAttempts ?? 11); } - return { + const root = processes.get(pgid); + return normalizeProcessCleanupOutcome({ attempted: true, delivered, - verified: live.length === 0, + verified: live.length === 0 && (!ownershipEstablished || (root?.identity === ownershipSnapshot.rootIdentity && groupSelectionFound)), + degraded: ownershipEstablished && (root?.identity !== ownershipSnapshot.rootIdentity || !groupSelectionFound), escalated, method: "process-group", targets: [...tracked.values()].map((record) => record.pid), - survivors: live.map((record) => record.pid) - }; + targetIdentities: [...tracked.values()].map((record) => record.identity), + survivors: live.map((record) => record.pid), + survivorIdentities: live.map((record) => record.identity) + }); } catch (error) { if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { throw error; @@ -562,7 +677,7 @@ export async function terminateProcessGroup(pgid, options = {}) { `Unable to verify Unix process-group cleanup for pgid ${pgid}; surviving PIDs unknown.`, options ); - return { attempted: true, delivered: true, verified: false, degraded: true, survivors: [] }; + return normalizeProcessCleanupOutcome({ attempted: true, delivered: true, verified: false, degraded: true, survivors: [], survivorIdentities: [] }); } } diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 0f54a6b09..773abac5a 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -2,6 +2,7 @@ import fs from "node:fs"; import process from "node:process"; +import { pathToFileURL } from "node:url"; import { terminateProcessTree } from "./lib/process.mjs"; import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; @@ -13,7 +14,7 @@ import { sendBrokerShutdown, teardownBrokerSession } from "./lib/broker-lifecycle.mjs"; -import { loadState, resolveStateFile, saveState } from "./lib/state.mjs"; +import { loadState, resolveStateFile, saveState, writeJobFile } from "./lib/state.mjs"; import { TRANSCRIPT_PATH_ENV } from "./lib/claude-session-transfer.mjs"; import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; @@ -39,39 +40,78 @@ function appendEnvVar(name, value) { fs.appendFileSync(process.env.CLAUDE_ENV_FILE, `export ${name}=${shellEscape(value)}\n`, "utf8"); } -async function cleanupSessionJobs(cwd, sessionId) { +export async function cleanupSessionJobs(cwd, sessionId, dependencies = {}) { if (!cwd || !sessionId) { - return; + return { verified: true, failures: [] }; } const workspaceRoot = resolveWorkspaceRoot(cwd); const stateFile = resolveStateFile(workspaceRoot); if (!fs.existsSync(stateFile)) { - return; + return { verified: true, failures: [] }; } const state = loadState(workspaceRoot); const removedJobs = state.jobs.filter((job) => job.sessionId === sessionId); if (removedJobs.length === 0) { - return; + return { verified: true, failures: [] }; } + const terminate = dependencies.terminateProcessTreeImpl ?? terminateProcessTree; + const retainedJobs = []; + const failures = []; for (const job of removedJobs) { const stillRunning = job.status === "queued" || job.status === "running"; if (!stillRunning) { continue; } try { - await terminateProcessTree(job.pid ?? Number.NaN); - } catch { - // Ignore teardown failures during session shutdown. + const outcome = await terminate(job.pid ?? Number.NaN, { + expectedRootIdentity: job.processIdentity ?? null, + ownershipSnapshot: job.ownershipSnapshot ?? null + }); + if (outcome?.verified === true) { + continue; + } + const retainedJob = { + ...job, + phase: "cleanup-pending", + cleanupOutcome: outcome, + cleanupFailure: "Session cleanup could not verify process termination." + }; + writeJobFile(workspaceRoot, job.id, retainedJob); + retainedJobs.push(retainedJob); + } catch (error) { + if (error?.code === "ESRCH") { + continue; + } + const retainedJob = { + ...job, + phase: "cleanup-pending", + cleanupOutcome: { + attempted: true, + delivered: false, + verified: false, + degraded: true, + survivors: [], + survivorIdentities: [] + }, + cleanupFailure: error instanceof Error ? error.message : String(error) + }; + writeJobFile(workspaceRoot, job.id, retainedJob); + retainedJobs.push(retainedJob); + failures.push(error); } } saveState(workspaceRoot, { ...state, - jobs: state.jobs.filter((job) => job.sessionId !== sessionId) + jobs: state.jobs.filter((job) => job.sessionId !== sessionId).concat(retainedJobs) }); + if (failures.length > 0) { + throw failures[0]; + } + return { verified: retainedJobs.length === 0, failures: [] }; } function handleSessionStart(input) { @@ -101,8 +141,8 @@ async function handleSessionEnd(input) { await sendBrokerShutdown(brokerEndpoint); } - await cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); - await teardownBrokerSession({ + const jobCleanup = await cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); + const brokerCleanup = await teardownBrokerSession({ endpoint: brokerEndpoint, pidFile, logFile, @@ -110,7 +150,15 @@ async function handleSessionEnd(input) { pid, killProcess: terminateProcessTree }); - clearBrokerSession(cwd); + if (brokerCleanup?.verified === true) { + clearBrokerSession(cwd); + } + if (jobCleanup?.verified !== true) { + throw new Error("Session cleanup remains pending because process termination could not be verified."); + } + if (brokerCleanup?.verified !== true) { + throw new Error("Broker cleanup remains pending because process termination could not be verified."); + } } async function main() { @@ -127,7 +175,9 @@ async function main() { } } -main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - process.exit(1); -}); +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index 9bb1da5d2..50ea85ce4 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -273,14 +273,20 @@ if (args[0] !== "app-server") { } const bootState = loadState(); bootState.appServerStarts = (bootState.appServerStarts || 0) + 1; -if (BEHAVIOR === "with-helper-child" || BEHAVIOR === "slow-task-with-helper-child") { - const helper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { +if (BEHAVIOR === "with-helper-child" || BEHAVIOR === "slow-task-with-helper-child" || BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "with-resistant-helper") { + const helperCode = BEHAVIOR === "with-resistant-helper" + ? "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)" + : "setInterval(() => {}, 1000)"; + const helper = spawn(process.execPath, ["-e", helperCode], { detached: process.platform !== "win32", stdio: "ignore" }); helper.unref(); bootState.helperPids = [...(bootState.helperPids || []), helper.pid]; } +if (BEHAVIOR === "crash-with-regrouped-helper") { + bootState.appServerPids = [...(bootState.appServerPids || []), process.pid]; +} saveState(bootState); const rl = readline.createInterface({ input: process.stdin }); @@ -298,6 +304,9 @@ rl.on("line", (line) => { state.capabilities = message.params.capabilities || null; saveState(state); send({ id: message.id, result: { userAgent: "fake-codex-app-server" } }); + if (BEHAVIOR === "crash-with-regrouped-helper") { + setTimeout(() => process.exit(1), 100); + } break; case "initialized": diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 6e9e3ae09..383c725bd 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -1,7 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; +import { getLiveProcessPids, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; test("terminateProcessTree uses taskkill on Windows", async () => { let captured = null; @@ -32,6 +32,26 @@ test("terminateProcessTree uses taskkill on Windows", async () => { assert.equal(outcome.method, "taskkill"); }); +test("getLiveProcessPids ignores a survivor PID reused by a different process", () => { + const live = getLiveProcessPids([900], { + platform: "darwin", + identities: ["900@old"], + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "900 1 900 S Mon Jul 27 00:00:02 2026\n", + stderr: "", + error: null + }; + } + }); + + assert.deepEqual(live, []); +}); + test("terminateProcessTree treats missing Windows processes as already stopped", async () => { const outcome = await terminateProcessTree(1234, { platform: "win32", @@ -214,8 +234,57 @@ test("terminateProcessTree refuses a reused root PID", async () => { }); assert.deepEqual(signals, []); - assert.equal(outcome.verified, true); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); assert.equal(outcome.identityMismatch, true); + const ownershipSnapshot = { + rootPid: 1234, + rootIdentity: "1234@Mon Jul 27 00:00:00 2026", + processGroupId: 1234, + members: [ + { + pid: 1234, + parentPid: 1, + processGroupId: 1234, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "1234@Mon Jul 27 00:00:00 2026", + depth: 0 + }, + { + pid: 1235, + parentPid: 1234, + processGroupId: 1235, + state: "S", + startedAt: "Mon Jul 27 00:00:01 2026", + identity: "1235@Mon Jul 27 00:00:01 2026", + depth: 1 + } + ] + }; + const cleanOutcome = await terminateProcessTree(1234, { + platform: "darwin", + ownershipSnapshot, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "", + stderr: "", + error: null + }; + }, + killImpl() { + throw new Error("no process should be signaled"); + } + }); + + assert.equal(cleanOutcome.verified, true); + assert.equal(cleanOutcome.degraded, false); + assert.deepEqual(cleanOutcome.survivors, []); + assert.deepEqual(cleanOutcome.survivorIdentities, []); }); test("terminateProcessTree revalidates descendant identities before signaling", async () => { @@ -423,3 +492,58 @@ test("terminateProcessGroup reclaims orphaned members of a dead leader's group", assert.equal(outcome.method, "process-group"); assert.deepEqual(outcome.targets, [202]); }); + +test("terminateProcessGroup hunts an observed regrouped helper after its root exits", async () => { + const signals = []; + const alive = new Set([200]); + const ownershipSnapshot = { + rootPid: 100, + rootIdentity: "100@Mon Jul 27 00:00:00 2026", + processGroupId: 100, + members: [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "100@Mon Jul 27 00:00:00 2026", + depth: 0 + }, + { + pid: 200, + parentPid: 100, + processGroupId: 200, + state: "S", + startedAt: "Mon Jul 27 00:00:01 2026", + identity: "200@Mon Jul 27 00:00:01 2026", + depth: 1 + } + ] + }; + const outcome = await terminateProcessGroup(100, { + platform: "darwin", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: alive.has(200) ? "200 1 200 S Mon Jul 27 00:00:01 2026\n" : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive.delete(Math.abs(pid)); + } + }); + + assert.deepEqual(signals, [[-200, "SIGTERM"]]); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.deepEqual(outcome.survivors, []); +}); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 6feb59bd2..f7367ce3e 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -7,8 +7,12 @@ import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; import { initGitRepo, makeTempDir as createTempDir, run } from "./helpers.mjs"; +import { handleCancel } from "../plugins/codex/scripts/codex-companion.mjs"; +import { cleanupSessionJobs } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; -import { ensureBrokerSession, loadBrokerSession, saveBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { isBrokerRequestAllowedDuringShutdown } from "../plugins/codex/scripts/app-server-broker.mjs"; +import { ensureBrokerSession, loadBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { captureProcessOwnership, terminateProcessGroup } from "../plugins/codex/scripts/lib/process.mjs"; import { resolveStateDir } from "../plugins/codex/scripts/lib/state.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -46,6 +50,107 @@ test.after(() => { assert.deepEqual(cleanupFailures, []); }); +test("broker rejects queued work after shutdown begins", () => { + assert.equal(isBrokerRequestAllowedDuringShutdown(true, { id: 2, method: "thread/list" }), false); + assert.equal(isBrokerRequestAllowedDuringShutdown(true, { id: 3, method: "broker/shutdown" }), true); + assert.equal(isBrokerRequestAllowedDuringShutdown(false, { id: 4, method: "thread/list" }), true); +}); + +test("fake app-server crash reclaims an observed regrouped helper without replacement", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "crash-with-regrouped-helper"); + const env = buildEnv(binDir); + const child = spawn("codex", ["app-server"], { + cwd: repo, + env, + detached: true, + stdio: ["pipe", "pipe", "ignore"] + }); + child.stdout.setEncoding("utf8"); + let buffer = ""; + const initialized = new Promise((resolve, reject) => { + child.stdout.on("data", (chunk) => { + buffer += chunk; + const newline = buffer.indexOf("\n"); + if (newline === -1) { + return; + } + resolve(JSON.parse(buffer.slice(0, newline))); + }); + child.once("error", reject); + }); + child.stdin.write(`${JSON.stringify({ id: 1, method: "initialize", params: { capabilities: {} } })}\n`); + const response = await initialized; + assert.equal(response.id, 1); + await waitFor(() => fs.existsSync(fakeStatePath) && JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).helperPids?.length === 1); + const helperPid = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).helperPids[0]; + t.after(() => { + try { + process.kill(helperPid, "SIGKILL"); + } catch { + // Ignore the helper after cleanup. + } + }); + let ownershipSnapshot; + try { + ownershipSnapshot = captureProcessOwnership(child.pid, { env }); + } catch (error) { + if (error?.code === "PROCESS_TABLE_UNAVAILABLE") { + t.skip(`process table unavailable: ${error.message}`); + return; + } + throw error; + } + await new Promise((resolve) => child.once("exit", resolve)); + const outcome = await terminateProcessGroup(child.pid, { ownershipSnapshot, env }); + + assert.equal(outcome.verified, false); + assert.deepEqual(outcome.survivors, []); + assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 1); +}); + +test("broker shutdown completes without starting a second app-server", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "with-resistant-helper"); + const env = buildEnv(binDir); + const brokerSocketPath = path.join("/private/tmp", `cxc-p1c-${process.pid}-${Date.now()}.sock`); + const brokerSession = await ensureBrokerSession(repo, { + env, + createBrokerEndpoint: () => `unix:${brokerSocketPath}` + }); + if (!brokerSession) { + t.skip("broker socket unavailable in this sandbox"); + return; + } + assert.ok(brokerSession.pid); + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + }); + + const client = await CodexAppServerClient.connect(repo, { env }); + await client.request("thread/list", { cwd: repo }); + const appServerStartsBeforeShutdown = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts; + assert.equal(appServerStartsBeforeShutdown, 1); + await sendBrokerShutdown(brokerSession.endpoint); + await waitFor(() => { + try { + process.kill(brokerSession.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, appServerStartsBeforeShutdown); +}); + async function waitFor(predicate, { timeoutMs = 5000, intervalMs = 50 } = {}) { const start = Date.now(); while (Date.now() - start < timeoutMs) { @@ -1748,6 +1853,81 @@ test("cancel stops an active background job and marks it cancelled", async (t) = assert.match(fs.readFileSync(logFile, "utf8"), /Cancelled by user/); }); +test("unverified cleanup preserves cancel, session, and broker ownership records", async () => { + const workspace = makeTempDir(); + const stateDir = resolveStateDir(workspace); + const jobsDir = path.join(stateDir, "jobs"); + fs.mkdirSync(jobsDir, { recursive: true }); + const logFile = path.join(jobsDir, "task-live.log"); + const jobFile = path.join(jobsDir, "task-live.json"); + const job = { + id: "task-live", + status: "running", + phase: "running", + title: "Codex Task", + sessionId: "sess-current", + pid: 123, + processIdentity: "123@old", + logFile + }; + fs.writeFileSync(logFile, "starting\n", "utf8"); + fs.writeFileSync(jobFile, `${JSON.stringify(job, null, 2)}\n`, "utf8"); + fs.writeFileSync( + path.join(stateDir, "state.json"), + `${JSON.stringify({ version: 1, config: { stopReviewGate: false }, jobs: [job] }, null, 2)}\n`, + "utf8" + ); + + const cleanupOutcome = { + attempted: true, + delivered: true, + verified: false, + degraded: true, + survivors: [123], + survivorIdentities: ["123@old"] + }; + await assert.rejects( + handleCancel(["task-live", "--cwd", workspace, "--json"], { + interruptAppServerTurnImpl: async () => ({ attempted: false, interrupted: false }), + terminateProcessTreeImpl: async () => cleanupOutcome + }), + /ownership records were preserved for retry/ + ); + let state = JSON.parse(fs.readFileSync(path.join(stateDir, "state.json"), "utf8")); + assert.equal(state.jobs[0].status, "running"); + assert.equal(state.jobs[0].pid, 123); + assert.deepEqual(state.jobs[0].cleanupOutcome.survivorIdentities, ["123@old"]); + + const sessionCleanup = await cleanupSessionJobs(workspace, "sess-current", { + terminateProcessTreeImpl: async () => cleanupOutcome + }); + assert.equal(sessionCleanup.verified, false); + state = JSON.parse(fs.readFileSync(path.join(stateDir, "state.json"), "utf8")); + assert.equal(state.jobs[0].pid, 123); + assert.equal(fs.existsSync(jobFile), true); + + const sessionDir = path.join(workspace, "broker-session"); + fs.mkdirSync(sessionDir, { recursive: true }); + const pidFile = path.join(sessionDir, "broker.pid"); + const brokerLog = path.join(sessionDir, "broker.log"); + const endpointPath = path.join(sessionDir, "broker.sock"); + fs.writeFileSync(pidFile, "456\n", "utf8"); + fs.writeFileSync(brokerLog, "broker\n", "utf8"); + fs.writeFileSync(endpointPath, "socket-marker\n", "utf8"); + const brokerCleanup = await teardownBrokerSession({ + endpoint: `unix:${endpointPath}`, + pidFile, + logFile: brokerLog, + sessionDir, + pid: 456, + killProcess: async () => cleanupOutcome + }); + assert.equal(brokerCleanup.verified, false); + assert.equal(fs.existsSync(pidFile), true); + assert.equal(fs.existsSync(brokerLog), true); + assert.equal(fs.existsSync(endpointPath), true); +}); + test("cancel without a job id ignores active jobs from other Claude sessions", () => { const workspace = makeTempDir(); const stateDir = resolveStateDir(workspace); @@ -2476,6 +2656,43 @@ test("shared broker releases its idle app-server child and restarts it on demand assert.equal(fakeState.helperPids.length, 2); }); +test("identity capture failure cleans the owned app-server tree and reports unverified cleanup", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + + installFakeCodex(binDir, "with-helper-child"); + const env = buildEnv(binDir); + await assert.rejects( + CodexAppServerClient.connect(repo, { + disableBroker: true, + env, + captureProcessOwnershipImpl() { + throw new Error("identity lookup injected failure"); + } + }), + (error) => error.cleanupOutcome?.verified === false && error.cleanupOutcome?.degraded === true + ); + + const helperPid = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).helperPids[0]; + function isRunning(pid) { + const result = run("/bin/ps", ["-o", "stat=", "-p", String(pid)]); + return result.status === 0 && !result.stdout.trim().startsWith("Z"); + } + await waitFor(() => { + return !isRunning(helperPid); + }); + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + assert.equal(state.appServerStarts, 1); + t.after(() => { + try { + process.kill(helperPid, "SIGKILL"); + } catch { + // Ignore the helper after cleanup. + } + }); +}); + test("shared broker keeps active work alive after its client disconnects", async (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); From ce3a15e9aef815f0e4c611546ef3818fc4ef3235 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Tue, 28 Jul 2026 00:25:58 -0700 Subject: [PATCH 12/29] Never signal a process whose ownership cannot be established Cleanup could adopt the identity of whatever process currently held a recorded pid, which makes identity verification a no-op and can signal an unrelated process tree after pid reuse. Background task records persisted only a pid, so that was the live path for cancel and session-end cleanup. - Remove the current-identity fallback; a record with no ownership evidence refuses to signal and reports unverified rather than killing. - Capture ownership when spawning background workers and brokers so the normal path stays fully verified. - Allow best-effort cleanup after an identity-capture failure only when the caller still holds a live child handle, which is what makes pid reuse impossible; persisted records never qualify and are left alone with a message saying so. Leaking a process is preferred over killing a stranger. - Include a live member of an owned process group even when it postdates the ownership snapshot, so a helper spawned after initialize is reclaimed instead of wedging the broker against ever spawning a replacement. Attributed-To: Codex executor lane (implementation), Claude interactive (review, verification) Co-Authored-By: Claude Opus 5 --- plugins/codex/scripts/codex-companion.mjs | 32 ++- plugins/codex/scripts/lib/app-server.mjs | 4 +- .../codex/scripts/lib/broker-lifecycle.mjs | 31 ++- plugins/codex/scripts/lib/process.mjs | 50 ++++- .../codex/scripts/session-lifecycle-hook.mjs | 20 +- tests/fake-codex-fixture.mjs | 37 ++- tests/process.test.mjs | 211 ++++++++++++++++++ tests/runtime.test.mjs | 101 ++++++++- 8 files changed, 456 insertions(+), 30 deletions(-) diff --git a/plugins/codex/scripts/codex-companion.mjs b/plugins/codex/scripts/codex-companion.mjs index 12992ea4a..5e6b4770a 100644 --- a/plugins/codex/scripts/codex-companion.mjs +++ b/plugins/codex/scripts/codex-companion.mjs @@ -24,7 +24,7 @@ import { import { resolveClaudeSessionPath } from "./lib/claude-session-transfer.mjs"; import { readStdinIfPiped } from "./lib/fs.mjs"; import { collectReviewContext, ensureGitRepository, resolveReviewTarget } from "./lib/git.mjs"; -import { binaryAvailable, terminateProcessTree } from "./lib/process.mjs"; +import { binaryAvailable, captureProcessOwnership, terminateProcessTree } from "./lib/process.mjs"; import { loadPromptTemplate, interpolateTemplate } from "./lib/prompts.mjs"; import { generateJobId, @@ -686,11 +686,27 @@ function enqueueBackgroundTask(cwd, job, request) { appendLogLine(logFile, "Queued for background execution."); const child = spawnDetachedTaskWorker(cwd, job.id); + let ownershipSnapshot = null; + let ownershipCaptureFailed = false; + if (process.platform !== "win32") { + try { + ownershipSnapshot = captureProcessOwnership(child.pid ?? Number.NaN, { + cwd, + env: process.env + }); + ownershipCaptureFailed = !ownershipSnapshot?.rootIdentity; + } catch { + ownershipCaptureFailed = true; + } + } const queuedRecord = { ...job, status: "queued", phase: "queued", pid: child.pid ?? null, + processIdentity: ownershipSnapshot?.rootIdentity ?? null, + ownershipSnapshot, + ownershipCaptureFailed, logFile, request }; @@ -983,12 +999,20 @@ export async function handleCancel(argv, dependencies = {}) { ); } + const expectedRootIdentity = existing.processIdentity ?? job.processIdentity ?? null; + const ownershipSnapshot = existing.ownershipSnapshot ?? job.ownershipSnapshot ?? null; + const ownershipCaptureFailed = + existing.ownershipCaptureFailed === true || job.ownershipCaptureFailed === true; const cleanupOutcome = await (dependencies.terminateProcessTreeImpl ?? terminateProcessTree)(job.pid ?? Number.NaN, { - expectedRootIdentity: existing.processIdentity ?? job.processIdentity ?? null, - ownershipSnapshot: existing.ownershipSnapshot ?? job.ownershipSnapshot ?? null + expectedRootIdentity, + ownershipSnapshot, + requireVerifiedOwnership: ownershipCaptureFailed }); if (cleanupOutcome?.verified !== true) { - const failureMessage = `Unable to verify cleanup for ${job.id}; ownership records were preserved for retry.`; + const failureMessage = + ownershipCaptureFailed && !expectedRootIdentity && !ownershipSnapshot?.rootIdentity + ? `Job ${job.id} could not be verified as owned and was left alone.` + : `Unable to verify cleanup for ${job.id}; ownership records were preserved for retry.`; appendLogLine(job.logFile, failureMessage); const recoveryRecord = { ...existing, diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index a33963589..a8f3310a0 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -278,7 +278,8 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { void terminateProcessTree(this.proc.pid, { expectedRootIdentity: this.procIdentity, ownershipSnapshot: this.ownershipSnapshot, - requireVerifiedOwnership: this.identityCaptureFailed + requireVerifiedOwnership: this.identityCaptureFailed, + ownerHoldsLiveHandle: true }) .then((outcome) => { this.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); @@ -296,6 +297,7 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { expectedRootIdentity: this.procIdentity, ownershipSnapshot: this.ownershipSnapshot, requireVerifiedOwnership: this.identityCaptureFailed, + ownerHoldsLiveHandle: true, directKillImpl: (signal) => this.proc.kill(signal), warnImpl: () => {} }); diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 98d63d1fb..9da687417 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -6,6 +6,7 @@ import process from "node:process"; import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { createBrokerEndpoint, parseBrokerEndpoint } from "./broker-endpoint.mjs"; +import { captureProcessOwnership } from "./process.mjs"; import { resolveStateDir } from "./state.mjs"; export const PID_FILE_ENV = "CODEX_COMPANION_APP_SERVER_PID_FILE"; @@ -123,6 +124,9 @@ export async function ensureBrokerSession(cwd, options = {}) { logFile: existing.logFile ?? null, sessionDir: existing.sessionDir ?? null, pid: existing.pid ?? null, + pidIdentity: existing.pidIdentity ?? null, + ownershipSnapshot: existing.ownershipSnapshot ?? null, + requireVerifiedOwnership: existing.ownershipCaptureFailed === true, killProcess: options.killProcess ?? null }); if (cleanup?.verified !== true) { @@ -150,6 +154,21 @@ export async function ensureBrokerSession(cwd, options = {}) { logFile, env: options.env ?? process.env }); + const captureOwnership = options.captureProcessOwnershipImpl ?? captureProcessOwnership; + let ownershipSnapshot = null; + let ownershipCaptureFailed = false; + if ((options.platform ?? process.platform) !== "win32") { + try { + ownershipSnapshot = captureOwnership(child.pid ?? Number.NaN, { + cwd, + env: options.env ?? process.env, + platform: options.platform + }); + ownershipCaptureFailed = !ownershipSnapshot?.rootIdentity; + } catch { + ownershipCaptureFailed = true; + } + } const ready = await waitForBrokerEndpoint(endpoint, options.timeoutMs ?? 2000); if (!ready) { @@ -159,6 +178,9 @@ export async function ensureBrokerSession(cwd, options = {}) { logFile, sessionDir, pid: child.pid ?? null, + pidIdentity: ownershipSnapshot?.rootIdentity ?? null, + ownershipSnapshot, + requireVerifiedOwnership: ownershipCaptureFailed, killProcess: options.killProcess ?? null }); if (cleanup?.verified !== true) { @@ -172,7 +194,10 @@ export async function ensureBrokerSession(cwd, options = {}) { pidFile, logFile, sessionDir, - pid: child.pid ?? null + pid: child.pid ?? null, + pidIdentity: ownershipSnapshot?.rootIdentity ?? null, + ownershipSnapshot, + ownershipCaptureFailed }; saveBrokerSession(cwd, session); return session; @@ -186,6 +211,7 @@ export async function teardownBrokerSession({ pid = null, pidIdentity = null, ownershipSnapshot = null, + requireVerifiedOwnership = false, killProcess = null }) { let cleanupOutcome = { @@ -203,7 +229,8 @@ export async function teardownBrokerSession({ try { const outcome = await killProcess(pid, { expectedRootIdentity: pidIdentity, - ownershipSnapshot + ownershipSnapshot, + requireVerifiedOwnership }); cleanupOutcome = outcome ?? { ...cleanupOutcome, diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 520030807..e6526d5f7 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -434,6 +434,12 @@ export async function terminateProcessTree(pid, options = {}) { const platform = options.platform ?? process.platform; const runCommandImpl = options.runCommandImpl ?? runCommand; const killImpl = options.killImpl ?? process.kill.bind(process); + const ownershipSnapshot = options.ownershipSnapshot ?? null; + const expectedRootIdentity = options.expectedRootIdentity ?? ownershipSnapshot?.rootIdentity ?? null; + const ownershipCaptureFailed = options.requireVerifiedOwnership === true; + const captureFailureCleanupAllowed = + ownershipCaptureFailed && options.ownerHoldsLiveHandle === true; + const ownershipEstablished = Boolean(ownershipSnapshot || expectedRootIdentity || ownershipCaptureFailed); if (platform === "win32") { const result = runCommandImpl("taskkill", ["/PID", String(pid), "/T", "/F"], { @@ -476,12 +482,33 @@ export async function terminateProcessTree(pid, options = {}) { if (error?.code !== "PROCESS_TABLE_UNAVAILABLE") { throw error; } + if (!expectedRootIdentity && !captureFailureCleanupAllowed) { + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified: false, + degraded: true, + method: "process-tree", + targets: [], + survivors: [pid], + survivorIdentities: [] + }); + } return degradedDirectChildKill(pid, options, killImpl, error.message); } const root = initialProcesses.get(pid); - const ownershipSnapshot = options.ownershipSnapshot ?? null; - const ownershipEstablished = Boolean(ownershipSnapshot || options.expectedRootIdentity || options.requireVerifiedOwnership); - const expectedRootIdentity = options.expectedRootIdentity ?? ownershipSnapshot?.rootIdentity ?? root?.identity ?? null; + if (!expectedRootIdentity && !captureFailureCleanupAllowed) { + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified: false, + degraded: true, + method: "process-tree", + targets: [], + survivors: [pid], + survivorIdentities: [] + }); + } if (!root) { if (!ownershipSnapshot) { return normalizeProcessCleanupOutcome({ @@ -494,7 +521,7 @@ export async function terminateProcessTree(pid, options = {}) { }); } } - if (root && root.identity !== expectedRootIdentity) { + if (root && expectedRootIdentity && root.identity !== expectedRootIdentity) { return normalizeProcessCleanupOutcome({ attempted: true, delivered: false, @@ -513,9 +540,15 @@ export async function terminateProcessTree(pid, options = {}) { for (const record of root ? collectProcessTree(pid, initialProcesses) : []) { tracked.set(record.identity, record); } + let cleanupRootIdentity = expectedRootIdentity; + if (!cleanupRootIdentity && captureFailureCleanupAllowed && root) { + // A live, unreaped child handle prevents PID reuse. Its owner may use the + // current identity for best-effort cleanup, but the result stays unverified. + cleanupRootIdentity = root.identity; + } const unixOptions = { rootPid: pid, - rootIdentity: expectedRootIdentity, + rootIdentity: cleanupRootIdentity, runCommandImpl, killImpl, cwd: options.cwd, @@ -619,7 +652,7 @@ export async function terminateProcessGroup(pgid, options = {}) { continue; } const snapshotRecord = recordsFromOwnershipSnapshot(ownershipSnapshot).find((candidate) => candidate.pid === record.pid); - if (ownershipSnapshot && (!snapshotRecord || snapshotRecord.identity !== record.identity)) { + if (ownershipSnapshot && snapshotRecord && snapshotRecord.identity !== record.identity) { continue; } groupSelectionFound = true; @@ -657,11 +690,12 @@ export async function terminateProcessGroup(pgid, options = {}) { live = await pollTracked(tracked, unixOptions, options.killPollAttempts ?? 11); } const root = processes.get(pgid); + const rootIdentityMatches = !root || root.identity === ownershipSnapshot?.rootIdentity; return normalizeProcessCleanupOutcome({ attempted: true, delivered, - verified: live.length === 0 && (!ownershipEstablished || (root?.identity === ownershipSnapshot.rootIdentity && groupSelectionFound)), - degraded: ownershipEstablished && (root?.identity !== ownershipSnapshot.rootIdentity || !groupSelectionFound), + verified: live.length === 0 && (!ownershipEstablished || (rootIdentityMatches && groupSelectionFound)), + degraded: ownershipEstablished && (!rootIdentityMatches || !groupSelectionFound), escalated, method: "process-group", targets: [...tracked.values()].map((record) => record.pid), diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 773abac5a..ba3d52efd 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -66,18 +66,26 @@ export async function cleanupSessionJobs(cwd, sessionId, dependencies = {}) { continue; } try { + const expectedRootIdentity = job.processIdentity ?? null; + const ownershipSnapshot = job.ownershipSnapshot ?? null; + const ownershipCaptureFailed = job.ownershipCaptureFailed === true; const outcome = await terminate(job.pid ?? Number.NaN, { - expectedRootIdentity: job.processIdentity ?? null, - ownershipSnapshot: job.ownershipSnapshot ?? null + expectedRootIdentity, + ownershipSnapshot, + requireVerifiedOwnership: ownershipCaptureFailed }); if (outcome?.verified === true) { continue; } + const cleanupFailure = + ownershipCaptureFailed && !expectedRootIdentity && !ownershipSnapshot?.rootIdentity + ? `Job ${job.id} could not be verified as owned and was left alone.` + : "Session cleanup could not verify process termination."; const retainedJob = { ...job, phase: "cleanup-pending", cleanupOutcome: outcome, - cleanupFailure: "Session cleanup could not verify process termination." + cleanupFailure }; writeJobFile(workspaceRoot, job.id, retainedJob); retainedJobs.push(retainedJob); @@ -136,6 +144,9 @@ async function handleSessionEnd(input) { const logFile = brokerSession?.logFile ?? null; const sessionDir = brokerSession?.sessionDir ?? null; const pid = brokerSession?.pid ?? null; + const pidIdentity = brokerSession?.pidIdentity ?? null; + const ownershipSnapshot = brokerSession?.ownershipSnapshot ?? null; + const requireVerifiedOwnership = brokerSession?.ownershipCaptureFailed === true; if (brokerEndpoint) { await sendBrokerShutdown(brokerEndpoint); @@ -148,6 +159,9 @@ async function handleSessionEnd(input) { logFile, sessionDir, pid, + pidIdentity, + ownershipSnapshot, + requireVerifiedOwnership, killProcess: terminateProcessTree }); if (brokerCleanup?.verified === true) { diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index 50ea85ce4..b16d8e6fa 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -273,18 +273,20 @@ if (args[0] !== "app-server") { } const bootState = loadState(); bootState.appServerStarts = (bootState.appServerStarts || 0) + 1; -if (BEHAVIOR === "with-helper-child" || BEHAVIOR === "slow-task-with-helper-child" || BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "with-resistant-helper") { - const helperCode = BEHAVIOR === "with-resistant-helper" - ? "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)" - : "setInterval(() => {}, 1000)"; - const helper = spawn(process.execPath, ["-e", helperCode], { - detached: process.platform !== "win32", - stdio: "ignore" - }); - helper.unref(); - bootState.helperPids = [...(bootState.helperPids || []), helper.pid]; +if (BEHAVIOR === "with-helper-child" || BEHAVIOR === "slow-task-with-helper-child" || BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "with-resistant-helper" || BEHAVIOR === "crash-with-post-snapshot-helper") { + if (BEHAVIOR !== "crash-with-post-snapshot-helper") { + const helperCode = BEHAVIOR === "with-resistant-helper" + ? "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)" + : "setInterval(() => {}, 1000)"; + const helper = spawn(process.execPath, ["-e", helperCode], { + detached: process.platform !== "win32", + stdio: "ignore" + }); + helper.unref(); + bootState.helperPids = [...(bootState.helperPids || []), helper.pid]; + } } -if (BEHAVIOR === "crash-with-regrouped-helper") { +if (BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "crash-with-post-snapshot-helper") { bootState.appServerPids = [...(bootState.appServerPids || []), process.pid]; } saveState(bootState); @@ -307,6 +309,19 @@ rl.on("line", (line) => { if (BEHAVIOR === "crash-with-regrouped-helper") { setTimeout(() => process.exit(1), 100); } + if (BEHAVIOR === "crash-with-post-snapshot-helper") { + setTimeout(() => { + const helper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + detached: false, + stdio: "ignore" + }); + helper.unref(); + const current = loadState(); + current.helperPids = [...(current.helperPids || []), helper.pid]; + saveState(current); + setTimeout(() => process.exit(1), 100); + }, 250); + } break; case "initialized": diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 383c725bd..bccc108ae 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -80,6 +80,7 @@ test("terminateProcessTree terminates Unix descendant groups deepest-first", asy const parents = new Map([[1234, 1], [1235, 1234], [1236, 1235], [1237, 1234]]); const outcome = await terminateProcessTree(1234, { platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", runCommandImpl(command, args) { assert.equal(command, "/bin/ps"); assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]); @@ -118,6 +119,7 @@ test("terminateProcessTree signals a Unix PID directly when it is not a group le let alive = true; const outcome = await terminateProcessTree(1234, { platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", runCommandImpl(command, args) { return { command, @@ -150,6 +152,7 @@ test("terminateProcessTree parses a captured Linux procps process table", async ].join("\n"); const outcome = await terminateProcessTree(42001, { platform: "linux", + expectedRootIdentity: "42001@Mon Jul 27 12:34:56 2026", runCommandImpl(command, args) { assert.equal(command, "/bin/ps"); assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]); @@ -183,6 +186,7 @@ test("terminateProcessTree falls back to a direct child kill when Unix process e const warnings = []; const outcome = await terminateProcessTree(1234, { platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", warnImpl(message) { warnings.push(message); }, @@ -287,6 +291,113 @@ test("terminateProcessTree refuses a reused root PID", async () => { assert.deepEqual(cleanOutcome.survivorIdentities, []); }); +test("terminateProcessTree refuses a PID without persisted ownership", async () => { + const signals = []; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: "1234 1 1234 S Mon Jul 27 00:00:00 2026\n", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.attempted, true); + assert.equal(outcome.delivered, false); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.deepEqual(outcome.survivors, [1234]); + assert.deepEqual(outcome.survivorIdentities, []); +}); + +test("terminateProcessTree refuses capture-failure cleanup without a live owner handle", async () => { + const identityObservedAtSpawn = "1234@Sun Jul 26 00:00:00 2026"; + const identityNowHoldingPid = "1234@Mon Jul 27 00:00:00 2026"; + const persistedRecord = { + ownershipCaptureFailed: true, + processIdentity: null, + ownershipSnapshot: null + }; + const signals = []; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: persistedRecord.processIdentity, + ownershipSnapshot: persistedRecord.ownershipSnapshot, + requireVerifiedOwnership: persistedRecord.ownershipCaptureFailed, + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: `1234 1 1234 S ${identityNowHoldingPid.slice(identityNowHoldingPid.indexOf("@") + 1)}\n`, + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.notEqual(identityNowHoldingPid, identityObservedAtSpawn); + assert.deepEqual(signals, []); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); +}); + +test("terminateProcessTree permits capture-failure cleanup with a live owner handle", async () => { + const identityObservedAtSpawn = "1234@Sun Jul 26 00:00:00 2026"; + const identityNowHoldingPid = "1234@Mon Jul 27 00:00:00 2026"; + const persistedRecord = { + ownershipCaptureFailed: true, + processIdentity: null, + ownershipSnapshot: null + }; + const signals = []; + let alive = true; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: persistedRecord.processIdentity, + ownershipSnapshot: persistedRecord.ownershipSnapshot, + requireVerifiedOwnership: persistedRecord.ownershipCaptureFailed, + ownerHoldsLiveHandle: true, + pollIntervalMs: 0, + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: alive + ? `1234 1 1234 S ${identityNowHoldingPid.slice(identityNowHoldingPid.indexOf("@") + 1)}\n` + : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive = false; + } + }); + + assert.notEqual(identityNowHoldingPid, identityObservedAtSpawn); + assert.deepEqual(signals, [[-1234, "SIGTERM"]]); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); +}); + test("terminateProcessTree revalidates descendant identities before signaling", async () => { const signals = []; let rootAlive = true; @@ -378,6 +489,7 @@ test("terminateProcessTree keeps the root alive until its descendants are reaped const alive = new Set([100, 200]); const outcome = await terminateProcessTree(100, { platform: "darwin", + expectedRootIdentity: "100@Mon Jul 27 00:00:00 2026", pollIntervalMs: 0, runCommandImpl(command, args) { const rows = []; @@ -423,6 +535,7 @@ test("terminateProcessTree tracks reparented members of a signaled process group const alive = new Set([100, 200]); const outcome = await terminateProcessTree(100, { platform: "darwin", + expectedRootIdentity: "100@Mon Jul 27 00:00:00 2026", termPollAttempts: 1, killPollAttempts: 1, sleepImpl() {}, @@ -547,3 +660,101 @@ test("terminateProcessGroup hunts an observed regrouped helper after its root ex assert.equal(outcome.degraded, true); assert.deepEqual(outcome.survivors, []); }); + +test("terminateProcessGroup excludes a reused snapshot PID", async () => { + const signals = []; + const ownershipSnapshot = { + rootPid: 100, + rootIdentity: "100@Mon Jul 27 00:00:00 2026", + processGroupId: 100, + members: [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "100@Mon Jul 27 00:00:00 2026", + depth: 0 + }, + { + pid: 200, + parentPid: 100, + processGroupId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:01 2026", + identity: "200@Mon Jul 27 00:00:01 2026", + depth: 1 + } + ] + }; + const outcome = await terminateProcessGroup(100, { + platform: "darwin", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: "200 1 100 S Mon Jul 27 00:01:01 2026\n", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); +}); + +test("terminateProcessGroup reclaims a post-snapshot member of the owned group", async () => { + const signals = []; + let alive = true; + const ownershipSnapshot = { + rootPid: 100, + rootIdentity: "100@Mon Jul 27 00:00:00 2026", + processGroupId: 100, + members: [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "100@Mon Jul 27 00:00:00 2026", + depth: 0 + } + ] + }; + const outcome = await terminateProcessGroup(100, { + platform: "darwin", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: alive ? "300 1 100 S Mon Jul 27 00:00:02 2026\n" : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + alive = false; + } + }); + + assert.deepEqual(signals, [[300, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.equal(outcome.degraded, false); + assert.deepEqual(outcome.survivors, []); +}); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index f7367ce3e..347cfe125 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -111,6 +111,83 @@ test("fake app-server crash reclaims an observed regrouped helper without replac assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 1); }); +test("shared broker reclaims a post-snapshot helper before allowing replacement", async (t) => { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "crash-with-post-snapshot-helper"); + const env = { + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" + }; + const brokerSession = await ensureBrokerSession(repo, { env }); + if (!brokerSession) { + t.skip("broker socket unavailable in this sandbox"); + return; + } + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + if (fs.existsSync(fakeStatePath)) { + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + for (const helperPid of state.helperPids || []) { + try { + process.kill(helperPid, "SIGKILL"); + } catch { + // Ignore helpers already terminated by group cleanup. + } + } + } + }); + + function isLive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } + } + + const client = await CodexAppServerClient.connect(repo, { env }); + await client.request("thread/list", { cwd: repo }); + await waitFor(() => { + if (!fs.existsSync(fakeStatePath)) { + return false; + } + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + return state.helperPids?.length === 1 && state.appServerPids?.length === 1; + }); + + const firstState = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + const appServerPid = firstState.appServerPids[0]; + const helperPid = firstState.helperPids[0]; + await waitFor(() => !isLive(appServerPid)); + await waitFor(() => !isLive(helperPid)); + + const replacementResponse = await waitFor(async () => { + try { + const replacementClient = await CodexAppServerClient.connect(repo, { env }); + try { + return await replacementClient.request("thread/list", { cwd: repo }); + } finally { + await replacementClient.close(); + } + } catch (error) { + if (error?.rpcCode === -32002) { + return false; + } + throw error; + } + }); + + assert.ok(Array.isArray(replacementResponse.data)); + assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 2); +}); + test("broker shutdown completes without starting a second app-server", async (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); @@ -1770,6 +1847,7 @@ test("cancel stops an active background job and marks it cancelled", async (t) = stdio: "ignore" }); sleeper.unref(); + const ownershipSnapshot = captureProcessOwnership(sleeper.pid, { cwd: workspace }); t.after(() => { try { @@ -1793,6 +1871,9 @@ test("cancel stops an active background job and marks it cancelled", async (t) = id: "task-live", status: "running", title: "Codex Task", + pid: sleeper.pid, + processIdentity: ownershipSnapshot.rootIdentity, + ownershipSnapshot, logFile }, null, @@ -1814,6 +1895,8 @@ test("cancel stops an active background job and marks it cancelled", async (t) = jobClass: "task", summary: "Investigate flaky test", pid: sleeper.pid, + processIdentity: ownershipSnapshot.rootIdentity, + ownershipSnapshot, logFile, createdAt: "2026-03-18T15:30:00.000Z", startedAt: "2026-03-18T15:30:01.000Z", @@ -2124,7 +2207,21 @@ test("session end fully cleans up jobs for the ending session", async (t) => { stdio: "ignore" }); sleeper.unref(); - fs.writeFileSync(runningJobFile, JSON.stringify({ id: "review-running" }, null, 2), "utf8"); + const ownershipSnapshot = captureProcessOwnership(sleeper.pid, { cwd: repo }); + fs.writeFileSync( + runningJobFile, + JSON.stringify( + { + id: "review-running", + pid: sleeper.pid, + processIdentity: ownershipSnapshot.rootIdentity, + ownershipSnapshot + }, + null, + 2 + ), + "utf8" + ); t.after(() => { try { @@ -2160,6 +2257,8 @@ test("session end fully cleans up jobs for the ending session", async (t) => { title: "Codex Review", sessionId: "sess-current", pid: sleeper.pid, + processIdentity: ownershipSnapshot.rootIdentity, + ownershipSnapshot, logFile: runningLog, createdAt: "2026-03-18T15:32:00.000Z", updatedAt: "2026-03-18T15:33:00.000Z" From 5f27c1a2f040a2a75d29a507964a8259d90458a4 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Tue, 28 Jul 2026 11:09:12 -0700 Subject: [PATCH 13/29] Give every persisted ownership record exactly one writer Ownership data had no single author. The parent wrote a job record at enqueue time and the worker rewrote the same file as it ran, both through unlocked whole-record writes, so correctness depended on an ordering that neither side controlled. Successive fixes kept trading one race for another. The invariant now enforced by data ownership rather than by conditionals: a process identity may be recorded only by the process it describes, or by a holder of a live unreaped handle to it, and every persisted file has exactly one writer at a time. - The enqueuer writes the queued record before spawning and never writes it again. It no longer scans the process table, which removes the window where a fast worker read a record that did not exist yet and exited. - The worker records its own identity. A process scanning for itself cannot be confused by pid reuse, because it is alive while scanning. - Task records no longer carry an ownership snapshot. Descendants and process-group members are still discovered at termination time, and the worker leads its own group, so late-spawned helpers are still reclaimed. - Cancellation of a job whose worker has not started yet uses a create-only flag file whose existence is the whole message, so it cannot be contended. This closes a case where cancelling a queued job reported success while the worker went on to run it to completion. - Spawn sites that hold a live child handle keep capturing ownership directly; that is the other half of the invariant, and it is unchanged. Attributed-To: Codex executor lane (implementation), Claude interactive (design review, verification) Co-Authored-By: Claude Opus 5 --- plugins/codex/scripts/codex-companion.mjs | 174 ++++--- plugins/codex/scripts/lib/state.mjs | 31 +- plugins/codex/scripts/lib/tracked-jobs.mjs | 22 +- .../codex/scripts/session-lifecycle-hook.mjs | 11 +- tests/runtime.test.mjs | 438 +++++++++++++++++- 5 files changed, 583 insertions(+), 93 deletions(-) diff --git a/plugins/codex/scripts/codex-companion.mjs b/plugins/codex/scripts/codex-companion.mjs index 5e6b4770a..ae354f6e1 100644 --- a/plugins/codex/scripts/codex-companion.mjs +++ b/plugins/codex/scripts/codex-companion.mjs @@ -24,7 +24,7 @@ import { import { resolveClaudeSessionPath } from "./lib/claude-session-transfer.mjs"; import { readStdinIfPiped } from "./lib/fs.mjs"; import { collectReviewContext, ensureGitRepository, resolveReviewTarget } from "./lib/git.mjs"; -import { binaryAvailable, captureProcessOwnership, terminateProcessTree } from "./lib/process.mjs"; +import { binaryAvailable, getProcessIdentity, terminateProcessTree } from "./lib/process.mjs"; import { loadPromptTemplate, interpolateTemplate } from "./lib/prompts.mjs"; import { generateJobId, @@ -32,6 +32,7 @@ import { listJobs, setConfig, upsertJob, + writeCancelFlag, writeJobFile } from "./lib/state.mjs"; import { @@ -681,38 +682,24 @@ function spawnDetachedTaskWorker(cwd, jobId) { return child; } -function enqueueBackgroundTask(cwd, job, request) { +export function enqueueBackgroundTask(cwd, job, request, dependencies = {}) { const { logFile } = createTrackedProgress(job); appendLogLine(logFile, "Queued for background execution."); - const child = spawnDetachedTaskWorker(cwd, job.id); - let ownershipSnapshot = null; - let ownershipCaptureFailed = false; - if (process.platform !== "win32") { - try { - ownershipSnapshot = captureProcessOwnership(child.pid ?? Number.NaN, { - cwd, - env: process.env - }); - ownershipCaptureFailed = !ownershipSnapshot?.rootIdentity; - } catch { - ownershipCaptureFailed = true; - } - } const queuedRecord = { ...job, status: "queued", phase: "queued", - pid: child.pid ?? null, - processIdentity: ownershipSnapshot?.rootIdentity ?? null, - ownershipSnapshot, - ownershipCaptureFailed, + pid: null, logFile, request }; writeJobFile(job.workspaceRoot, job.id, queuedRecord); upsertJob(job.workspaceRoot, queuedRecord); + const spawnWorker = dependencies.spawnDetachedTaskWorkerImpl ?? spawnDetachedTaskWorker; + spawnWorker(cwd, job.id); + return { payload: { jobId: job.id, @@ -851,7 +838,7 @@ async function handleTransfer(argv) { outputCommandResult(payload, rendered, options.json); } -async function handleTaskWorker(argv) { +export async function handleTaskWorker(argv, dependencies = {}) { const { options } = parseCommandInput(argv, { valueOptions: ["cwd", "job-id"] }); @@ -867,6 +854,20 @@ async function handleTaskWorker(argv) { throw new Error(`No stored job found for ${options["job-id"]}.`); } + const { + processIdentity: _storedProcessIdentity, + ownershipSnapshot: _storedOwnershipSnapshot, + ownershipCaptureFailed: _storedOwnershipCaptureFailed, + ...storedTask + } = storedJob; + let workerOwnership; + try { + const processIdentity = (dependencies.getProcessIdentityImpl ?? getProcessIdentity)(process.pid); + workerOwnership = processIdentity ? { processIdentity } : { ownershipCaptureFailed: true }; + } catch { + workerOwnership = { ownershipCaptureFailed: true }; + } + const request = storedJob.request; if (!request || typeof request !== "object") { throw new Error(`Stored job ${options["job-id"]} is missing its task request payload.`); @@ -874,18 +875,19 @@ async function handleTaskWorker(argv) { const { logFile, progress } = createTrackedProgress( { - ...storedJob, + ...storedTask, workspaceRoot }, { logFile: storedJob.logFile ?? null } ); - await runTrackedJob( + await (dependencies.runTrackedJobImpl ?? runTrackedJob)( { - ...storedJob, + ...storedTask, workspaceRoot, - logFile + logFile, + ...workerOwnership }, () => executeTaskRun({ @@ -976,6 +978,43 @@ function handleTaskResumeCandidate(argv) { outputCommandResult(payload, rendered, options.json); } +function finishCancelledJob(workspaceRoot, record, interrupt, options) { + appendLogLine(record.logFile, "Cancelled by user."); + + const completedAt = nowIso(); + const nextJob = { + ...record, + status: "cancelled", + phase: "cancelled", + pid: null, + completedAt, + errorMessage: "Cancelled by user." + }; + + writeJobFile(workspaceRoot, record.id, { + ...nextJob, + cancelledAt: completedAt + }); + upsertJob(workspaceRoot, { + id: record.id, + status: "cancelled", + phase: "cancelled", + pid: null, + errorMessage: "Cancelled by user.", + completedAt + }); + + const payload = { + jobId: record.id, + status: "cancelled", + title: record.title, + turnInterruptAttempted: interrupt.attempted, + turnInterrupted: interrupt.interrupted + }; + + outputCommandResult(payload, renderCancelReport(nextJob), options.json); +} + export async function handleCancel(argv, dependencies = {}) { const { options, positionals } = parseCommandInput(argv, { valueOptions: ["cwd"], @@ -985,90 +1024,71 @@ export async function handleCancel(argv, dependencies = {}) { const cwd = resolveCommandCwd(options); const reference = positionals[0] ?? ""; const { workspaceRoot, job } = resolveCancelableJob(cwd, reference, { env: process.env }); - const existing = readStoredJob(workspaceRoot, job.id) ?? {}; - const threadId = existing.threadId ?? job.threadId ?? null; - const turnId = existing.turnId ?? job.turnId ?? null; + let existing = readStoredJob(workspaceRoot, job.id) ?? {}; + let record = { ...job, ...existing }; + + if (!Number.isFinite(record.pid)) { + writeCancelFlag(workspaceRoot, job.id); + existing = readStoredJob(workspaceRoot, job.id) ?? existing; + record = { ...job, ...existing }; + if (!Number.isFinite(record.pid)) { + finishCancelledJob( + workspaceRoot, + record, + { attempted: false, interrupted: false }, + options + ); + return; + } + } + + const threadId = record.threadId ?? null; + const turnId = record.turnId ?? null; const interrupt = await (dependencies.interruptAppServerTurnImpl ?? interruptAppServerTurn)(cwd, { threadId, turnId }); if (interrupt.attempted) { appendLogLine( - job.logFile, + record.logFile, interrupt.interrupted ? `Requested Codex turn interrupt for ${turnId} on ${threadId}.` : `Codex turn interrupt failed${interrupt.detail ? `: ${interrupt.detail}` : "."}` ); } - const expectedRootIdentity = existing.processIdentity ?? job.processIdentity ?? null; - const ownershipSnapshot = existing.ownershipSnapshot ?? job.ownershipSnapshot ?? null; - const ownershipCaptureFailed = - existing.ownershipCaptureFailed === true || job.ownershipCaptureFailed === true; - const cleanupOutcome = await (dependencies.terminateProcessTreeImpl ?? terminateProcessTree)(job.pid ?? Number.NaN, { + const expectedRootIdentity = existing.processIdentity ?? null; + const ownershipCaptureFailed = existing.ownershipCaptureFailed === true; + const cleanupOutcome = await (dependencies.terminateProcessTreeImpl ?? terminateProcessTree)(record.pid, { expectedRootIdentity, - ownershipSnapshot, + ownershipSnapshot: null, requireVerifiedOwnership: ownershipCaptureFailed }); if (cleanupOutcome?.verified !== true) { const failureMessage = - ownershipCaptureFailed && !expectedRootIdentity && !ownershipSnapshot?.rootIdentity + ownershipCaptureFailed && !expectedRootIdentity ? `Job ${job.id} could not be verified as owned and was left alone.` : `Unable to verify cleanup for ${job.id}; ownership records were preserved for retry.`; - appendLogLine(job.logFile, failureMessage); + appendLogLine(record.logFile, failureMessage); const recoveryRecord = { - ...existing, - ...job, - status: job.status, + ...record, + status: record.status, phase: "cleanup-pending", - pid: job.pid ?? existing.pid ?? null, + pid: record.pid, cleanupOutcome, cleanupFailure: failureMessage }; writeJobFile(workspaceRoot, job.id, recoveryRecord); upsertJob(workspaceRoot, { id: job.id, - status: job.status, + status: record.status, phase: "cleanup-pending", - pid: job.pid ?? existing.pid ?? null, + pid: record.pid, cleanupOutcome, cleanupFailure: failureMessage }); throw new Error(failureMessage); } - appendLogLine(job.logFile, "Cancelled by user."); - - const completedAt = nowIso(); - const nextJob = { - ...job, - status: "cancelled", - phase: "cancelled", - pid: null, - completedAt, - errorMessage: "Cancelled by user." - }; - writeJobFile(workspaceRoot, job.id, { - ...existing, - ...nextJob, - cancelledAt: completedAt - }); - upsertJob(workspaceRoot, { - id: job.id, - status: "cancelled", - phase: "cancelled", - pid: null, - errorMessage: "Cancelled by user.", - completedAt - }); - - const payload = { - jobId: job.id, - status: "cancelled", - title: job.title, - turnInterruptAttempted: interrupt.attempted, - turnInterrupted: interrupt.interrupted - }; - - outputCommandResult(payload, renderCancelReport(nextJob), options.json); + finishCancelledJob(workspaceRoot, record, interrupt, options); } async function main() { diff --git a/plugins/codex/scripts/lib/state.mjs b/plugins/codex/scripts/lib/state.mjs index 2da23498f..6878ae4f8 100644 --- a/plugins/codex/scripts/lib/state.mjs +++ b/plugins/codex/scripts/lib/state.mjs @@ -107,7 +107,7 @@ export function saveState(cwd, state) { if (retainedIds.has(job.id)) { continue; } - removeJobFile(resolveJobFile(cwd, job.id)); + removeJobFile(cwd, job.id); removeFileIfExists(job.logFile); } @@ -174,10 +174,29 @@ export function readJobFile(jobFile) { return JSON.parse(fs.readFileSync(jobFile, "utf8")); } -function removeJobFile(jobFile) { - if (fs.existsSync(jobFile)) { - fs.unlinkSync(jobFile); +export function writeCancelFlag(cwd, jobId) { + const cancelFlag = resolveCancelFlag(cwd, jobId); + try { + fs.writeFileSync(cancelFlag, "", { flag: "wx" }); + } catch (error) { + if (error?.code !== "EEXIST") { + throw error; + } } + return cancelFlag; +} + +export function hasCancelFlag(cwd, jobId) { + return fs.existsSync(resolveCancelFlag(cwd, jobId)); +} + +export function removeCancelFlag(cwd, jobId) { + removeFileIfExists(resolveCancelFlag(cwd, jobId)); +} + +function removeJobFile(cwd, jobId) { + removeFileIfExists(resolveJobFile(cwd, jobId)); + removeCancelFlag(cwd, jobId); } export function resolveJobLogFile(cwd, jobId) { @@ -189,3 +208,7 @@ export function resolveJobFile(cwd, jobId) { ensureStateDir(cwd); return path.join(resolveJobsDir(cwd), `${jobId}.json`); } + +function resolveCancelFlag(cwd, jobId) { + return resolveJobFile(cwd, jobId).replace(/\.json$/, ".cancelled"); +} diff --git a/plugins/codex/scripts/lib/tracked-jobs.mjs b/plugins/codex/scripts/lib/tracked-jobs.mjs index 902869012..3414ed3c5 100644 --- a/plugins/codex/scripts/lib/tracked-jobs.mjs +++ b/plugins/codex/scripts/lib/tracked-jobs.mjs @@ -1,9 +1,10 @@ import fs from "node:fs"; import process from "node:process"; -import { readJobFile, resolveJobFile, resolveJobLogFile, upsertJob, writeJobFile } from "./state.mjs"; +import { hasCancelFlag, readJobFile, resolveJobFile, resolveJobLogFile, upsertJob, writeJobFile } from "./state.mjs"; export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; +const JOB_CANCELLED_CODE = "JOB_CANCELLED"; export function nowIso() { return new Date().toISOString(); @@ -139,6 +140,13 @@ function readStoredJobOrNull(workspaceRoot, jobId) { return readJobFile(jobFile); } +function createJobCancelledError(jobId) { + const error = new Error(`Job ${jobId} was cancelled before execution.`); + error.name = "JobCancelledError"; + error.code = JOB_CANCELLED_CODE; + return error; +} + export async function runTrackedJob(job, runner, options = {}) { const runningRecord = { ...job, @@ -152,6 +160,9 @@ export async function runTrackedJob(job, runner, options = {}) { upsertJob(job.workspaceRoot, runningRecord); try { + if (hasCancelFlag(job.workspaceRoot, job.id)) { + throw createJobCancelledError(job.id); + } const execution = await runner(); const completionStatus = execution.exitStatus === 0 ? "completed" : "failed"; const completedAt = nowIso(); @@ -182,10 +193,11 @@ export async function runTrackedJob(job, runner, options = {}) { const errorMessage = error instanceof Error ? error.message : String(error); const existing = readStoredJobOrNull(job.workspaceRoot, job.id) ?? runningRecord; const completedAt = nowIso(); + const terminalStatus = error?.code === JOB_CANCELLED_CODE ? "cancelled" : "failed"; writeJobFile(job.workspaceRoot, job.id, { ...existing, - status: "failed", - phase: "failed", + status: terminalStatus, + phase: terminalStatus, errorMessage, pid: null, completedAt, @@ -193,8 +205,8 @@ export async function runTrackedJob(job, runner, options = {}) { }); upsertJob(job.workspaceRoot, { id: job.id, - status: "failed", - phase: "failed", + status: terminalStatus, + phase: terminalStatus, pid: null, errorMessage, completedAt diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index ba3d52efd..f96f29225 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -14,7 +14,7 @@ import { sendBrokerShutdown, teardownBrokerSession } from "./lib/broker-lifecycle.mjs"; -import { loadState, resolveStateFile, saveState, writeJobFile } from "./lib/state.mjs"; +import { loadState, resolveStateFile, saveState, writeCancelFlag, writeJobFile } from "./lib/state.mjs"; import { TRANSCRIPT_PATH_ENV } from "./lib/claude-session-transfer.mjs"; import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; @@ -65,20 +65,23 @@ export async function cleanupSessionJobs(cwd, sessionId, dependencies = {}) { if (!stillRunning) { continue; } + if (job.status === "queued" && !Number.isFinite(job.pid)) { + writeCancelFlag(workspaceRoot, job.id); + continue; + } try { const expectedRootIdentity = job.processIdentity ?? null; - const ownershipSnapshot = job.ownershipSnapshot ?? null; const ownershipCaptureFailed = job.ownershipCaptureFailed === true; const outcome = await terminate(job.pid ?? Number.NaN, { expectedRootIdentity, - ownershipSnapshot, + ownershipSnapshot: null, requireVerifiedOwnership: ownershipCaptureFailed }); if (outcome?.verified === true) { continue; } const cleanupFailure = - ownershipCaptureFailed && !expectedRootIdentity && !ownershipSnapshot?.rootIdentity + ownershipCaptureFailed && !expectedRootIdentity ? `Job ${job.id} could not be verified as owned and was left alone.` : "Session cleanup could not verify process termination."; const retainedJob = { diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 347cfe125..afe0ad470 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -7,13 +7,15 @@ import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; import { initGitRepo, makeTempDir as createTempDir, run } from "./helpers.mjs"; -import { handleCancel } from "../plugins/codex/scripts/codex-companion.mjs"; +import { enqueueBackgroundTask, handleCancel, handleTaskWorker } from "../plugins/codex/scripts/codex-companion.mjs"; import { cleanupSessionJobs } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; import { isBrokerRequestAllowedDuringShutdown } from "../plugins/codex/scripts/app-server-broker.mjs"; import { ensureBrokerSession, loadBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; -import { captureProcessOwnership, terminateProcessGroup } from "../plugins/codex/scripts/lib/process.mjs"; -import { resolveStateDir } from "../plugins/codex/scripts/lib/state.mjs"; +import { readStoredJob } from "../plugins/codex/scripts/lib/job-control.mjs"; +import { captureProcessOwnership, getProcessIdentity, terminateProcessGroup } from "../plugins/codex/scripts/lib/process.mjs"; +import { hasCancelFlag, listJobs, loadState, resolveStateDir, saveState, upsertJob, writeCancelFlag, writeJobFile } from "../plugins/codex/scripts/lib/state.mjs"; +import { runTrackedJob } from "../plugins/codex/scripts/lib/tracked-jobs.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const PLUGIN_ROOT = path.join(ROOT, "plugins", "codex"); @@ -56,6 +58,121 @@ test("broker rejects queued work after shutdown begins", () => { assert.equal(isBrokerRequestAllowedDuringShutdown(false, { id: 4, method: "thread/list" }), true); }); +test("background task is persisted before its worker can start", () => { + const workspace = makeTempDir(); + const job = { + id: "task-persist-before-spawn", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Exercise the startup race", + createdAt: "2026-07-28T08:00:00.000Z" + }; + const request = { + cwd: workspace, + prompt: "Exercise the startup race", + jobId: job.id + }; + const workerProgress = { + status: "running", + phase: "investigating", + pid: 43210, + progressMarker: "worker-read-succeeded" + }; + let observedQueuedRecord = null; + + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl() { + observedQueuedRecord = readStoredJob(workspace, job.id); + writeJobFile(workspace, job.id, { + ...observedQueuedRecord, + ...workerProgress + }); + upsertJob(workspace, { + id: job.id, + ...workerProgress + }); + } + }); + + assert.equal(observedQueuedRecord.status, "queued"); + assert.equal(observedQueuedRecord.phase, "queued"); + assert.equal(observedQueuedRecord.pid, null); + assert.deepEqual(observedQueuedRecord.request, request); + assert.equal(Object.hasOwn(observedQueuedRecord, "processIdentity"), false); + assert.equal(Object.hasOwn(observedQueuedRecord, "ownershipSnapshot"), false); + assert.equal(Object.hasOwn(observedQueuedRecord, "ownershipCaptureFailed"), false); + + const storedJob = readStoredJob(workspace, job.id); + assert.equal(storedJob.status, workerProgress.status); + assert.equal(storedJob.phase, workerProgress.phase); + assert.equal(storedJob.pid, workerProgress.pid); + assert.equal(storedJob.progressMarker, workerProgress.progressMarker); + + const indexedJob = listJobs(workspace).find((candidate) => candidate.id === job.id); + assert.equal(indexedJob.status, workerProgress.status); + assert.equal(indexedJob.phase, workerProgress.phase); + assert.equal(indexedJob.pid, workerProgress.pid); + assert.equal(indexedJob.progressMarker, workerProgress.progressMarker); +}); + +test("background task stays runnable when worker identity capture fails", async () => { + const workspace = makeTempDir(); + const job = { + id: "task-capture-failure", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Exercise ownership capture failure", + createdAt: "2026-07-28T08:01:00.000Z" + }; + const request = { + cwd: workspace, + prompt: "Exercise ownership capture failure", + jobId: job.id + }; + let workerJob = null; + let workPerformed = false; + + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl() {} + }); + await handleTaskWorker(["--cwd", workspace, "--job-id", job.id], { + getProcessIdentityImpl() { + throw new Error("injected ownership capture failure"); + }, + runTrackedJobImpl(candidate, _runner, options) { + workerJob = candidate; + return runTrackedJob( + candidate, + async () => { + workPerformed = true; + return { + exitStatus: 0, + payload: { ok: true }, + rendered: "Worker completed.", + summary: "Worker completed." + }; + }, + options + ); + } + }); + + assert.equal(workPerformed, true); + assert.equal(workerJob.ownershipCaptureFailed, true); + assert.equal(Object.hasOwn(workerJob, "processIdentity"), false); + assert.equal(Object.hasOwn(workerJob, "ownershipSnapshot"), false); + + const storedJob = readStoredJob(workspace, job.id); + assert.equal(storedJob.status, "completed"); + assert.equal(storedJob.ownershipCaptureFailed, true); + assert.equal(Object.hasOwn(storedJob, "processIdentity"), false); + assert.equal(Object.hasOwn(storedJob, "ownershipSnapshot"), false); +}); + test("fake app-server crash reclaims an observed regrouped helper without replacement", async (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); @@ -1835,6 +1952,111 @@ test("result for a finished write-capable task returns the raw Codex final respo assert.match(result.stdout, /Resume in Codex: codex resume thr_[a-z0-9]+/i); }); +test("task worker persists its own process identity and cancel verifies cleanup", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identity is not available on Windows."); + return; + } + + const workspace = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "interruptible-slow-task"); + initGitRepo(workspace); + fs.writeFileSync(path.join(workspace, "README.md"), "hello\n", "utf8"); + + const job = { + id: "task-worker-self-identity", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Verify worker self-identity", + createdAt: "2026-07-28T08:03:00.000Z" + }; + const request = { + cwd: workspace, + model: null, + effort: null, + prompt: "Wait while cancellation verifies worker ownership.", + write: false, + resumeLast: false, + jobId: job.id + }; + let worker = null; + + t.after(() => { + if (!worker?.pid) { + return; + } + try { + process.kill(-worker.pid, "SIGKILL"); + } catch { + try { + process.kill(worker.pid, "SIGKILL"); + } catch { + // Ignore a worker already reclaimed by cancellation. + } + } + }); + + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl(cwd, jobId) { + worker = spawn(process.execPath, [SCRIPT, "task-worker", "--cwd", cwd, "--job-id", jobId], { + cwd, + env: buildEnv(binDir), + detached: true, + stdio: "ignore" + }); + worker.unref(); + return worker; + } + }); + + const runningJob = await waitFor(() => { + const stored = readStoredJob(workspace, job.id); + return stored?.status === "running" && Number.isFinite(stored.pid) ? stored : null; + }); + + if (runningJob.ownershipCaptureFailed === true) { + t.skip("Process table unavailable for worker self-identity verification."); + return; + } + + let liveIdentity; + try { + liveIdentity = getProcessIdentity(runningJob.pid); + } catch (error) { + if (error?.code === "PROCESS_TABLE_UNAVAILABLE") { + t.skip(`process table unavailable: ${error.message}`); + return; + } + throw error; + } + + assert.equal(runningJob.pid, worker.pid); + assert.equal(runningJob.processIdentity, liveIdentity); + assert.equal(Object.hasOwn(runningJob, "ownershipSnapshot"), false); + assert.equal(Object.hasOwn(runningJob, "ownershipCaptureFailed"), false); + + await handleCancel([job.id, "--cwd", workspace, "--json"], { + interruptAppServerTurnImpl: async () => ({ attempted: false, interrupted: false }) + }); + + await waitFor(() => { + try { + process.kill(worker.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + + const cancelledJob = readStoredJob(workspace, job.id); + assert.equal(cancelledJob.status, "cancelled"); + assert.equal(cancelledJob.pid, null); + assert.equal(cancelledJob.processIdentity, liveIdentity); +}); + test("cancel stops an active background job and marks it cancelled", async (t) => { const workspace = makeTempDir(); const stateDir = resolveStateDir(workspace); @@ -1936,6 +2158,216 @@ test("cancel stops an active background job and marks it cancelled", async (t) = assert.match(fs.readFileSync(logFile, "utf8"), /Cancelled by user/); }); +test("cancelling a queued pid-less job prevents its worker from performing work", async () => { + const workspace = makeTempDir(); + const job = { + id: "task-queued-cancel", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + status: "queued", + phase: "queued", + pid: null, + createdAt: "2026-07-28T08:02:00.000Z", + updatedAt: "2026-07-28T08:02:00.000Z" + }; + writeJobFile(workspace, job.id, job); + upsertJob(workspace, job); + + let terminateCalled = false; + await handleCancel([job.id, "--cwd", workspace, "--json"], { + interruptAppServerTurnImpl: async () => ({ attempted: false, interrupted: false }), + terminateProcessTreeImpl: async () => { + terminateCalled = true; + return { + attempted: false, + delivered: false, + verified: true, + degraded: false + }; + } + }); + + assert.equal(terminateCalled, false); + assert.equal(hasCancelFlag(workspace, job.id), true); + + const workMarker = path.join(workspace, "work-performed"); + await assert.rejects( + runTrackedJob(job, async () => { + fs.writeFileSync(workMarker, "ran\n", "utf8"); + return { + exitStatus: 0, + payload: { ok: true }, + rendered: "Work ran.", + summary: "Work ran." + }; + }), + (error) => error?.code === "JOB_CANCELLED" + ); + + assert.equal(fs.existsSync(workMarker), false); +}); + +test("worker converges a flagged running record to cancelled", async () => { + const workspace = makeTempDir(); + const job = { + id: "task-flag-convergence", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + status: "queued", + phase: "queued", + pid: null, + createdAt: "2026-07-28T08:04:00.000Z" + }; + writeJobFile(workspace, job.id, job); + upsertJob(workspace, job); + writeCancelFlag(workspace, job.id); + + let workPerformed = false; + await assert.rejects( + runTrackedJob(job, async () => { + workPerformed = true; + return { + exitStatus: 0, + payload: { ok: true }, + rendered: "Work ran.", + summary: "Work ran." + }; + }), + (error) => error?.name === "JobCancelledError" && error?.code === "JOB_CANCELLED" + ); + + assert.equal(workPerformed, false); + const storedJob = readStoredJob(workspace, job.id); + assert.equal(storedJob.status, "cancelled"); + assert.equal(storedJob.phase, "cancelled"); + assert.equal(storedJob.pid, null); + assert.match(storedJob.startedAt, /^\d{4}-\d{2}-\d{2}T/); + + const indexedJob = listJobs(workspace).find((candidate) => candidate.id === job.id); + assert.equal(indexedJob.status, "cancelled"); + assert.equal(indexedJob.phase, "cancelled"); + assert.equal(indexedJob.pid, null); + + saveState(workspace, { + ...loadState(workspace), + jobs: [] + }); + assert.equal(readStoredJob(workspace, job.id), null); + assert.equal(hasCancelFlag(workspace, job.id), false); +}); + +test("cancel reclaims a helper spawned after worker identity capture without an ownership snapshot", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process groups are not available on Windows."); + return; + } + + const workspace = makeTempDir(); + const helperPidFile = path.join(workspace, "late-helper.pid"); + const leader = spawn( + process.execPath, + [ + "-e", + ` + const fs = require("node:fs"); + const { spawn } = require("node:child_process"); + setTimeout(() => { + const helper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + stdio: "ignore" + }); + helper.unref(); + fs.writeFileSync(${JSON.stringify(helperPidFile)}, String(helper.pid)); + }, 750); + setInterval(() => {}, 1000); + ` + ], + { + cwd: workspace, + detached: true, + stdio: "ignore" + } + ); + leader.unref(); + let helperPid = null; + + t.after(() => { + try { + process.kill(-leader.pid, "SIGKILL"); + } catch { + try { + process.kill(leader.pid, "SIGKILL"); + } catch { + // Ignore a leader already reclaimed by cancellation. + } + } + if (Number.isFinite(helperPid)) { + try { + process.kill(helperPid, "SIGKILL"); + } catch { + // Ignore a helper already reclaimed with the worker group. + } + } + }); + + let leaderIdentity; + try { + leaderIdentity = getProcessIdentity(leader.pid); + } catch (error) { + if (error?.code === "PROCESS_TABLE_UNAVAILABLE") { + t.skip(`process table unavailable: ${error.message}`); + return; + } + throw error; + } + assert.ok(leaderIdentity); + assert.equal(fs.existsSync(helperPidFile), false); + + const logFile = path.join(workspace, "late-helper.log"); + fs.writeFileSync(logFile, "", "utf8"); + const job = { + id: "task-late-helper", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + status: "running", + phase: "running", + pid: leader.pid, + processIdentity: leaderIdentity, + logFile, + createdAt: "2026-07-28T08:05:00.000Z", + updatedAt: "2026-07-28T08:05:00.000Z" + }; + writeJobFile(workspace, job.id, job); + upsertJob(workspace, job); + + helperPid = await waitFor(() => { + if (!fs.existsSync(helperPidFile)) { + return null; + } + return Number(fs.readFileSync(helperPidFile, "utf8")); + }); + assert.ok(Number.isFinite(helperPid)); + + await handleCancel([job.id, "--cwd", workspace, "--json"], { + interruptAppServerTurnImpl: async () => ({ attempted: false, interrupted: false }) + }); + + function isRunning(pid) { + const result = run("/bin/ps", ["-o", "stat=", "-p", String(pid)]); + return result.status === 0 && !result.stdout.trim().startsWith("Z"); + } + + await waitFor(() => !isRunning(leader.pid)); + await waitFor(() => !isRunning(helperPid)); + assert.equal(readStoredJob(workspace, job.id).status, "cancelled"); + assert.equal(Object.hasOwn(readStoredJob(workspace, job.id), "ownershipSnapshot"), false); +}); + test("unverified cleanup preserves cancel, session, and broker ownership records", async () => { const workspace = makeTempDir(); const stateDir = resolveStateDir(workspace); From 03eac6e0da6e731b4104a9f6493d7a6f72e5c0a4 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Tue, 28 Jul 2026 11:46:21 -0700 Subject: [PATCH 14/29] Let a cancel tombstone outlive the pass that wrote it, and let absent roots converge Two defects in the previous commit, both found by review and both confirmed against the source before changing anything. A cancel tombstone for a queued job was written and then deleted in the same session-end pass, because the job was pruned and pruning sweeps its flag. A worker caught between reading its queued record and publishing its pid would find no flag and run the job after the session had ended. The job is now retained until the worker acknowledges the cancellation with its own terminal write; pruning still sweeps the flag once the job is genuinely finished. Cleanup could not converge for a worker that exited abnormally. Since task records stopped carrying an ownership snapshot, cancellation supplies a persisted identity instead, and the branch handling a root that is absent from a readable process table still keyed on the snapshot. It reported unverified forever, leaving the job permanently cleanup-pending and failing session end. The decision now keys on the process table having been read successfully: a root absent from a readable table, with nothing owned unaccounted for, is a verified success whichever form the ownership evidence took. The failure cases are unchanged and covered: an unreadable table stays unverified, a live tracked descendant stays unverified, a present root whose identity does not match stays a refusal, and an absent pid with no ownership evidence at all is still not a success. Attributed-To: Codex executor lane (implementation), Claude interactive (triage, verification) Co-Authored-By: Claude Opus 5 --- plugins/codex/scripts/lib/process.mjs | 32 +++-- .../codex/scripts/session-lifecycle-hook.mjs | 1 + tests/process.test.mjs | 27 ++++ tests/runtime.test.mjs | 134 +++++++++++++++++- 4 files changed, 178 insertions(+), 16 deletions(-) diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index e6526d5f7..5a605ff4c 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -509,18 +509,6 @@ export async function terminateProcessTree(pid, options = {}) { survivorIdentities: [] }); } - if (!root) { - if (!ownershipSnapshot) { - return normalizeProcessCleanupOutcome({ - attempted: true, - delivered: false, - verified: !ownershipEstablished, - degraded: ownershipEstablished, - method: "process-tree", - targets: [] - }); - } - } if (root && expectedRootIdentity && root.identity !== expectedRootIdentity) { return normalizeProcessCleanupOutcome({ attempted: true, @@ -540,6 +528,20 @@ export async function terminateProcessTree(pid, options = {}) { for (const record of root ? collectProcessTree(pid, initialProcesses) : []) { tracked.set(record.identity, record); } + const sortedTracked = () => + [...tracked.values()].sort((left, right) => right.depth - left.depth); + if (!root && listLiveTracked(tracked, initialProcesses).length === 0) { + const verified = !options.requireVerifiedOwnership && ownershipEstablished; + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified, + degraded: ownershipEstablished && !verified, + method: "process-tree", + targets: sortedTracked().map((record) => record.pid), + targetIdentities: sortedTracked().map((record) => record.identity) + }); + } let cleanupRootIdentity = expectedRootIdentity; if (!cleanupRootIdentity && captureFailureCleanupAllowed && root) { // A live, unreaped child handle prevents PID reuse. Its owner may use the @@ -585,7 +587,7 @@ export async function terminateProcessTree(pid, options = {}) { const verified = live.length === 0 && !options.requireVerifiedOwnership && - (root !== undefined || ownershipSnapshot !== null); + ownershipEstablished; return normalizeProcessCleanupOutcome({ attempted: true, delivered, @@ -595,8 +597,8 @@ export async function terminateProcessTree(pid, options = {}) { method: "process-tree", // The algorithm covers same-process-group descendants plus those observed at scan time. // A post-scan setsid descendant can escape the tracked process tree. - targets: [...tracked.values()].sort((left, right) => right.depth - left.depth).map((record) => record.pid), - targetIdentities: [...tracked.values()].sort((left, right) => right.depth - left.depth).map((record) => record.identity), + targets: sortedTracked().map((record) => record.pid), + targetIdentities: sortedTracked().map((record) => record.identity), survivors: live.map((record) => record.pid), survivorIdentities: live.map((record) => record.identity) }); diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index f96f29225..5e54ce9e1 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -67,6 +67,7 @@ export async function cleanupSessionJobs(cwd, sessionId, dependencies = {}) { } if (job.status === "queued" && !Number.isFinite(job.pid)) { writeCancelFlag(workspaceRoot, job.id); + retainedJobs.push(job); continue; } try { diff --git a/tests/process.test.mjs b/tests/process.test.mjs index bccc108ae..862a93c0f 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -320,6 +320,33 @@ test("terminateProcessTree refuses a PID without persisted ownership", async () assert.deepEqual(outcome.survivorIdentities, []); }); +test("terminateProcessTree refuses an absent PID without persisted ownership", async () => { + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "", + stderr: "", + error: null + }; + }, + killImpl() { + throw new Error("a PID without persisted ownership must not be signaled"); + } + }); + + assert.equal(outcome.attempted, true); + assert.equal(outcome.delivered, false); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.deepEqual(outcome.survivors, [1234]); + assert.deepEqual(outcome.survivorIdentities, []); +}); + test("terminateProcessTree refuses capture-failure cleanup without a live owner handle", async () => { const identityObservedAtSpawn = "1234@Sun Jul 26 00:00:00 2026"; const identityNowHoldingPid = "1234@Mon Jul 27 00:00:00 2026"; diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index afe0ad470..100c4b212 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -13,7 +13,7 @@ import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mj import { isBrokerRequestAllowedDuringShutdown } from "../plugins/codex/scripts/app-server-broker.mjs"; import { ensureBrokerSession, loadBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { readStoredJob } from "../plugins/codex/scripts/lib/job-control.mjs"; -import { captureProcessOwnership, getProcessIdentity, terminateProcessGroup } from "../plugins/codex/scripts/lib/process.mjs"; +import { captureProcessOwnership, getProcessIdentity, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; import { hasCancelFlag, listJobs, loadState, resolveStateDir, saveState, upsertJob, writeCancelFlag, writeJobFile } from "../plugins/codex/scripts/lib/state.mjs"; import { runTrackedJob } from "../plugins/codex/scripts/lib/tracked-jobs.mjs"; @@ -2209,6 +2209,72 @@ test("cancelling a queued pid-less job prevents its worker from performing work" assert.equal(fs.existsSync(workMarker), false); }); +test("session cleanup preserves a queued cancel between worker read and pid publication", async () => { + const workspace = makeTempDir(); + const sessionId = "sess-queued-worker-race"; + const job = { + id: "task-session-end-race", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Exercise the SessionEnd startup race", + sessionId, + createdAt: "2026-07-28T08:03:00.000Z" + }; + const request = { + cwd: workspace, + prompt: "Do not run after SessionEnd.", + jobId: job.id + }; + const workMarker = path.join(workspace, "work-performed"); + let firstCleanupPromise = null; + let cancelFlagSurvivedCleanup = false; + + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl() {} + }); + + await assert.rejects( + handleTaskWorker(["--cwd", workspace, "--job-id", job.id], { + getProcessIdentityImpl(pid) { + // handleTaskWorker has already read the queued record, but runTrackedJob + // has not yet published this worker's pid. + firstCleanupPromise = cleanupSessionJobs(workspace, sessionId); + cancelFlagSurvivedCleanup = hasCancelFlag(workspace, job.id); + return `${pid}@Mon Jul 28 08:03:01 2026`; + }, + runTrackedJobImpl(candidate, _runner, options) { + return runTrackedJob( + candidate, + async () => { + fs.writeFileSync(workMarker, "ran\n", "utf8"); + return { + exitStatus: 0, + payload: { ok: true }, + rendered: "Work ran.", + summary: "Work ran." + }; + }, + options + ); + } + }), + (error) => error?.code === "JOB_CANCELLED" + ); + + const firstCleanup = await firstCleanupPromise; + assert.equal(firstCleanup.verified, false); + assert.equal(cancelFlagSurvivedCleanup, true); + assert.equal(fs.existsSync(workMarker), false); + assert.equal(readStoredJob(workspace, job.id).status, "cancelled"); + + const finalCleanup = await cleanupSessionJobs(workspace, sessionId); + assert.equal(finalCleanup.verified, true); + assert.equal(readStoredJob(workspace, job.id), null); + assert.equal(hasCancelFlag(workspace, job.id), false); +}); + test("worker converges a flagged running record to cancelled", async () => { const workspace = makeTempDir(); const job = { @@ -2368,6 +2434,72 @@ test("cancel reclaims a helper spawned after worker identity capture without an assert.equal(Object.hasOwn(readStoredJob(workspace, job.id), "ownershipSnapshot"), false); }); +test("cancel and session cleanup converge after an identity-tracked worker exits abnormally", async () => { + const workspace = makeTempDir(); + const sessionId = "sess-abnormal-worker-exit"; + const job = { + id: "task-abnormal-worker-exit", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Clean up an exited worker", + sessionId, + status: "running", + phase: "running", + pid: 43210, + processIdentity: "43210@Mon Jul 28 08:05:00 2026", + createdAt: "2026-07-28T08:05:00.000Z" + }; + let processTableReads = 0; + let terminateCalledDuringSessionCleanup = false; + + writeJobFile(workspace, job.id, job); + upsertJob(workspace, job); + + await handleCancel([job.id, "--cwd", workspace, "--json"], { + interruptAppServerTurnImpl: async () => ({ attempted: false, interrupted: false }), + terminateProcessTreeImpl(pid, options) { + return terminateProcessTree(pid, { + ...options, + platform: "darwin", + runCommandImpl(command, args) { + processTableReads += 1; + return { + command, + args, + status: 0, + signal: null, + stdout: "", + stderr: "", + error: null + }; + }, + killImpl() { + throw new Error("an absent worker must not be signaled"); + } + }); + } + }); + + const cancelledJob = readStoredJob(workspace, job.id); + assert.equal(processTableReads, 1); + assert.equal(cancelledJob.status, "cancelled"); + assert.equal(cancelledJob.pid, null); + assert.equal(cancelledJob.processIdentity, job.processIdentity); + assert.equal(Object.hasOwn(cancelledJob, "ownershipSnapshot"), false); + + const sessionCleanup = await cleanupSessionJobs(workspace, sessionId, { + terminateProcessTreeImpl() { + terminateCalledDuringSessionCleanup = true; + throw new Error("terminal jobs must not be terminated again"); + } + }); + assert.equal(sessionCleanup.verified, true); + assert.equal(terminateCalledDuringSessionCleanup, false); + assert.equal(readStoredJob(workspace, job.id), null); +}); + test("unverified cleanup preserves cancel, session, and broker ownership records", async () => { const workspace = makeTempDir(); const stateDir = resolveStateDir(workspace); From 3f95424ce18d26ba508557ffec10532c8d849404 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Tue, 28 Jul 2026 12:04:12 -0700 Subject: [PATCH 15/29] Refuse a process group whose leader identity no longer matches Admitting live group members that postdate the ownership snapshot made reclaiming a late-spawned helper possible, but it also admitted the members of an unrelated group when the group id itself had been reused. Those members are absent from the snapshot, so the per-record identity check could not exclude them, and the leader identity check ran only after signals had already been delivered. The result was signalling a stranger's process tree and reporting it as unverified afterwards. Check the leader before admitting anything: if a live process holds the group id and is not the root we recorded, nothing in that group is ours, so refuse without signalling. A missing leader is still not a mismatch, which keeps the crashed-root case working where surviving helpers do need reclaiming. Attributed-To: Claude interactive Co-Authored-By: Claude Opus 5 --- plugins/codex/scripts/lib/process.mjs | 20 +++++++++++ tests/process.test.mjs | 48 +++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 5a605ff4c..525100ecf 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -645,6 +645,26 @@ export async function terminateProcessGroup(pgid, options = {}) { const tracked = new Map(); const ownershipSnapshot = options.ownershipSnapshot ?? null; const ownershipEstablished = Boolean(ownershipSnapshot); + // If a live process holds this group id but is not the root we recorded, the + // id has been reused and nothing in the group is ours. Refuse before + // admitting any member: members of an unrelated group are absent from the + // snapshot, so the per-record check below would admit them, and the leader + // identity check runs only after signals have already been delivered. + // A missing leader is not a mismatch — that is the crashed-root case where + // surviving helpers still need reclaiming. + const groupLeader = processes.get(pgid); + if (ownershipSnapshot && groupLeader && groupLeader.identity !== ownershipSnapshot.rootIdentity) { + return normalizeProcessCleanupOutcome({ + attempted: false, + delivered: false, + verified: false, + degraded: true, + identityMismatch: true, + method: "process-group", + survivors: recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.pid), + survivorIdentities: recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.identity) + }); + } for (const record of recordsFromOwnershipSnapshot(ownershipSnapshot)) { tracked.set(record.identity, record); } diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 862a93c0f..bd9014e68 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -785,3 +785,51 @@ test("terminateProcessGroup reclaims a post-snapshot member of the owned group", assert.equal(outcome.degraded, false); assert.deepEqual(outcome.survivors, []); }); + +test("terminateProcessGroup signals nothing when the group id was reused by another leader", async () => { + const signals = []; + const ownershipSnapshot = { + rootPid: 100, + rootIdentity: "100@Mon Jul 27 00:00:00 2026", + processGroupId: 100, + members: [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "100@Mon Jul 27 00:00:00 2026", + depth: 0 + } + ] + }; + // pid 100 now belongs to an unrelated process that leads its own group, and + // pid 301 is a member of that stranger's group. 301 is absent from the + // snapshot, so the per-record check cannot exclude it. + const outcome = await terminateProcessGroup(100, { + platform: "darwin", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl() { + return { + command: "/bin/ps", + args: [], + status: 0, + signal: null, + stdout: + "100 1 100 S Tue Jul 28 09:00:00 2026\n301 100 100 S Tue Jul 28 09:00:01 2026\n", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.deepEqual(outcome.survivors, [100]); +}); From 19ac644d826b1455329b3129081d07aedd60b1b8 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Tue, 28 Jul 2026 12:16:14 -0700 Subject: [PATCH 16/29] Tie stream ownership to the turn rather than to its client socket Retaining stream lifetime after a client disconnects left one piece still keyed on the socket. A client that disconnected before a detached review/start response arrived cleared the active stream socket, which suppressed recording the review thread id from that result. The completion notification then carried a thread the broker had never recorded, so it never matched, the stream stayed marked running, and the broker remained busy indefinitely. Each streaming request now takes a turn number and the result-derived update is gated on that turn still being the active one. The owning socket may come and go; the turn is what the stream actually belongs to. Known gap, stated rather than implied: unlike the other fixes on this branch, this one has no dedicated regression test. Covering it needs a broker integration test around detached review streams, which is the least stable part of the suite. The full suite passes unchanged. Attributed-To: Claude interactive Co-Authored-By: Claude Opus 5 --- plugins/codex/scripts/app-server-broker.mjs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 3cd1a4ebf..8f89a3a72 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -104,6 +104,11 @@ async function main() { let activeStreamSocket = null; let activeStreamThreadIds = null; let activeStreamRunning = false; + // Identifies the turn that owns the current stream. The owning socket may + // disconnect while the turn keeps running, so socket identity cannot be used + // to decide whether a late result still belongs to the active stream. + let activeStreamTurn = 0; + let streamTurnCounter = 0; let blockedCleanup = null; const sockets = new Set(); @@ -127,6 +132,7 @@ async function main() { activeStreamRunning = false; activeStreamSocket = null; activeStreamThreadIds = null; + activeStreamTurn = 0; } function recordUnverifiedCleanup(client) { @@ -445,10 +451,13 @@ async function main() { const isStreaming = STREAMING_METHODS.has(message.method); activeRequestSocket = socket; + let streamTurn = 0; if (isStreaming) { activeStreamRunning = true; activeStreamSocket = socket; activeStreamThreadIds = buildStreamThreadIds(message.method, message.params ?? {}, null); + streamTurn = ++streamTurnCounter; + activeStreamTurn = streamTurn; } inFlightRequests += 1; @@ -456,7 +465,11 @@ async function main() { const client = await getAppClient(); const result = await client.request(message.method, message.params ?? {}); send(socket, { id: message.id, result }); - if (isStreaming && activeStreamRunning && activeStreamSocket === socket) { + // Gate on the turn, not the socket: a client that disconnects mid-turn + // clears activeStreamSocket while the turn keeps running, and without + // the result-derived thread ids the completion notification can never + // match, leaving the broker busy forever. + if (isStreaming && activeStreamRunning && activeStreamTurn === streamTurn) { activeStreamThreadIds = buildStreamThreadIds(message.method, message.params ?? {}, result); } if (activeRequestSocket === socket) { From 6e8659be6ce7cd81ab61ce79543775196124b72d Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Wed, 29 Jul 2026 16:35:27 -0700 Subject: [PATCH 17/29] [CC-5571] Make detached test helpers self-expire Attributed-To: Codex --- tests/fake-codex-fixture.mjs | 8 +++++--- tests/runtime.test.mjs | 37 ++++++++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index b16d8e6fa..6f69e352d 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -15,6 +15,8 @@ const readline = require("node:readline"); const STATE_PATH = ${JSON.stringify(statePath)}; const BEHAVIOR = ${JSON.stringify(behavior)}; + const DETACHED_FIXTURE_TTL_MS = 5 * 60 * 1000; + const SELF_EXPIRING_KEEPALIVE = "setTimeout(() => process.exit(0), " + DETACHED_FIXTURE_TTL_MS + "); setInterval(() => {}, 1000)"; const interruptibleTurns = new Map(); const { spawn } = require("node:child_process"); @@ -276,8 +278,8 @@ bootState.appServerStarts = (bootState.appServerStarts || 0) + 1; if (BEHAVIOR === "with-helper-child" || BEHAVIOR === "slow-task-with-helper-child" || BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "with-resistant-helper" || BEHAVIOR === "crash-with-post-snapshot-helper") { if (BEHAVIOR !== "crash-with-post-snapshot-helper") { const helperCode = BEHAVIOR === "with-resistant-helper" - ? "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)" - : "setInterval(() => {}, 1000)"; + ? "process.on('SIGTERM', () => {}); " + SELF_EXPIRING_KEEPALIVE + : SELF_EXPIRING_KEEPALIVE; const helper = spawn(process.execPath, ["-e", helperCode], { detached: process.platform !== "win32", stdio: "ignore" @@ -311,7 +313,7 @@ rl.on("line", (line) => { } if (BEHAVIOR === "crash-with-post-snapshot-helper") { setTimeout(() => { - const helper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + const helper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { detached: false, stdio: "ignore" }); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 100c4b212..c54cc50f9 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -22,6 +22,8 @@ const PLUGIN_ROOT = path.join(ROOT, "plugins", "codex"); const SCRIPT = path.join(PLUGIN_ROOT, "scripts", "codex-companion.mjs"); const STOP_HOOK = path.join(PLUGIN_ROOT, "scripts", "stop-review-gate-hook.mjs"); const SESSION_HOOK = path.join(PLUGIN_ROOT, "scripts", "session-lifecycle-hook.mjs"); +const DETACHED_FIXTURE_TTL_MS = 5 * 60 * 1000; +const SELF_EXPIRING_KEEPALIVE = selfExpiringKeepaliveCode(); const runtimeTempDirs = new Set(); const runtimePluginDataDir = createTempDir("codex-plugin-runtime-state-"); process.env.CLAUDE_PLUGIN_DATA = runtimePluginDataDir; @@ -32,6 +34,10 @@ function makeTempDir(prefix) { return tempDir; } +function selfExpiringKeepaliveCode(ttlMs = DETACHED_FIXTURE_TTL_MS) { + return `setTimeout(() => process.exit(0), ${ttlMs}); setInterval(() => {}, 1000)`; +} + test.after(() => { const cleanupFailures = []; @@ -58,6 +64,29 @@ test("broker rejects queued work after shutdown begins", () => { assert.equal(isBrokerRequestAllowedDuringShutdown(false, { id: 4, method: "thread/list" }), true); }); +test("detached fixture keepalive self-expires when parent cleanup is skipped", async (t) => { + if (process.platform === "win32") { + t.skip("Unix detached fixture behavior is not available on Windows."); + return; + } + + const sleeper = spawn(process.execPath, ["-e", selfExpiringKeepaliveCode(500)], { + detached: true, + stdio: "ignore" + }); + sleeper.unref(); + assert.doesNotThrow(() => process.kill(sleeper.pid, 0)); + + await waitFor(() => { + try { + process.kill(sleeper.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }, { timeoutMs: 3000 }); +}); + test("background task is persisted before its worker can start", () => { const workspace = makeTempDir(); const job = { @@ -2063,7 +2092,7 @@ test("cancel stops an active background job and marks it cancelled", async (t) = const jobsDir = path.join(stateDir, "jobs"); fs.mkdirSync(jobsDir, { recursive: true }); - const sleeper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + const sleeper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { cwd: workspace, detached: true, stdio: "ignore" @@ -2342,13 +2371,13 @@ test("cancel reclaims a helper spawned after worker identity capture without an const fs = require("node:fs"); const { spawn } = require("node:child_process"); setTimeout(() => { - const helper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + const helper = spawn(process.execPath, ["-e", ${JSON.stringify(SELF_EXPIRING_KEEPALIVE)}], { stdio: "ignore" }); helper.unref(); fs.writeFileSync(${JSON.stringify(helperPidFile)}, String(helper.pid)); }, 750); - setInterval(() => {}, 1000); + ${SELF_EXPIRING_KEEPALIVE}; ` ], { @@ -2765,7 +2794,7 @@ test("session end fully cleans up jobs for the ending session", async (t) => { fs.writeFileSync(completedJobFile, JSON.stringify({ id: "review-completed" }, null, 2), "utf8"); fs.writeFileSync(otherJobFile, JSON.stringify({ id: "review-other" }, null, 2), "utf8"); - const sleeper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + const sleeper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { cwd: repo, detached: true, stdio: "ignore" From 9d0b378d42b9a12e9deaa6579c94858f6678f657 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Wed, 29 Jul 2026 20:29:12 -0700 Subject: [PATCH 18/29] [CC-5576] Close crash cleanup and spawn failure gaps Attributed-To: Codex executor lane --- plugins/codex/scripts/app-server-broker.mjs | 15 +-- plugins/codex/scripts/codex-companion.mjs | 39 +++++- plugins/codex/scripts/lib/app-server.mjs | 80 ++++++++++-- tests/runtime.test.mjs | 136 ++++++++++++++++++++ 4 files changed, 247 insertions(+), 23 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 8f89a3a72..c3d093410 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -9,7 +9,7 @@ import { fileURLToPath } from "node:url"; import { parseArgs } from "./lib/args.mjs"; import { BROKER_BUSY_RPC_CODE, CodexAppServerClient } from "./lib/app-server.mjs"; import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs"; -import { getLiveProcessPids, normalizeProcessCleanupOutcome, terminateProcessGroup } from "./lib/process.mjs"; +import { getLiveProcessPids } from "./lib/process.mjs"; const DEFAULT_CHILD_IDLE_MS = 5 * 60 * 1000; const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]); @@ -284,12 +284,11 @@ async function main() { if (!client.closed && childPid != null && process.platform !== "win32") { // The child is a detached process-group leader; on an unexpected // exit its surviving helpers reparent away from the broker, so - // reclaim the group before allowing a replacement to spawn. - appClientClosePromise = terminateProcessGroup(childPid, { - ownershipSnapshot: client.ownershipSnapshot - }) - .then((outcome) => { - client.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); + // wait for the client's shared direct/broker crash cleanup before + // allowing a replacement to spawn. + appClientClosePromise = client + .waitForUnexpectedExitCleanup() + ?.then(() => { recordUnverifiedCleanup(client); }) .catch((error) => { @@ -298,7 +297,7 @@ async function main() { }) .finally(() => { appClientClosePromise = null; - }); + }) ?? null; } scheduleChildIdleClose(); }); diff --git a/plugins/codex/scripts/codex-companion.mjs b/plugins/codex/scripts/codex-companion.mjs index ae354f6e1..e41f80210 100644 --- a/plugins/codex/scripts/codex-companion.mjs +++ b/plugins/codex/scripts/codex-companion.mjs @@ -682,6 +682,34 @@ function spawnDetachedTaskWorker(cwd, jobId) { return child; } +function recordTaskWorkerSpawnFailure(workspaceRoot, jobId, error) { + const storedJob = readStoredJob(workspaceRoot, jobId); + if (!storedJob || storedJob.status !== "queued" || storedJob.pid != null) { + return; + } + + const errorMessage = error instanceof Error ? error.message : String(error); + const completedAt = nowIso(); + const failedRecord = { + ...storedJob, + status: "failed", + phase: "failed", + pid: null, + errorMessage, + completedAt + }; + writeJobFile(workspaceRoot, jobId, failedRecord); + upsertJob(workspaceRoot, { + id: jobId, + status: "failed", + phase: "failed", + pid: null, + errorMessage, + completedAt + }); + appendLogLine(storedJob.logFile, `Worker spawn failed: ${errorMessage}`); +} + export function enqueueBackgroundTask(cwd, job, request, dependencies = {}) { const { logFile } = createTrackedProgress(job); appendLogLine(logFile, "Queued for background execution."); @@ -698,7 +726,16 @@ export function enqueueBackgroundTask(cwd, job, request, dependencies = {}) { upsertJob(job.workspaceRoot, queuedRecord); const spawnWorker = dependencies.spawnDetachedTaskWorkerImpl ?? spawnDetachedTaskWorker; - spawnWorker(cwd, job.id); + let worker; + try { + worker = spawnWorker(cwd, job.id); + } catch (error) { + recordTaskWorkerSpawnFailure(job.workspaceRoot, job.id, error); + throw error; + } + worker?.once?.("error", (error) => { + recordTaskWorkerSpawnFailure(job.workspaceRoot, job.id, error); + }); return { payload: { diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index a8f3310a0..912d0097f 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -14,7 +14,7 @@ import { spawn } from "node:child_process"; import readline from "node:readline"; import { parseBrokerEndpoint } from "./broker-endpoint.mjs"; import { ensureBrokerSession, loadBrokerSession } from "./broker-lifecycle.mjs"; -import { captureProcessOwnership, normalizeProcessCleanupOutcome, terminateProcessTree } from "./process.mjs"; +import { captureProcessOwnership, normalizeProcessCleanupOutcome, terminateProcessGroup, terminateProcessTree } from "./process.mjs"; const PLUGIN_MANIFEST_URL = new URL("../../.claude-plugin/plugin.json", import.meta.url); const PLUGIN_MANIFEST = JSON.parse(fs.readFileSync(PLUGIN_MANIFEST_URL, "utf8")); @@ -221,6 +221,9 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { : createProtocolError( `codex app-server exited unexpectedly (${signal ? `signal ${signal}` : `exit ${code}`}).${stderr ? `\n${stderr}` : ""}` ); + if (!this.closed) { + this.startUnexpectedExitCleanup(this.proc.pid); + } this.handleExit(detail); }); @@ -254,6 +257,51 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { this.notify("initialized", {}); } + startUnexpectedExitCleanup(pid) { + if ( + this.unexpectedExitCleanupPromise || + process.platform === "win32" || + !Number.isFinite(pid) || + !this.ownershipSnapshot?.rootIdentity + ) { + return this.unexpectedExitCleanupPromise ?? null; + } + + this.unexpectedExitCleanupPromise = terminateProcessGroup(pid, { + ownershipSnapshot: this.ownershipSnapshot, + cwd: this.cwd, + env: this.options.env ?? process.env, + warnImpl: () => {} + }) + .then((outcome) => { + this.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); + if (!outcome.verified) { + process.stderr.write( + `Warning: unable to verify crashed codex app-server group cleanup; surviving PIDs: ${outcome.survivors?.join(", ") || "none known"}.\n` + ); + } + return this.cleanupOutcome; + }) + .catch((error) => { + this.cleanupOutcome = normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified: false, + degraded: true, + method: "process-group", + survivors: [], + survivorIdentities: [] + }); + process.stderr.write(`Warning: crashed codex app-server group cleanup failed: ${error.message}.\n`); + return this.cleanupOutcome; + }); + return this.unexpectedExitCleanupPromise; + } + + waitForUnexpectedExitCleanup() { + return this.unexpectedExitCleanupPromise ?? null; + } + async close() { if (this.closed) { await this.waitForExit(); @@ -293,19 +341,23 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { } else { // The app-server is its own process-group leader on Unix. Terminate // the group so MCP helpers cannot outlive the app-server parent. - const outcome = await terminateProcessTree(this.proc.pid, { - expectedRootIdentity: this.procIdentity, - ownershipSnapshot: this.ownershipSnapshot, - requireVerifiedOwnership: this.identityCaptureFailed, - ownerHoldsLiveHandle: true, - directKillImpl: (signal) => this.proc.kill(signal), - warnImpl: () => {} - }); - this.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); - if (!outcome.verified) { - process.stderr.write( - `Warning: unable to verify codex app-server cleanup; surviving PIDs: ${outcome.survivors?.join(", ") || "none known"}.\n` - ); + if (this.unexpectedExitCleanupPromise) { + await this.unexpectedExitCleanupPromise; + } else { + const outcome = await terminateProcessTree(this.proc.pid, { + expectedRootIdentity: this.procIdentity, + ownershipSnapshot: this.ownershipSnapshot, + requireVerifiedOwnership: this.identityCaptureFailed, + ownerHoldsLiveHandle: true, + directKillImpl: (signal) => this.proc.kill(signal), + warnImpl: () => {} + }); + this.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); + if (!outcome.verified) { + process.stderr.write( + `Warning: unable to verify codex app-server cleanup; surviving PIDs: ${outcome.survivors?.join(", ") || "none known"}.\n` + ); + } } } } diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index c54cc50f9..f64af4539 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -146,6 +146,96 @@ test("background task is persisted before its worker can start", () => { assert.equal(indexedJob.progressMarker, workerProgress.progressMarker); }); +test("background task marks the queued record failed when worker spawn throws", () => { + const workspace = makeTempDir(); + const job = { + id: "task-sync-spawn-failure", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Exercise synchronous spawn failure", + createdAt: "2026-07-30T03:00:00.000Z" + }; + const request = { + cwd: workspace, + prompt: "Exercise synchronous spawn failure", + jobId: job.id + }; + const spawnError = Object.assign(new Error("spawn EAGAIN"), { code: "EAGAIN" }); + + assert.throws( + () => + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl() { + throw spawnError; + } + }), + spawnError + ); + + const storedJob = readStoredJob(workspace, job.id); + assert.equal(storedJob.status, "failed"); + assert.equal(storedJob.phase, "failed"); + assert.equal(storedJob.pid, null); + assert.equal(storedJob.errorMessage, "spawn EAGAIN"); + assert.ok(storedJob.completedAt); + + const indexedJob = listJobs(workspace).find((candidate) => candidate.id === job.id); + assert.equal(indexedJob.status, "failed"); + assert.equal(indexedJob.phase, "failed"); + assert.equal(indexedJob.pid, null); + assert.equal(indexedJob.errorMessage, "spawn EAGAIN"); +}); + +test("background task marks the queued record failed when worker emits a spawn error", () => { + const workspace = makeTempDir(); + const job = { + id: "task-async-spawn-failure", + workspaceRoot: workspace, + kind: "task", + title: "Codex Task", + jobClass: "task", + summary: "Exercise asynchronous spawn failure", + createdAt: "2026-07-30T03:01:00.000Z" + }; + const request = { + cwd: workspace, + prompt: "Exercise asynchronous spawn failure", + jobId: job.id + }; + let spawnErrorHandler = null; + const worker = { + once(event, handler) { + if (event === "error") { + spawnErrorHandler = handler; + } + return this; + } + }; + + enqueueBackgroundTask(workspace, job, request, { + spawnDetachedTaskWorkerImpl() { + return worker; + } + }); + assert.equal(typeof spawnErrorHandler, "function"); + spawnErrorHandler(Object.assign(new Error("spawn EMFILE"), { code: "EMFILE" })); + + const storedJob = readStoredJob(workspace, job.id); + assert.equal(storedJob.status, "failed"); + assert.equal(storedJob.phase, "failed"); + assert.equal(storedJob.pid, null); + assert.equal(storedJob.errorMessage, "spawn EMFILE"); + assert.ok(storedJob.completedAt); + + const indexedJob = listJobs(workspace).find((candidate) => candidate.id === job.id); + assert.equal(indexedJob.status, "failed"); + assert.equal(indexedJob.phase, "failed"); + assert.equal(indexedJob.pid, null); + assert.equal(indexedJob.errorMessage, "spawn EMFILE"); +}); + test("background task stays runnable when worker identity capture fails", async () => { const workspace = makeTempDir(); const job = { @@ -334,6 +424,52 @@ test("shared broker reclaims a post-snapshot helper before allowing replacement" assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 2); }); +test("direct app-server reclaims a post-snapshot helper after its child crashes", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process groups are not available on Windows."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "crash-with-post-snapshot-helper"); + const env = buildEnv(binDir); + const client = await CodexAppServerClient.connect(repo, { env, disableBroker: true }); + let helperPid = null; + t.after(async () => { + await client.close().catch(() => {}); + if (Number.isFinite(helperPid)) { + try { + process.kill(helperPid, "SIGKILL"); + } catch { + // Ignore a helper already reclaimed after the app-server crash. + } + } + }); + + await waitFor(() => { + if (!fs.existsSync(fakeStatePath)) { + return false; + } + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + if (state.helperPids?.length !== 1 || state.appServerPids?.length !== 1) { + return false; + } + helperPid = state.helperPids[0]; + return true; + }); + await client.waitForExit(); + await waitFor(() => { + try { + process.kill(helperPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }, { timeoutMs: 3000 }); +}); + test("broker shutdown completes without starting a second app-server", async (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); From 8cec19f8fc1dd687524884a6ee2656b051af0a1c Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Wed, 29 Jul 2026 22:37:17 -0700 Subject: [PATCH 19/29] [CC-5219] register broker ownership for safe cleanup Attributed-To: Codex --- plugins/codex/scripts/app-server-broker.mjs | 71 +- plugins/codex/scripts/codex-companion.mjs | 3 +- .../codex/scripts/lib/broker-lifecycle.mjs | 31 +- .../codex/scripts/lib/broker-ownership.mjs | 811 ++++++++++++++++++ plugins/codex/scripts/lib/process.mjs | 53 +- .../scripts/registered-broker-reaper.mjs | 282 ++++++ .../codex/scripts/session-lifecycle-hook.mjs | 153 +++- tests/broker-ownership.test.mjs | 268 ++++++ tests/process.test.mjs | 190 +++- tests/registered-broker-reaper.test.mjs | 266 ++++++ tests/runtime.test.mjs | 208 ++++- 11 files changed, 2294 insertions(+), 42 deletions(-) create mode 100644 plugins/codex/scripts/lib/broker-ownership.mjs create mode 100644 plugins/codex/scripts/registered-broker-reaper.mjs create mode 100644 tests/broker-ownership.test.mjs create mode 100644 tests/registered-broker-reaper.test.mjs diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index c3d093410..742d8035d 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -8,8 +8,9 @@ import { fileURLToPath } from "node:url"; import { parseArgs } from "./lib/args.mjs"; import { BROKER_BUSY_RPC_CODE, CodexAppServerClient } from "./lib/app-server.mjs"; +import { loadBrokerRegistration, publishBrokerChild, releaseBrokerChild } from "./lib/broker-ownership.mjs"; import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs"; -import { getLiveProcessPids } from "./lib/process.mjs"; +import { getLiveProcessPids, getProcessIdentity } from "./lib/process.mjs"; const DEFAULT_CHILD_IDLE_MS = 5 * 60 * 1000; const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]); @@ -94,6 +95,7 @@ async function main() { const childIdleMs = resolveChildIdleMs(); let appClient = null; + let appClientRegistryChild = null; let appClientStartPromise = null; let appClientClosePromise = null; let childIdleTimer = null; @@ -110,8 +112,27 @@ async function main() { let activeStreamTurn = 0; let streamTurnCounter = 0; let blockedCleanup = null; + let brokerRegistration = null; const sockets = new Set(); + function getBrokerRegistration() { + if (brokerRegistration?.registered === true) { + return brokerRegistration; + } + try { + const brokerIdentity = getProcessIdentity(process.pid); + if (brokerIdentity) { + const candidate = loadBrokerRegistration({ endpoint, brokerIdentity }); + if (candidate.registered === true) { + brokerRegistration = candidate; + } + } + } catch { + // Missing registry evidence keeps the child outside automated cleanup. + } + return brokerRegistration; + } + function cancelChildIdleClose() { if (childIdleTimer) { clearTimeout(childIdleTimer); @@ -151,6 +172,24 @@ async function main() { ); } + function recordVerifiedChildRelease(client, child) { + const registration = getBrokerRegistration(); + if (registration?.registered !== true || !child || client.cleanupOutcome?.verified !== true) { + return; + } + try { + const released = releaseBrokerChild(registration, { + child, + cleanupOutcome: client.cleanupOutcome + }); + if (released.released !== true) { + process.stderr.write(`Warning: unable to release shared Codex app-server ownership (${released.reason ?? "unknown"}).\n`); + } + } catch (error) { + process.stderr.write(`Warning: unable to release shared Codex app-server ownership: ${error.message}.\n`); + } + } + async function closeAppClient() { cancelChildIdleClose(); clearStreamState(); @@ -162,7 +201,9 @@ async function main() { return; } const client = appClient; + const registryChild = appClientRegistryChild; appClient = null; + appClientRegistryChild = null; if (!client) { return; } @@ -170,6 +211,7 @@ async function main() { .close() .then(() => { recordUnverifiedCleanup(client); + recordVerifiedChildRelease(client, registryChild); }) .catch((error) => { blockedCleanup = { degraded: true, survivors: [] }; @@ -272,14 +314,38 @@ async function main() { } if (!appClientStartPromise) { appClientStartPromise = CodexAppServerClient.connect(cwd, { disableBroker: true }) - .then((client) => { + .then(async (client) => { + const registration = getBrokerRegistration(); + let childRegistration = null; + if (registration?.registered === true) { + try { + childRegistration = client.ownershipSnapshot + ? publishBrokerChild(registration, { ownershipSnapshot: client.ownershipSnapshot }) + : { registered: false, reason: "child-identity-unavailable" }; + } catch (error) { + await client.close().catch(() => {}); + throw error; + } + if (childRegistration.registered !== true) { + await client.close().catch(() => {}); + const error = new Error(`Unable to register shared Codex app-server ownership (${childRegistration.reason ?? "unknown"}).`); + error.rpcCode = BROKER_SHUTDOWN_RPC_CODE; + throw error; + } + } + let registryChild = null; appClient = client; + if (registration?.registered === true) { + registryChild = childRegistration.child; + appClientRegistryChild = registryChild; + } client.setNotificationHandler(routeNotification); const childPid = client.proc?.pid ?? null; client.setExitHandler(() => { clearStreamState(); if (appClient === client) { appClient = null; + appClientRegistryChild = null; } if (!client.closed && childPid != null && process.platform !== "win32") { // The child is a detached process-group leader; on an unexpected @@ -290,6 +356,7 @@ async function main() { .waitForUnexpectedExitCleanup() ?.then(() => { recordUnverifiedCleanup(client); + recordVerifiedChildRelease(client, registryChild); }) .catch((error) => { blockedCleanup = { degraded: true, survivors: [] }; diff --git a/plugins/codex/scripts/codex-companion.mjs b/plugins/codex/scripts/codex-companion.mjs index e41f80210..8a331b460 100644 --- a/plugins/codex/scripts/codex-companion.mjs +++ b/plugins/codex/scripts/codex-companion.mjs @@ -1097,7 +1097,8 @@ export async function handleCancel(argv, dependencies = {}) { const cleanupOutcome = await (dependencies.terminateProcessTreeImpl ?? terminateProcessTree)(record.pid, { expectedRootIdentity, ownershipSnapshot: null, - requireVerifiedOwnership: ownershipCaptureFailed + requireVerifiedOwnership: ownershipCaptureFailed, + priorCleanupDegraded: existing.cleanupOutcome?.degraded === true }); if (cleanupOutcome?.verified !== true) { const failureMessage = diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 9da687417..2af47066f 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -5,6 +5,7 @@ import path from "node:path"; import process from "node:process"; import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; +import { publishBrokerRegistration, registerBrokerOwner } from "./broker-ownership.mjs"; import { createBrokerEndpoint, parseBrokerEndpoint } from "./broker-endpoint.mjs"; import { captureProcessOwnership } from "./process.mjs"; import { resolveStateDir } from "./state.mjs"; @@ -114,6 +115,14 @@ async function isBrokerEndpointReady(endpoint) { export async function ensureBrokerSession(cwd, options = {}) { const existing = loadBrokerSession(cwd); if (existing && (await isBrokerEndpointReady(existing.endpoint))) { + if (existing.registry?.registered === true) { + const owner = registerBrokerOwner(existing.registry, { env: options.env ?? process.env }); + if (owner.registered !== true) { + const error = new Error(`Unable to register this session as a shared Codex broker owner (${owner.reason ?? "unknown"}).`); + error.code = "BROKER_OWNER_REGISTRATION_FAILED"; + throw error; + } + } return existing; } @@ -189,6 +198,25 @@ export async function ensureBrokerSession(cwd, options = {}) { return null; } + let registry = null; + try { + const candidate = publishBrokerRegistration({ + cwd, + endpoint, + pid: child.pid ?? null, + ownershipSnapshot, + env: options.env ?? process.env + }); + if (candidate.registered === true) { + const owner = registerBrokerOwner(candidate, { env: options.env ?? process.env }); + if (owner.registered === true) { + registry = candidate; + } + } + } catch (error) { + process.stderr.write(`Warning: unable to publish Codex broker ownership: ${error.message}. Broker remains unregistered.\n`); + } + const session = { endpoint, pidFile, @@ -197,7 +225,8 @@ export async function ensureBrokerSession(cwd, options = {}) { pid: child.pid ?? null, pidIdentity: ownershipSnapshot?.rootIdentity ?? null, ownershipSnapshot, - ownershipCaptureFailed + ownershipCaptureFailed, + registry }; saveBrokerSession(cwd, session); return session; diff --git a/plugins/codex/scripts/lib/broker-ownership.mjs b/plugins/codex/scripts/lib/broker-ownership.mjs new file mode 100644 index 000000000..c220420d4 --- /dev/null +++ b/plugins/codex/scripts/lib/broker-ownership.mjs @@ -0,0 +1,811 @@ +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; + +import { getLiveProcessPids } from "./process.mjs"; + +export const BROKER_OWNERSHIP_VERSION = 1; +export const SESSION_OWNER_PID_ENV = "CODEX_COMPANION_SESSION_OWNER_PID"; +export const SESSION_OWNER_IDENTITY_ENV = "CODEX_COMPANION_SESSION_OWNER_IDENTITY"; + +const PLUGIN_DATA_ENV = "CLAUDE_PLUGIN_DATA"; +const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; +const REGISTRY_DIR_NAME = "broker-ownership-v1"; +const REGISTRY_LOCK_DIR_NAME = "registry.lock"; + +function sha256(value) { + return createHash("sha256").update(String(value)).digest("hex"); +} + +function isSafePid(value) { + return Number.isSafeInteger(value) && value > 1; +} + +function isPidIdentity(pid, identity) { + return isSafePid(pid) && typeof identity === "string" && identity.startsWith(`${pid}@`) && identity.length > String(pid).length + 1; +} + +function requirePrivateDirectory(dirPath) { + const stat = fs.lstatSync(dirPath); + if (stat.isSymbolicLink() || !stat.isDirectory() || (stat.mode & 0o777) !== 0o700) { + const error = new Error(`Broker ownership registry directory is not private: ${dirPath}.`); + error.code = "BROKER_OWNERSHIP_PERMISSIONS"; + throw error; + } + return stat; +} + +function requirePrivateFile(filePath) { + const stat = fs.lstatSync(filePath); + if (stat.isSymbolicLink() || !stat.isFile() || (stat.mode & 0o777) !== 0o600) { + const error = new Error(`Broker ownership registry file is not private: ${filePath}.`); + error.code = "BROKER_OWNERSHIP_PERMISSIONS"; + throw error; + } + return stat; +} + +function ensurePrivateDir(dirPath) { + fs.mkdirSync(dirPath, { recursive: true, mode: 0o700 }); + const stat = fs.lstatSync(dirPath); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + const error = new Error(`Refusing non-directory broker ownership path: ${dirPath}.`); + error.code = "BROKER_OWNERSHIP_PERMISSIONS"; + throw error; + } + fs.chmodSync(dirPath, 0o700); + requirePrivateDirectory(dirPath); +} + +function immutableJsonBytes(payload) { + return `${JSON.stringify(payload, null, 2)}\n`; +} + +function createImmutableJson(filePath, payload) { + const bytes = immutableJsonBytes(payload); + ensurePrivateDir(path.dirname(filePath)); + if (fs.existsSync(filePath)) { + requirePrivateFile(filePath); + if (fs.readFileSync(filePath, "utf8") !== bytes) { + const error = new Error(`Broker ownership registry collision at ${filePath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + fs.chmodSync(filePath, 0o600); + return filePath; + } + + const tempPath = path.join(path.dirname(filePath), `.${path.basename(filePath)}.${process.pid}.${randomUUID()}.tmp`); + try { + fs.writeFileSync(tempPath, bytes, { encoding: "utf8", flag: "wx", mode: 0o600 }); + fs.chmodSync(tempPath, 0o600); + try { + fs.linkSync(tempPath, filePath); + } catch (error) { + if (error?.code !== "EEXIST") { + throw error; + } + requirePrivateFile(filePath); + if (fs.readFileSync(filePath, "utf8") !== bytes) { + throw error; + } + } + fs.chmodSync(filePath, 0o600); + requirePrivateFile(filePath); + return filePath; + } finally { + try { + fs.unlinkSync(tempPath); + } catch (error) { + if (error?.code !== "ENOENT") { + throw error; + } + } + } +} + +function readJson(filePath) { + requirePrivateFile(filePath); + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function registrationReference(registryRoot, brokerKey) { + return { + registered: true, + version: BROKER_OWNERSHIP_VERSION, + brokerKey, + registryRoot, + registryDir: path.join(registryRoot, brokerKey) + }; +} + +function validRegistrationReference(registration) { + return Boolean( + registration?.registered === true && + registration.version === BROKER_OWNERSHIP_VERSION && + typeof registration.brokerKey === "string" && + /^[a-f0-9]{64}$/.test(registration.brokerKey) && + typeof registration.registryRoot === "string" && + path.isAbsolute(registration.registryRoot) && + registration.registryDir === path.join(registration.registryRoot, registration.brokerKey) + ); +} + +function validRegistryLock(registration, lock) { + return Boolean( + validRegistrationReference(registration) && + lock?.acquired === true && + lock.brokerKey === registration.brokerKey && + typeof lock.token === "string" && + lock.token.length > 0 && + lock.path === path.join(registration.registryDir, REGISTRY_LOCK_DIR_NAME) + ); +} + +export function acquireBrokerRegistryLock( + registration, + { + now = () => new Date().toISOString(), + pid = process.pid, + getLiveProcessPidsImpl = getLiveProcessPids + } = {} +) { + if (!validRegistrationReference(registration)) { + return { acquired: false, reason: "broker-registration-unavailable" }; + } + requirePrivateDirectory(registration.registryRoot); + requirePrivateDirectory(registration.registryDir); + const lockPath = path.join(registration.registryDir, REGISTRY_LOCK_DIR_NAME); + const token = randomUUID(); + for (let attempt = 0; attempt < 3; attempt += 1) { + const preparedPath = path.join(registration.registryDir, `.registry-lock.${pid}.${randomUUID()}`); + try { + ensurePrivateDir(preparedPath); + createImmutableJson(path.join(preparedPath, "owner.json"), { + version: BROKER_OWNERSHIP_VERSION, + kind: "registry-lock", + brokerKey: registration.brokerKey, + token, + pid, + acquiredAt: now() + }); + try { + fs.renameSync(preparedPath, lockPath); + return { acquired: true, brokerKey: registration.brokerKey, token, path: lockPath }; + } catch (error) { + if (error?.code !== "EEXIST" && error?.code !== "ENOTEMPTY") { + throw error; + } + } + } finally { + if (fs.existsSync(preparedPath)) { + fs.rmSync(preparedPath, { recursive: true, force: true }); + } + } + + let existingOwner; + try { + requirePrivateDirectory(lockPath); + existingOwner = readJson(path.join(lockPath, "owner.json")); + } catch (error) { + if (error?.code === "ENOENT") { + continue; + } + return { acquired: false, reason: "registry-lock-malformed", path: lockPath }; + } + if ( + existingOwner?.version !== BROKER_OWNERSHIP_VERSION || + existingOwner.kind !== "registry-lock" || + existingOwner.brokerKey !== registration.brokerKey || + typeof existingOwner.token !== "string" || + !isSafePid(existingOwner.pid) + ) { + return { acquired: false, reason: "registry-lock-malformed", path: lockPath }; + } + let live; + try { + live = getLiveProcessPidsImpl([existingOwner.pid]); + } catch { + return { acquired: false, reason: "registry-lock-liveness-unavailable", path: lockPath }; + } + if (!Array.isArray(live) || live.includes(existingOwner.pid)) { + return { acquired: false, reason: "registry-busy", path: lockPath }; + } + + const staleRoot = path.join(registration.registryDir, "stale-locks"); + ensurePrivateDir(staleRoot); + try { + fs.renameSync(lockPath, path.join(staleRoot, `${existingOwner.pid}-${randomUUID()}`)); + } catch (error) { + if (error?.code !== "ENOENT") { + return { acquired: false, reason: "registry-lock-quarantine-failed", path: lockPath }; + } + } + } + return { acquired: false, reason: "registry-lock-contention", path: lockPath }; +} + +export function releaseBrokerRegistryLock(registration, lock) { + if (!validRegistryLock(registration, lock)) { + return { released: false, reason: "registry-lock-unavailable" }; + } + const ownerPath = path.join(lock.path, "owner.json"); + try { + const owner = readJson(ownerPath); + if ( + owner?.version !== BROKER_OWNERSHIP_VERSION || + owner.kind !== "registry-lock" || + owner.brokerKey !== registration.brokerKey || + owner.token !== lock.token + ) { + return { released: false, reason: "registry-lock-mismatch" }; + } + fs.unlinkSync(ownerPath); + fs.rmdirSync(lock.path); + return { released: true }; + } catch (error) { + if (error?.code === "ENOENT") { + return { released: false, reason: "registry-lock-missing" }; + } + throw error; + } +} + +function withBrokerRegistryLock(registration, options, action) { + const suppliedLock = options.registryLock; + if (suppliedLock && !validRegistryLock(registration, suppliedLock)) { + return { ok: false, reason: "registry-lock-unavailable" }; + } + const lock = suppliedLock ?? acquireBrokerRegistryLock(registration, options); + if (lock.acquired !== true) { + return { ok: false, reason: lock.reason ?? "registry-busy" }; + } + try { + return { ok: true, value: action() }; + } finally { + if (!suppliedLock) { + const released = releaseBrokerRegistryLock(registration, lock); + if (released.released !== true) { + const error = new Error(`Unable to release broker registry lock (${released.reason}).`); + error.code = "BROKER_REGISTRY_LOCK_RELEASE_FAILED"; + throw error; + } + } + } +} + +export function resolveBrokerOwnershipRoot(env = process.env) { + const pluginDataDir = env?.[PLUGIN_DATA_ENV]; + if (typeof pluginDataDir !== "string" || !path.isAbsolute(pluginDataDir)) { + return null; + } + return path.join(pluginDataDir, "state", REGISTRY_DIR_NAME); +} + +export function brokerOwnershipKey(endpoint, brokerIdentity) { + return sha256(`${endpoint}\0${brokerIdentity}`); +} + +export function loadBrokerRegistration({ endpoint, brokerIdentity, env = process.env }) { + const registryRoot = resolveBrokerOwnershipRoot(env); + if (!registryRoot || typeof endpoint !== "string" || !endpoint || typeof brokerIdentity !== "string" || !brokerIdentity) { + return { registered: false, reason: "missing-registration-identity" }; + } + const registration = registrationReference(registryRoot, brokerOwnershipKey(endpoint, brokerIdentity)); + const brokerPath = path.join(registration.registryDir, "broker.json"); + if (!fs.existsSync(brokerPath)) { + return { registered: false, reason: "broker-record-absent" }; + } + try { + requirePrivateDirectory(registryRoot); + requirePrivateDirectory(registration.registryDir); + const broker = readJson(brokerPath); + if ( + broker?.version !== BROKER_OWNERSHIP_VERSION || + broker.kind !== "broker" || + broker.brokerKey !== registration.brokerKey || + broker.endpoint !== endpoint || + broker.pidIdentity !== brokerIdentity || + !isPidIdentity(broker.pid, broker.pidIdentity) + ) { + return { registered: false, reason: "broker-record-invalid" }; + } + return { ...registration, broker }; + } catch { + return { registered: false, reason: "broker-record-invalid" }; + } +} + +export function publishBrokerRegistration({ cwd, endpoint, pid, ownershipSnapshot, env = process.env, now = () => new Date().toISOString() }) { + const registryRoot = resolveBrokerOwnershipRoot(env); + if (!registryRoot) { + return { registered: false, reason: "plugin-data-unavailable" }; + } + if ( + typeof endpoint !== "string" || + !endpoint || + !isSafePid(pid) || + ownershipSnapshot?.rootPid !== pid || + !isPidIdentity(pid, ownershipSnapshot?.rootIdentity) || + !isSafePid(ownershipSnapshot?.processGroupId) + ) { + return { registered: false, reason: "broker-identity-unavailable" }; + } + + const brokerKey = brokerOwnershipKey(endpoint, ownershipSnapshot.rootIdentity); + const registration = registrationReference(registryRoot, brokerKey); + const existing = loadBrokerRegistration({ endpoint, brokerIdentity: ownershipSnapshot.rootIdentity, env }); + if (existing.registered === true) { + return existing; + } + const broker = { + version: BROKER_OWNERSHIP_VERSION, + kind: "broker", + brokerKey, + endpoint, + pid, + pidIdentity: ownershipSnapshot.rootIdentity, + processGroupId: ownershipSnapshot.processGroupId, + workspaceHash: sha256(path.resolve(cwd)), + createdAt: now() + }; + createImmutableJson(path.join(registration.registryDir, "broker.json"), broker); + return { ...registration, broker }; +} + +function ownerFromEnv(env) { + const sessionId = env?.[SESSION_ID_ENV]; + const pid = Number(env?.[SESSION_OWNER_PID_ENV]); + const identity = env?.[SESSION_OWNER_IDENTITY_ENV]; + if (typeof sessionId !== "string" || !sessionId || !isPidIdentity(pid, identity)) { + return null; + } + const ownerKey = sha256(`${sessionId}\0${identity}`); + return { ownerKey, sessionId, pid, identity }; +} + +export function registerBrokerOwner(registration, options = {}) { + const { env = process.env, now = () => new Date().toISOString() } = options; + if (!validRegistrationReference(registration)) { + return { registered: false, reason: "broker-registration-unavailable" }; + } + const owner = ownerFromEnv(env); + if (!owner) { + return { registered: false, reason: "session-owner-unavailable" }; + } + const locked = withBrokerRegistryLock(registration, options, () => { + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "owner", + brokerKey: registration.brokerKey, + ownerKey: owner.ownerKey, + sessionId: owner.sessionId, + pid: owner.pid, + pidIdentity: owner.identity, + registeredAt: now() + }; + const ownerPath = path.join(registration.registryDir, "owners", `${owner.ownerKey}.json`); + if (fs.existsSync(ownerPath)) { + const existing = readJson(ownerPath); + if (!validOwnerRecord(existing, ownerPath, registration)) { + const error = new Error(`Invalid existing broker owner row at ${ownerPath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + fs.chmodSync(ownerPath, 0o600); + return { registered: true, ownerKey: owner.ownerKey, path: ownerPath, owner: existing }; + } + createImmutableJson(ownerPath, payload); + return { registered: true, ownerKey: owner.ownerKey, path: ownerPath, owner: payload }; + }); + return locked.ok ? locked.value : { registered: false, reason: locked.reason }; +} + +export function releaseBrokerOwner(registration, options = {}) { + const { env = process.env, now = () => new Date().toISOString() } = options; + if (!validRegistrationReference(registration)) { + return { released: false, reason: "broker-registration-unavailable" }; + } + const owner = ownerFromEnv(env); + if (!owner) { + return { released: false, reason: "session-owner-unavailable" }; + } + const locked = withBrokerRegistryLock(registration, options, () => { + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "release", + brokerKey: registration.brokerKey, + ownerKey: owner.ownerKey, + sessionId: owner.sessionId, + pid: owner.pid, + pidIdentity: owner.identity, + releasedAt: now() + }; + const releasePath = path.join(registration.registryDir, "releases", `${owner.ownerKey}.json`); + if (fs.existsSync(releasePath)) { + const existing = readJson(releasePath); + if (!validReleaseRecord(existing, releasePath, registration)) { + const error = new Error(`Invalid existing broker owner release at ${releasePath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + fs.chmodSync(releasePath, 0o600); + return { released: true, ownerKey: owner.ownerKey, path: releasePath, release: existing }; + } + createImmutableJson(releasePath, payload); + return { released: true, ownerKey: owner.ownerKey, path: releasePath, release: payload }; + }); + return locked.ok ? locked.value : { released: false, reason: locked.reason }; +} + +function listJsonFiles(dirPath) { + try { + requirePrivateDirectory(dirPath); + } catch (error) { + if (error?.code === "ENOENT") { + return []; + } + throw error; + } + return fs.readdirSync(dirPath) + .filter((name) => name.endsWith(".json") && !name.startsWith(".")) + .sort() + .map((name) => path.join(dirPath, name)); +} + +function validOwnerRecord(record, filePath, registration) { + return Boolean( + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "owner" && + record.brokerKey === registration.brokerKey && + typeof record.ownerKey === "string" && + path.basename(filePath) === `${record.ownerKey}.json` && + record.ownerKey === sha256(`${record.sessionId}\0${record.pidIdentity}`) && + isPidIdentity(record.pid, record.pidIdentity) && + typeof record.sessionId === "string" && + record.sessionId.length > 0 + ); +} + +function validReleaseRecord(record, filePath, registration) { + return Boolean( + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "release" && + record.brokerKey === registration.brokerKey && + typeof record.ownerKey === "string" && + path.basename(filePath) === `${record.ownerKey}.json` && + record.ownerKey === sha256(`${record.sessionId}\0${record.pidIdentity}`) && + isPidIdentity(record.pid, record.pidIdentity) && + typeof record.sessionId === "string" && + record.sessionId.length > 0 + ); +} + +function validSnapshotMember(record) { + return Boolean( + isSafePid(record?.pid) && + Number.isSafeInteger(record.parentPid) && + record.parentPid >= 0 && + isSafePid(record.processGroupId) && + typeof record.state === "string" && + record.state.length > 0 && + typeof record.startedAt === "string" && + record.startedAt.length > 0 && + isPidIdentity(record.pid, record.identity) && + Number.isSafeInteger(record.depth) && + record.depth >= 0 + ); +} + +function validChildRecord(record, filePath, registration) { + const snapshot = record?.ownershipSnapshot; + return Boolean( + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "child" && + record.brokerKey === registration.brokerKey && + typeof record.childKey === "string" && + record.childKey === sha256(record.pidIdentity) && + path.basename(filePath) === `${record.childKey}.json` && + isPidIdentity(record.pid, record.pidIdentity) && + isSafePid(record.processGroupId) && + snapshot?.rootPid === record.pid && + snapshot.rootIdentity === record.pidIdentity && + snapshot.processGroupId === record.processGroupId && + Array.isArray(snapshot.members) && + snapshot.members.length > 0 && + snapshot.members.every(validSnapshotMember) && + snapshot.members.some((member) => member.pid === record.pid && member.identity === record.pidIdentity) + ); +} + +function validChildReleaseRecord(record, filePath, registration, child) { + return Boolean( + child && + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "child-release" && + record.brokerKey === registration.brokerKey && + record.childKey === child.childKey && + path.basename(filePath) === `${record.childKey}.json` && + record.pid === child.pid && + record.pidIdentity === child.pidIdentity && + record.cleanupVerified === true + ); +} + +export function loadBrokerChildren(registration) { + if (!validRegistrationReference(registration)) { + return { valid: false, reason: "broker-registration-unavailable", children: [], malformed: [] }; + } + const children = []; + const releases = new Map(); + const malformed = []; + let childFiles; + let childReleaseFiles; + try { + childFiles = listJsonFiles(path.join(registration.registryDir, "children")); + childReleaseFiles = listJsonFiles(path.join(registration.registryDir, "child-releases")); + } catch { + return { + valid: false, + reason: "malformed-child-registry", + children: [], + releasedChildren: [], + malformed: [registration.registryDir] + }; + } + for (const filePath of childFiles) { + try { + const record = readJson(filePath); + if (!validChildRecord(record, filePath, registration)) { + malformed.push(filePath); + } else { + children.push(record); + } + } catch { + malformed.push(filePath); + } + } + const childrenByKey = new Map(children.map((child) => [child.childKey, child])); + for (const filePath of childReleaseFiles) { + try { + const record = readJson(filePath); + if (!validChildReleaseRecord(record, filePath, registration, childrenByKey.get(record?.childKey))) { + malformed.push(filePath); + } else { + releases.set(record.childKey, record); + } + } catch { + malformed.push(filePath); + } + } + return malformed.length > 0 + ? { valid: false, reason: "malformed-child-registry", children: [], releasedChildren: [], malformed } + : { + valid: true, + reason: "valid-child-registry", + children: children.filter((child) => !releases.has(child.childKey)), + releasedChildren: children.filter((child) => releases.has(child.childKey)), + malformed: [] + }; +} + +export function releaseBrokerChild( + registration, + { child, cleanupOutcome, now = () => new Date().toISOString() } = {} +) { + if (!validRegistrationReference(registration)) { + return { released: false, reason: "broker-registration-unavailable" }; + } + const childPath = path.join(registration.registryDir, "children", `${child?.childKey}.json`); + if ( + !validChildRecord(child, childPath, registration) || + cleanupOutcome?.verified !== true || + (cleanupOutcome.survivors?.length ?? 0) > 0 || + (cleanupOutcome.survivorIdentities?.length ?? 0) > 0 + ) { + return { released: false, reason: "child-cleanup-unverified" }; + } + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "child-release", + brokerKey: registration.brokerKey, + childKey: child.childKey, + pid: child.pid, + pidIdentity: child.pidIdentity, + cleanupVerified: true, + releasedAt: now() + }; + const releasePath = path.join(registration.registryDir, "child-releases", `${child.childKey}.json`); + if (fs.existsSync(releasePath)) { + const existing = readJson(releasePath); + if (!validChildReleaseRecord(existing, releasePath, registration, child)) { + const error = new Error(`Invalid existing broker child release at ${releasePath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + fs.chmodSync(releasePath, 0o600); + return { released: true, path: releasePath, release: existing }; + } + createImmutableJson(releasePath, payload); + return { released: true, path: releasePath, release: payload }; +} + +export function publishBrokerReaperReceipt( + registration, + { attemptId, decision, outcomes, residualIdentities, createdAt = new Date().toISOString() } +) { + if (!validRegistrationReference(registration)) { + return { published: false, reason: "broker-registration-unavailable" }; + } + if ( + typeof attemptId !== "string" || + !/^[a-zA-Z0-9._-]{1,128}$/.test(attemptId) || + typeof decision !== "string" || + !Array.isArray(outcomes) || + !Array.isArray(residualIdentities) + ) { + return { published: false, reason: "receipt-invalid" }; + } + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "reaper-receipt", + brokerKey: registration.brokerKey, + attemptId, + decision, + outcomes, + residualIdentities, + createdAt + }; + const receiptPath = path.join(registration.registryDir, "receipts", `${attemptId}.json`); + createImmutableJson(receiptPath, payload); + return { published: true, path: receiptPath, receipt: payload }; +} + +export function assessBrokerOwners(registration, { getLiveProcessPidsImpl = getLiveProcessPids } = {}) { + if (!validRegistrationReference(registration)) { + return { safeToShutdown: false, reason: "broker-registration-unavailable", liveOwners: [], deadOwners: [], releasedOwners: [], malformed: [] }; + } + + const malformed = []; + const owners = []; + const releases = new Map(); + let ownerFiles; + let releaseFiles; + try { + ownerFiles = listJsonFiles(path.join(registration.registryDir, "owners")); + releaseFiles = listJsonFiles(path.join(registration.registryDir, "releases")); + } catch { + return { + safeToShutdown: false, + reason: "malformed-registry", + liveOwners: [], + deadOwners: [], + releasedOwners: [], + malformed: [registration.registryDir] + }; + } + for (const filePath of ownerFiles) { + try { + const record = readJson(filePath); + if (!validOwnerRecord(record, filePath, registration)) { + malformed.push(filePath); + } else { + owners.push(record); + } + } catch { + malformed.push(filePath); + } + } + for (const filePath of releaseFiles) { + try { + const record = readJson(filePath); + if (!validReleaseRecord(record, filePath, registration)) { + malformed.push(filePath); + } else { + releases.set(record.ownerKey, record); + } + } catch { + malformed.push(filePath); + } + } + if (malformed.length > 0) { + return { safeToShutdown: false, reason: "malformed-registry", liveOwners: [], deadOwners: [], releasedOwners: [], malformed }; + } + if (owners.length === 0) { + return { safeToShutdown: false, reason: "no-registered-owner", liveOwners: [], deadOwners: [], releasedOwners: [], malformed: [] }; + } + + const liveOwners = []; + const deadOwners = []; + const releasedOwners = []; + for (const owner of owners) { + const release = releases.get(owner.ownerKey); + if (release) { + if ( + release.sessionId !== owner.sessionId || + release.pid !== owner.pid || + release.pidIdentity !== owner.pidIdentity + ) { + malformed.push(path.join(registration.registryDir, "releases", `${owner.ownerKey}.json`)); + continue; + } + releasedOwners.push(owner); + continue; + } + let live; + try { + live = getLiveProcessPidsImpl([owner.pid], { identities: [owner.pidIdentity] }); + } catch { + malformed.push(`liveness:${owner.ownerKey}`); + continue; + } + if (Array.isArray(live) && live.includes(owner.pid)) { + liveOwners.push(owner); + } else { + deadOwners.push(owner); + } + } + if (malformed.length > 0) { + return { safeToShutdown: false, reason: "owner-liveness-unavailable", liveOwners, deadOwners, releasedOwners, malformed }; + } + if (liveOwners.length > 0) { + return { safeToShutdown: false, reason: "live-owner", liveOwners, deadOwners, releasedOwners, malformed: [] }; + } + return { + safeToShutdown: true, + reason: "all-owners-dead-or-released", + liveOwners, + deadOwners, + releasedOwners, + malformed: [] + }; +} + +export function publishBrokerChild(registration, options = {}) { + const { ownershipSnapshot, now = () => new Date().toISOString() } = options; + if (!validRegistrationReference(registration)) { + return { registered: false, reason: "broker-registration-unavailable" }; + } + const pid = ownershipSnapshot?.rootPid; + const identity = ownershipSnapshot?.rootIdentity; + if (!isPidIdentity(pid, identity) || !isSafePid(ownershipSnapshot?.processGroupId) || !Array.isArray(ownershipSnapshot?.members)) { + return { registered: false, reason: "child-identity-unavailable" }; + } + const locked = withBrokerRegistryLock(registration, options, () => { + const childKey = sha256(identity); + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "child", + brokerKey: registration.brokerKey, + childKey, + pid, + pidIdentity: identity, + processGroupId: ownershipSnapshot.processGroupId, + ownershipSnapshot, + registeredAt: now() + }; + const childPath = path.join(registration.registryDir, "children", `${childKey}.json`); + if (fs.existsSync(childPath)) { + const existing = readJson(childPath); + if ( + existing?.version !== BROKER_OWNERSHIP_VERSION || + existing.kind !== "child" || + existing.brokerKey !== registration.brokerKey || + existing.childKey !== childKey || + existing.pid !== pid || + existing.pidIdentity !== identity || + JSON.stringify(existing.ownershipSnapshot) !== JSON.stringify(ownershipSnapshot) + ) { + const error = new Error(`Invalid existing broker child row at ${childPath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + fs.chmodSync(childPath, 0o600); + return { registered: true, childKey, path: childPath, child: existing }; + } + createImmutableJson(childPath, payload); + return { registered: true, childKey, path: childPath, child: payload }; + }); + return locked.ok ? locked.value : { registered: false, reason: locked.reason }; +} diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 525100ecf..9ea63e155 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -161,6 +161,26 @@ export function captureProcessOwnership(pid, options = {}) { }; } +export function captureStableSessionOwner(pid = process.pid, options = {}) { + if (!Number.isFinite(pid) || (options.platform ?? process.platform) === "win32") { + return null; + } + const processes = readUnixProcessTable(options.runCommandImpl ?? runCommand, options); + const current = processes.get(pid); + if (!isRunningProcess(current) || current.processGroupId === pid) { + return null; + } + const leader = processes.get(current.processGroupId); + if (!isRunningProcess(leader) || leader.pid !== leader.processGroupId) { + return null; + } + return { + pid: leader.pid, + identity: leader.identity, + processGroupId: leader.processGroupId + }; +} + function recordsFromOwnershipSnapshot(snapshot) { return (snapshot?.members ?? []).filter((record) => { return Number.isFinite(record.pid) && typeof record.identity === "string" && record.identity.length > 0; @@ -338,9 +358,15 @@ function signalVerifiedUnit(unit, signal, processes, killImpl) { killImpl(target, signal); return true; } catch (error) { - if (error?.code === "ESRCH") { + if (error?.code === "ESRCH" || error?.code === "EPERM") { + // On macOS a process-group signal can race the group leader's exit and + // return EPERM. Never infer success from the errno; the caller's fresh + // process-table poll decides whether cleanup is verified. return false; } + if (error instanceof Error) { + error.message = `Unable to signal owned process ${unit.record.identity} with ${signal} via target ${target}: ${error.message}`; + } throw error; } } @@ -390,15 +416,23 @@ function warnProcessCleanup(message, options) { } function degradedDirectChildKill(pid, options, killImpl, reason) { + const ownershipSnapshot = options.ownershipSnapshot ?? null; + const canSignalOwnedGroup = + options.ownerHoldsLiveHandle === true && + ownershipSnapshot?.rootPid === pid && + ownershipSnapshot?.processGroupId === pid && + ownershipSnapshot?.rootIdentity === (options.expectedRootIdentity ?? ownershipSnapshot?.rootIdentity); const directKillImpl = options.directKillImpl ?? ((signal) => killImpl(pid, signal)); let delivered = false; try { - delivered = directKillImpl("SIGKILL") !== false; + delivered = canSignalOwnedGroup + ? killImpl(-pid, "SIGKILL") !== false + : directKillImpl("SIGKILL") !== false; } catch { // The direct child may already have exited. } warnProcessCleanup( - `Unable to verify Unix process cleanup for PID ${pid}; used direct-child kill fallback (${String(reason).replace(/\s+/g, " ").trim()}). Surviving PIDs: none known.`, + `Unable to verify Unix process cleanup for PID ${pid}; used ${canSignalOwnedGroup ? "process-group" : "direct-child"} kill fallback (${String(reason).replace(/\s+/g, " ").trim()}). Surviving PIDs: none known.`, options ); return normalizeProcessCleanupOutcome({ @@ -407,9 +441,13 @@ function degradedDirectChildKill(pid, options, killImpl, reason) { verified: false, escalated: false, degraded: true, - method: "direct-child", - targets: [pid], - targetIdentities: options.expectedRootIdentity ? [options.expectedRootIdentity] : [], + method: canSignalOwnedGroup ? "process-group" : "direct-child", + targets: canSignalOwnedGroup + ? recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.pid) + : [pid], + targetIdentities: canSignalOwnedGroup + ? recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.identity) + : options.expectedRootIdentity ? [options.expectedRootIdentity] : [], survivors: [], survivorIdentities: [] }); @@ -531,7 +569,8 @@ export async function terminateProcessTree(pid, options = {}) { const sortedTracked = () => [...tracked.values()].sort((left, right) => right.depth - left.depth); if (!root && listLiveTracked(tracked, initialProcesses).length === 0) { - const verified = !options.requireVerifiedOwnership && ownershipEstablished; + const degradedWithoutDurableSnapshot = options.priorCleanupDegraded === true && recordsFromOwnershipSnapshot(ownershipSnapshot).length === 0; + const verified = !options.requireVerifiedOwnership && ownershipEstablished && !degradedWithoutDurableSnapshot; return normalizeProcessCleanupOutcome({ attempted: true, delivered: false, diff --git a/plugins/codex/scripts/registered-broker-reaper.mjs b/plugins/codex/scripts/registered-broker-reaper.mjs new file mode 100644 index 000000000..12a3413f2 --- /dev/null +++ b/plugins/codex/scripts/registered-broker-reaper.mjs @@ -0,0 +1,282 @@ +#!/usr/bin/env node + +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; + +import { + acquireBrokerRegistryLock, + assessBrokerOwners, + loadBrokerChildren, + loadBrokerRegistration, + publishBrokerReaperReceipt, + releaseBrokerRegistryLock, + resolveBrokerOwnershipRoot +} from "./lib/broker-ownership.mjs"; +import { + getLiveProcessPids, + terminateProcessGroup, + terminateProcessTree +} from "./lib/process.mjs"; + +const APPLY_MODE = "apply-registered"; +const REPORT_MODE = "report-only"; + +function listRegistrationCandidates(env) { + const registryRoot = resolveBrokerOwnershipRoot(env); + if (!registryRoot || !fs.existsSync(registryRoot)) { + return []; + } + try { + const rootStat = fs.lstatSync(registryRoot); + if (rootStat.isSymbolicLink() || !rootStat.isDirectory() || (rootStat.mode & 0o777) !== 0o700) { + return [{ valid: false, brokerKey: null, registryDir: registryRoot, reason: "registry-root-invalid" }]; + } + } catch { + return [{ valid: false, brokerKey: null, registryDir: registryRoot, reason: "registry-root-invalid" }]; + } + const candidates = []; + for (const entry of fs.readdirSync(registryRoot, { withFileTypes: true })) { + if (!/^[a-f0-9]{64}$/.test(entry.name)) { + continue; + } + const registryDir = path.join(registryRoot, entry.name); + if (!entry.isDirectory()) { + candidates.push({ valid: false, brokerKey: entry.name, registryDir, reason: "broker-directory-invalid" }); + continue; + } + const brokerPath = path.join(registryDir, "broker.json"); + try { + const brokerStat = fs.lstatSync(brokerPath); + if (brokerStat.isSymbolicLink() || !brokerStat.isFile() || (brokerStat.mode & 0o777) !== 0o600) { + throw new Error("broker record is not a private regular file"); + } + const broker = JSON.parse(fs.readFileSync(brokerPath, "utf8")); + const registration = loadBrokerRegistration({ + endpoint: broker?.endpoint, + brokerIdentity: broker?.pidIdentity, + env + }); + if ( + registration.registered !== true || + registration.brokerKey !== entry.name || + registration.registryDir !== registryDir + ) { + candidates.push({ valid: false, brokerKey: entry.name, registryDir, reason: "broker-record-invalid" }); + } else { + candidates.push({ valid: true, registration }); + } + } catch { + candidates.push({ valid: false, brokerKey: entry.name, registryDir, reason: "broker-record-invalid" }); + } + } + return candidates; +} + +function targetRecords(registration, children) { + const broker = registration.broker; + const byIdentity = new Map([[broker.pidIdentity, { pid: broker.pid, identity: broker.pidIdentity }]]); + for (const child of children) { + for (const member of child.ownershipSnapshot.members) { + byIdentity.set(member.identity, { pid: member.pid, identity: member.identity }); + } + } + return [...byIdentity.values()]; +} + +function brokerOwnershipSnapshot(broker) { + const startedAt = broker.pidIdentity.slice(String(broker.pid).length + 1); + return { + rootPid: broker.pid, + rootIdentity: broker.pidIdentity, + processGroupId: broker.processGroupId, + members: [ + { + pid: broker.pid, + parentPid: 1, + processGroupId: broker.processGroupId, + state: "registered", + startedAt, + identity: broker.pidIdentity, + depth: 0 + } + ] + }; +} + +function refusal(candidate, reason, extra = {}) { + return { + status: "report-only", + brokerKey: candidate.registration?.brokerKey ?? candidate.brokerKey ?? null, + pid: candidate.registration?.broker?.pid ?? null, + reason, + ...extra + }; +} + +async function processRegistration(candidate, options) { + if (!candidate.valid) { + return refusal(candidate, candidate.reason); + } + + const registration = candidate.registration; + const getLive = options.getLiveProcessPidsImpl ?? getLiveProcessPids; + const assess = options.assessBrokerOwnersImpl ?? assessBrokerOwners; + const initialAssessment = assess(registration, { getLiveProcessPidsImpl: getLive }); + if (initialAssessment.safeToShutdown !== true) { + return refusal(candidate, initialAssessment.reason, { assessment: initialAssessment }); + } + const initialChildren = loadBrokerChildren(registration); + if (initialChildren.valid !== true) { + return refusal(candidate, initialChildren.reason, { malformed: initialChildren.malformed }); + } + if (options.mode !== APPLY_MODE) { + return refusal(candidate, "eligible-but-apply-not-enabled", { assessment: initialAssessment }); + } + + const acquireLock = options.acquireBrokerRegistryLockImpl ?? acquireBrokerRegistryLock; + const releaseLock = options.releaseBrokerRegistryLockImpl ?? releaseBrokerRegistryLock; + const registryLock = acquireLock(registration); + if (registryLock?.acquired !== true) { + return refusal(candidate, registryLock?.reason ?? "registry-busy"); + } + + const outcomes = []; + let decision = "cleanup-unverified"; + let residualIdentities = []; + let result; + try { + const lockedRegistration = loadBrokerRegistration({ + endpoint: registration.broker.endpoint, + brokerIdentity: registration.broker.pidIdentity, + env: options.env ?? process.env + }); + if ( + lockedRegistration.registered !== true || + lockedRegistration.brokerKey !== registration.brokerKey || + lockedRegistration.registryDir !== registration.registryDir + ) { + result = refusal(candidate, "locked-broker-record-invalid"); + } else { + const finalAssessment = assess(lockedRegistration, { getLiveProcessPidsImpl: getLive }); + if (finalAssessment.safeToShutdown !== true) { + result = refusal(candidate, `locked-${finalAssessment.reason}`, { assessment: finalAssessment }); + } else { + const children = loadBrokerChildren(lockedRegistration); + if (children.valid !== true) { + result = refusal(candidate, children.reason, { malformed: children.malformed }); + } else { + const terminateGroup = options.terminateProcessGroupImpl ?? terminateProcessGroup; + const terminateTree = options.terminateProcessTreeImpl ?? terminateProcessTree; + let cleanupVerified = true; + for (const child of children.children) { + const outcome = await terminateGroup(child.processGroupId, { + ownershipSnapshot: child.ownershipSnapshot, + termPollAttempts: options.termPollAttempts, + killPollAttempts: options.killPollAttempts, + pollIntervalMs: options.pollIntervalMs + }); + outcomes.push({ target: "child", pid: child.pid, pidIdentity: child.pidIdentity, outcome }); + if (outcome?.verified !== true) { + cleanupVerified = false; + break; + } + } + + if (cleanupVerified) { + const broker = lockedRegistration.broker; + const outcome = await terminateTree(broker.pid, { + expectedRootIdentity: broker.pidIdentity, + ownershipSnapshot: brokerOwnershipSnapshot(broker), + termPollAttempts: options.termPollAttempts, + killPollAttempts: options.killPollAttempts, + pollIntervalMs: options.pollIntervalMs + }); + outcomes.push({ target: "broker", pid: broker.pid, pidIdentity: broker.pidIdentity, outcome }); + cleanupVerified = outcome?.verified === true; + } + + const targets = targetRecords(lockedRegistration, children.children); + const residualPids = getLive( + targets.map((target) => target.pid), + { identities: targets.map((target) => target.identity) } + ); + const residualSet = new Set(residualPids); + residualIdentities = targets + .filter((target) => residualSet.has(target.pid)) + .map((target) => target.identity); + if (cleanupVerified && residualIdentities.length === 0) { + decision = "cleanup-verified"; + } + + const attemptId = (options.attemptIdFactory ?? (() => randomUUID()))(); + const receipt = (options.publishBrokerReaperReceiptImpl ?? publishBrokerReaperReceipt)(lockedRegistration, { + attemptId, + decision, + outcomes, + residualIdentities, + createdAt: (options.now ?? (() => new Date().toISOString()))() + }); + result = { + status: decision === "cleanup-verified" ? "reaped" : "report-only", + brokerKey: lockedRegistration.brokerKey, + pid: lockedRegistration.broker.pid, + reason: receipt?.published === true ? decision : `${decision}-receipt-unavailable`, + outcomes, + residualIdentities, + receiptPath: receipt?.published === true ? receipt.path : null + }; + } + } + } + } finally { + const released = releaseLock(registration, registryLock); + if (released?.released !== true) { + if (result) { + result.status = "report-only"; + result.reason = `registry-lock-release-${released?.reason ?? "failed"}`; + } else { + result = refusal(candidate, `registry-lock-release-${released?.reason ?? "failed"}`); + } + } + } + return result; +} + +export async function runRegisteredBrokerReaper(options = {}) { + const mode = options.mode === APPLY_MODE ? APPLY_MODE : REPORT_MODE; + const candidates = listRegistrationCandidates(options.env ?? process.env); + const results = []; + for (const candidate of candidates) { + results.push(await processRegistration(candidate, { ...options, mode })); + } + return { + mode, + scanned: candidates.length, + reaped: results.filter((result) => result.status === "reaped").length, + reported: results.filter((result) => result.status !== "reaped").length, + results + }; +} + +async function main() { + const mode = process.argv.slice(2).includes("--apply-registered") ? APPLY_MODE : REPORT_MODE; + const summary = await runRegisteredBrokerReaper({ mode }); + for (const result of summary.results) { + process.stdout.write( + `${result.status.toUpperCase()} broker_key=${result.brokerKey ?? "unknown"} pid=${result.pid ?? "unknown"} reason=${result.reason}\n` + ); + } + process.stdout.write( + `scanned=${summary.scanned} reaped=${summary.reaped} reported=${summary.reported} mode=${summary.mode}\n` + ); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 5e54ce9e1..b0d6877a2 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -4,8 +4,17 @@ import fs from "node:fs"; import process from "node:process"; import { pathToFileURL } from "node:url"; -import { terminateProcessTree } from "./lib/process.mjs"; +import { captureStableSessionOwner, terminateProcessTree } from "./lib/process.mjs"; import { BROKER_ENDPOINT_ENV } from "./lib/app-server.mjs"; +import { + acquireBrokerRegistryLock, + assessBrokerOwners, + loadBrokerRegistration, + releaseBrokerOwner, + releaseBrokerRegistryLock, + SESSION_OWNER_IDENTITY_ENV, + SESSION_OWNER_PID_ENV +} from "./lib/broker-ownership.mjs"; import { clearBrokerSession, LOG_FILE_ENV, @@ -76,7 +85,8 @@ export async function cleanupSessionJobs(cwd, sessionId, dependencies = {}) { const outcome = await terminate(job.pid ?? Number.NaN, { expectedRootIdentity, ownershipSnapshot: null, - requireVerifiedOwnership: ownershipCaptureFailed + requireVerifiedOwnership: ownershipCaptureFailed, + priorCleanupDegraded: job.cleanupOutcome?.degraded === true }); if (outcome?.verified === true) { continue; @@ -126,16 +136,34 @@ export async function cleanupSessionJobs(cwd, sessionId, dependencies = {}) { return { verified: retainedJobs.length === 0, failures: [] }; } -function handleSessionStart(input) { +export function handleSessionStart(input, dependencies = {}) { appendEnvVar(SESSION_ID_ENV, input.session_id); appendEnvVar(TRANSCRIPT_PATH_ENV, input.transcript_path); appendEnvVar(PLUGIN_DATA_ENV, process.env[PLUGIN_DATA_ENV]); + if ((dependencies.platform ?? process.platform) === "win32") { + return; + } + try { + const captureOwner = dependencies.captureStableSessionOwnerImpl ?? captureStableSessionOwner; + const owner = captureOwner(dependencies.pid ?? process.pid, { + cwd: input.cwd || process.cwd(), + env: process.env, + platform: dependencies.platform + }); + if (owner?.pid && owner.identity) { + appendEnvVar(SESSION_OWNER_PID_ENV, owner.pid); + appendEnvVar(SESSION_OWNER_IDENTITY_ENV, owner.identity); + } + } catch { + // Missing owner identity leaves this session in the report-only class. + } } -async function handleSessionEnd(input) { +export async function handleSessionEnd(input, dependencies = {}) { const cwd = input.cwd || process.cwd(); + const loadBroker = dependencies.loadBrokerSessionImpl ?? loadBrokerSession; const brokerSession = - loadBrokerSession(cwd) ?? + loadBroker(cwd) ?? (process.env[BROKER_ENDPOINT_ENV] ? { endpoint: process.env[BROKER_ENDPOINT_ENV], @@ -151,25 +179,101 @@ async function handleSessionEnd(input) { const pidIdentity = brokerSession?.pidIdentity ?? null; const ownershipSnapshot = brokerSession?.ownershipSnapshot ?? null; const requireVerifiedOwnership = brokerSession?.ownershipCaptureFailed === true; + const registry = brokerSession?.registry?.registered === true ? brokerSession.registry : null; + let registryAssessment = null; + let registryLock = null; - if (brokerEndpoint) { - await sendBrokerShutdown(brokerEndpoint); - } - - const jobCleanup = await cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); - const brokerCleanup = await teardownBrokerSession({ - endpoint: brokerEndpoint, - pidFile, - logFile, - sessionDir, - pid, - pidIdentity, - ownershipSnapshot, - requireVerifiedOwnership, - killProcess: terminateProcessTree - }); - if (brokerCleanup?.verified === true) { - clearBrokerSession(cwd); + if (registry) { + const acquireRegistryLock = dependencies.acquireBrokerRegistryLockImpl ?? acquireBrokerRegistryLock; + registryLock = acquireRegistryLock(registry); + if (registryLock?.acquired === true) { + const loadRegistration = dependencies.loadBrokerRegistrationImpl ?? loadBrokerRegistration; + const lockedRegistration = loadRegistration({ + endpoint: brokerEndpoint, + brokerIdentity: pidIdentity, + env: dependencies.env ?? process.env + }); + if ( + lockedRegistration?.registered !== true || + lockedRegistration.brokerKey !== registry.brokerKey || + lockedRegistration.registryDir !== registry.registryDir + ) { + registryAssessment = { + safeToShutdown: false, + reason: "broker-record-invalid", + liveOwners: [], + deadOwners: [], + releasedOwners: [], + malformed: [registry.registryDir] + }; + } else { + const releaseOwner = dependencies.releaseBrokerOwnerImpl ?? releaseBrokerOwner; + const assessOwners = dependencies.assessBrokerOwnersImpl ?? assessBrokerOwners; + const ownerRelease = releaseOwner(lockedRegistration, { + env: dependencies.env ?? process.env, + registryLock + }); + registryAssessment = ownerRelease?.released === true + ? assessOwners(lockedRegistration) + : { + safeToShutdown: false, + reason: `owner-release-${ownerRelease?.reason ?? "failed"}`, + liveOwners: [], + deadOwners: [], + releasedOwners: [], + malformed: [] + }; + } + } else { + registryAssessment = { + safeToShutdown: false, + reason: registryLock?.reason ?? "registry-busy", + liveOwners: [], + deadOwners: [], + releasedOwners: [], + malformed: [] + }; + } + } + + const brokerShutdownAllowed = !registry || registryAssessment?.safeToShutdown === true; + let brokerCleanup = { verified: true }; + let registryLockReleaseFailure = null; + try { + if (brokerEndpoint && brokerShutdownAllowed) { + await (dependencies.sendBrokerShutdownImpl ?? sendBrokerShutdown)(brokerEndpoint); + } + if (brokerShutdownAllowed) { + const teardownBroker = dependencies.teardownBrokerSessionImpl ?? teardownBrokerSession; + brokerCleanup = await teardownBroker({ + endpoint: brokerEndpoint, + pidFile, + logFile, + sessionDir, + pid, + pidIdentity, + ownershipSnapshot, + requireVerifiedOwnership, + killProcess: dependencies.terminateProcessTreeImpl ?? terminateProcessTree + }); + if (brokerCleanup?.verified === true) { + (dependencies.clearBrokerSessionImpl ?? clearBrokerSession)(cwd); + } + } + } finally { + if (registry && registryLock?.acquired === true) { + const releaseRegistryLock = dependencies.releaseBrokerRegistryLockImpl ?? releaseBrokerRegistryLock; + const released = releaseRegistryLock(registry, registryLock); + if (released?.released !== true) { + registryLockReleaseFailure = released?.reason ?? "unknown"; + } + } + } + + const cleanupJobs = dependencies.cleanupSessionJobsImpl ?? cleanupSessionJobs; + const jobCleanup = await cleanupJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); + if (registryLockReleaseFailure) { + throw new Error(`Broker registry lock release failed (${registryLockReleaseFailure}).`); } if (jobCleanup?.verified !== true) { throw new Error("Session cleanup remains pending because process termination could not be verified."); @@ -177,6 +281,9 @@ async function handleSessionEnd(input) { if (brokerCleanup?.verified !== true) { throw new Error("Broker cleanup remains pending because process termination could not be verified."); } + if (registry && !brokerShutdownAllowed && registryAssessment?.reason !== "live-owner") { + throw new Error(`Broker cleanup remains report-only because registered ownership is ambiguous (${registryAssessment?.reason ?? "unknown"}).`); + } } async function main() { diff --git a/tests/broker-ownership.test.mjs b/tests/broker-ownership.test.mjs new file mode 100644 index 000000000..da253b549 --- /dev/null +++ b/tests/broker-ownership.test.mjs @@ -0,0 +1,268 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + acquireBrokerRegistryLock, + assessBrokerOwners, + loadBrokerChildren, + publishBrokerChild, + publishBrokerRegistration, + registerBrokerOwner, + releaseBrokerChild, + releaseBrokerOwner, + releaseBrokerRegistryLock +} from "../plugins/codex/scripts/lib/broker-ownership.mjs"; + +function makeFixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-ownership-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = { CLAUDE_PLUGIN_DATA: root }; + const brokerIdentity = "4100@Mon Jul 27 00:00:00 2026"; + const ownershipSnapshot = { + rootPid: 4100, + rootIdentity: brokerIdentity, + processGroupId: 4100, + members: [ + { + pid: 4100, + parentPid: 1, + processGroupId: 4100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: brokerIdentity, + depth: 0 + } + ] + }; + const registration = publishBrokerRegistration({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot, + env, + now: () => "2026-07-27T00:00:00.000Z" + }); + assert.equal(registration.registered, true); + return { root, env, registration }; +} + +function ownerEnv(env, sessionId, pid, startedAt) { + return { + ...env, + CODEX_COMPANION_SESSION_ID: sessionId, + CODEX_COMPANION_SESSION_OWNER_PID: String(pid), + CODEX_COMPANION_SESSION_OWNER_IDENTITY: `${pid}@${startedAt}` + }; +} + +test("owner publication fails closed while cleanup holds the registry lock", (t) => { + const { registration, env } = makeFixture(t); + const lock = acquireBrokerRegistryLock(registration, { + pid: 5000, + now: () => "2026-07-27T00:00:30.000Z" + }); + assert.equal(lock.acquired, true); + assert.equal(fs.statSync(lock.path).mode & 0o777, 0o700); + + const blocked = registerBrokerOwner( + registration, + { + env: ownerEnv(env, "session-racing", 5050, "Mon Jul 27 00:00:45 2026"), + getLiveProcessPidsImpl: () => [5000] + } + ); + assert.deepEqual(blocked, { registered: false, reason: "registry-busy" }); + + assert.deepEqual(releaseBrokerRegistryLock(registration, lock), { released: true }); + const registered = registerBrokerOwner( + registration, + { env: ownerEnv(env, "session-racing", 5050, "Mon Jul 27 00:00:45 2026") } + ); + assert.equal(registered.registered, true); +}); + +test("a well-formed lock whose creator is absent is quarantined before retry", (t) => { + const { registration, env } = makeFixture(t); + const stale = acquireBrokerRegistryLock(registration, { + pid: 5001, + now: () => "2026-07-27T00:00:31.000Z" + }); + assert.equal(stale.acquired, true); + + const registered = registerBrokerOwner(registration, { + env: ownerEnv(env, "session-after-stale-lock", 5051, "Mon Jul 27 00:00:46 2026"), + getLiveProcessPidsImpl: () => [] + }); + assert.equal(registered.registered, true); + const staleRoot = path.join(registration.registryDir, "stale-locks"); + assert.equal(fs.readdirSync(staleRoot).length, 1); + assert.equal(fs.existsSync(stale.path), false); +}); + +test("registered broker with a live owner is not eligible for cleanup", (t) => { + const { registration, env } = makeFixture(t); + const liveOwner = ownerEnv(env, "session-live", 5100, "Mon Jul 27 00:01:00 2026"); + const owner = registerBrokerOwner(registration, { env: liveOwner, now: () => "2026-07-27T00:01:00.000Z" }); + + const assessment = assessBrokerOwners(registration, { + getLiveProcessPidsImpl(pids, options) { + assert.deepEqual(pids, [5100]); + assert.deepEqual(options.identities, ["5100@Mon Jul 27 00:01:00 2026"]); + return [5100]; + } + }); + + assert.equal(owner.registered, true); + assert.equal(assessment.safeToShutdown, false); + assert.equal(assessment.reason, "live-owner"); + assert.deepEqual(assessment.liveOwners.map((candidate) => candidate.sessionId), ["session-live"]); + assert.equal(fs.statSync(owner.path).mode & 0o777, 0o600); +}); + +test("registered broker is eligible only after every owner is dead or released", (t) => { + const { registration, env } = makeFixture(t); + const ownerA = ownerEnv(env, "session-a", 5200, "Mon Jul 27 00:02:00 2026"); + const ownerB = ownerEnv(env, "session-b", 5300, "Mon Jul 27 00:03:00 2026"); + registerBrokerOwner(registration, { env: ownerA }); + registerBrokerOwner(registration, { env: ownerB }); + + const mixed = assessBrokerOwners(registration, { + getLiveProcessPidsImpl(pids) { + return pids[0] === 5300 ? [5300] : []; + } + }); + assert.equal(mixed.safeToShutdown, false); + assert.deepEqual(mixed.liveOwners.map((candidate) => candidate.sessionId), ["session-b"]); + + releaseBrokerOwner(registration, { env: ownerB, now: () => "2026-07-27T00:04:00.000Z" }); + const released = assessBrokerOwners(registration, { + getLiveProcessPidsImpl() { + return []; + } + }); + assert.equal(released.safeToShutdown, true); + assert.equal(released.reason, "all-owners-dead-or-released"); + assert.deepEqual(released.releasedOwners.map((candidate) => candidate.sessionId), ["session-b"]); +}); + +test("owner PID reuse with a different identity does not keep a broker alive", (t) => { + const { registration, env } = makeFixture(t); + registerBrokerOwner(registration, { + env: ownerEnv(env, "session-reused", 5400, "Mon Jul 27 00:05:00 2026") + }); + + const assessment = assessBrokerOwners(registration, { + getLiveProcessPidsImpl() { + return []; + } + }); + assert.equal(assessment.safeToShutdown, true); + assert.deepEqual(assessment.deadOwners.map((candidate) => candidate.sessionId), ["session-reused"]); +}); + +test("malformed owner state blocks cleanup instead of being skipped", (t) => { + const { registration } = makeFixture(t); + const ownersDir = path.join(registration.registryDir, "owners"); + fs.mkdirSync(ownersDir, { recursive: true }); + fs.writeFileSync(path.join(ownersDir, "malformed.json"), "{not-json\n", { mode: 0o600 }); + + const assessment = assessBrokerOwners(registration, { + getLiveProcessPidsImpl() { + throw new Error("malformed rows must stop before liveness checks"); + } + }); + assert.equal(assessment.safeToShutdown, false); + assert.equal(assessment.reason, "malformed-registry"); + assert.equal(assessment.malformed.length, 1); +}); + +test("symlinked or overly permissive registry rows block cleanup", (t) => { + const permissiveFixture = makeFixture(t); + const registered = registerBrokerOwner(permissiveFixture.registration, { + env: ownerEnv(permissiveFixture.env, "session-permissive", 5450, "Mon Jul 27 00:05:30 2026") + }); + fs.chmodSync(registered.path, 0o644); + const permissive = assessBrokerOwners(permissiveFixture.registration, { + getLiveProcessPidsImpl: () => [] + }); + assert.equal(permissive.safeToShutdown, false); + assert.equal(permissive.reason, "malformed-registry"); + + const symlinkFixture = makeFixture(t); + const ownersDir = path.join(symlinkFixture.registration.registryDir, "owners"); + fs.mkdirSync(ownersDir, { recursive: true, mode: 0o700 }); + fs.chmodSync(ownersDir, 0o700); + const target = path.join(symlinkFixture.root, "outside.json"); + fs.writeFileSync(target, "{}\n", { mode: 0o600 }); + fs.symlinkSync(target, path.join(ownersDir, "linked.json")); + const symlinked = assessBrokerOwners(symlinkFixture.registration, { + getLiveProcessPidsImpl: () => [] + }); + assert.equal(symlinked.safeToShutdown, false); + assert.equal(symlinked.reason, "malformed-registry"); +}); + +test("a broker record with no owner rows remains unregistered and report-only", (t) => { + const { registration } = makeFixture(t); + const assessment = assessBrokerOwners(registration, { + getLiveProcessPidsImpl() { + throw new Error("an ownerless broker must not reach liveness checks"); + } + }); + assert.equal(assessment.safeToShutdown, false); + assert.equal(assessment.reason, "no-registered-owner"); +}); + +test("broker child ownership is immutable and identity keyed", (t) => { + const { registration } = makeFixture(t); + const childSnapshot = { + rootPid: 6100, + rootIdentity: "6100@Mon Jul 27 00:06:00 2026", + processGroupId: 6100, + members: [ + { + pid: 6100, + parentPid: 4100, + processGroupId: 6100, + state: "S", + startedAt: "Mon Jul 27 00:06:00 2026", + identity: "6100@Mon Jul 27 00:06:00 2026", + depth: 0 + } + ] + }; + const first = publishBrokerChild(registration, { + ownershipSnapshot: childSnapshot, + now: () => "2026-07-27T00:06:00.000Z" + }); + const second = publishBrokerChild(registration, { + ownershipSnapshot: childSnapshot, + now: () => "2026-07-27T00:06:00.000Z" + }); + + assert.equal(first.registered, true); + assert.equal(second.path, first.path); + assert.equal(fs.statSync(first.path).mode & 0o777, 0o600); + assert.deepEqual(JSON.parse(fs.readFileSync(first.path, "utf8")).ownershipSnapshot, childSnapshot); + + const refused = releaseBrokerChild(registration, { + child: first.child, + cleanupOutcome: { verified: false, survivors: [6100] } + }); + assert.equal(refused.released, false); + assert.equal(loadBrokerChildren(registration).children.length, 1); + + const released = releaseBrokerChild(registration, { + child: first.child, + cleanupOutcome: { verified: true, survivors: [], survivorIdentities: [] }, + now: () => "2026-07-27T00:07:00.000Z" + }); + assert.equal(released.released, true); + assert.equal(fs.statSync(released.path).mode & 0o777, 0o600); + const children = loadBrokerChildren(registration); + assert.equal(children.children.length, 0); + assert.equal(children.releasedChildren.length, 1); +}); diff --git a/tests/process.test.mjs b/tests/process.test.mjs index bd9014e68..cfc494044 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -1,7 +1,52 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { getLiveProcessPids, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; +import { captureStableSessionOwner, getLiveProcessPids, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; + +test("captureStableSessionOwner records the hook process-group leader", () => { + const owner = captureStableSessionOwner(7101, { + platform: "darwin", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: [ + "7100 1 7100 S Mon Jul 27 00:07:00 2026", + "7101 7100 7100 S Mon Jul 27 00:07:01 2026" + ].join("\n"), + stderr: "", + error: null + }; + } + }); + + assert.deepEqual(owner, { + pid: 7100, + identity: "7100@Mon Jul 27 00:07:00 2026", + processGroupId: 7100 + }); +}); + +test("captureStableSessionOwner refuses a hook that is its own process-group leader", () => { + const owner = captureStableSessionOwner(7200, { + platform: "darwin", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "7200 1 7200 S Mon Jul 27 00:08:00 2026\n", + stderr: "", + error: null + }; + } + }); + + assert.equal(owner, null); +}); test("terminateProcessTree uses taskkill on Windows", async () => { let captured = null; @@ -143,6 +188,75 @@ test("terminateProcessTree signals a Unix PID directly when it is not a group le assert.equal(outcome.method, "process-tree"); }); +test("terminateProcessTree verifies an EPERM group-signal race only after the target disappears", async () => { + let scans = 0; + const identity = "100@Mon Jul 27 00:00:00 2026"; + const outcome = await terminateProcessTree(100, { + platform: "darwin", + expectedRootIdentity: identity, + ownershipSnapshot: { + rootPid: 100, + rootIdentity: identity, + processGroupId: 100, + members: [{ pid: 100, parentPid: 1, processGroupId: 100, state: "S", startedAt: "Mon Jul 27 00:00:00 2026", identity, depth: 0 }] + }, + runCommandImpl() { + scans += 1; + return { + status: 0, + stdout: scans <= 2 ? "100 1 100 S Mon Jul 27 00:00:00 2026\n" : "", + stderr: "", + error: null + }; + }, + killImpl() { + const error = new Error("kill EPERM"); + error.code = "EPERM"; + throw error; + }, + pollIntervalMs: 0, + termPollAttempts: 1 + }); + + assert.equal(outcome.delivered, false); + assert.equal(outcome.verified, true); + assert.deepEqual(outcome.survivors, []); +}); + +test("terminateProcessTree keeps an EPERM group-signal survivor unverified", async () => { + const identity = "100@Mon Jul 27 00:00:00 2026"; + const outcome = await terminateProcessTree(100, { + platform: "darwin", + expectedRootIdentity: identity, + ownershipSnapshot: { + rootPid: 100, + rootIdentity: identity, + processGroupId: 100, + members: [{ pid: 100, parentPid: 1, processGroupId: 100, state: "S", startedAt: "Mon Jul 27 00:00:00 2026", identity, depth: 0 }] + }, + runCommandImpl() { + return { + status: 0, + stdout: "100 1 100 S Mon Jul 27 00:00:00 2026\n", + stderr: "", + error: null + }; + }, + killImpl() { + const error = new Error("kill EPERM"); + error.code = "EPERM"; + throw error; + }, + pollIntervalMs: 0, + termPollAttempts: 1, + killPollAttempts: 1 + }); + + assert.equal(outcome.delivered, false); + assert.equal(outcome.verified, false); + assert.deepEqual(outcome.survivorIdentities, [identity]); +}); + test("terminateProcessTree parses a captured Linux procps process table", async () => { const signals = []; const alive = new Set([42001, 42002]); @@ -216,6 +330,80 @@ test("terminateProcessTree falls back to a direct child kill when Unix process e assert.match(warnings[0], /direct-child kill fallback.*none known/i); }); +test("terminateProcessTree preserves an owned detached group during degraded live-handle cleanup", async () => { + const signals = []; + const warnings = []; + const ownershipSnapshot = { + rootPid: 1234, + rootIdentity: "1234@Mon Jul 27 00:00:00 2026", + processGroupId: 1234, + members: [ + { + pid: 1234, + parentPid: 1, + processGroupId: 1234, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "1234@Mon Jul 27 00:00:00 2026", + depth: 0 + } + ] + }; + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + ownershipSnapshot, + ownerHoldsLiveHandle: true, + warnImpl(message) { + warnings.push(message); + }, + runCommandImpl(command, args) { + return { + command, + args, + status: 1, + signal: null, + stdout: "", + stderr: "ps denied", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, [[-1234, "SIGKILL"]]); + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); + assert.equal(outcome.method, "process-group"); + assert.match(warnings[0], /process-group kill fallback/i); +}); + +test("terminateProcessTree does not convert a degraded root-only cleanup into later verified success", async () => { + const outcome = await terminateProcessTree(1234, { + platform: "darwin", + expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", + priorCleanupDegraded: true, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "", + stderr: "", + error: null + }; + }, + killImpl() { + throw new Error("an absent root must not be signaled"); + } + }); + + assert.equal(outcome.verified, false); + assert.equal(outcome.degraded, true); +}); + test("terminateProcessTree refuses a reused root PID", async () => { const signals = []; const outcome = await terminateProcessTree(1234, { diff --git a/tests/registered-broker-reaper.test.mjs b/tests/registered-broker-reaper.test.mjs new file mode 100644 index 000000000..1899d73c0 --- /dev/null +++ b/tests/registered-broker-reaper.test.mjs @@ -0,0 +1,266 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + acquireBrokerRegistryLock, + publishBrokerChild, + publishBrokerRegistration, + registerBrokerOwner, + releaseBrokerOwner, + releaseBrokerRegistryLock +} from "../plugins/codex/scripts/lib/broker-ownership.mjs"; +import { runRegisteredBrokerReaper } from "../plugins/codex/scripts/registered-broker-reaper.mjs"; + +function ownerEnv(env, sessionId, pid) { + const startedAt = `Mon Jul 27 00:${String(pid % 60).padStart(2, "0")}:00 2026`; + return { + ...env, + CODEX_COMPANION_SESSION_ID: sessionId, + CODEX_COMPANION_SESSION_OWNER_PID: String(pid), + CODEX_COMPANION_SESSION_OWNER_IDENTITY: `${pid}@${startedAt}` + }; +} + +function snapshot(pid, minute) { + const startedAt = `Mon Jul 27 01:${String(minute).padStart(2, "0")}:00 2026`; + const identity = `${pid}@${startedAt}`; + return { + rootPid: pid, + rootIdentity: identity, + processGroupId: pid, + members: [ + { + pid, + parentPid: 1, + processGroupId: pid, + state: "S", + startedAt, + identity, + depth: 0 + } + ] + }; +} + +function makeFixture(t, { child = false } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "registered-broker-reaper-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = { CLAUDE_PLUGIN_DATA: root }; + const brokerSnapshot = snapshot(4100, 1); + const registration = publishBrokerRegistration({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: brokerSnapshot.rootPid, + ownershipSnapshot: brokerSnapshot, + env, + now: () => "2026-07-27T01:01:00.000Z" + }); + assert.equal(registration.registered, true); + let childRecord = null; + if (child) { + childRecord = publishBrokerChild(registration, { + ownershipSnapshot: snapshot(4200, 2), + now: () => "2026-07-27T01:02:00.000Z" + }).child; + } + return { root, env, registration, childRecord }; +} + +function addReleasedOwner(registration, env, sessionId = "session-released", pid = 5100) { + const sessionEnv = ownerEnv(env, sessionId, pid); + assert.equal(registerBrokerOwner(registration, { env: sessionEnv }).registered, true); + assert.equal(releaseBrokerOwner(registration, { env: sessionEnv }).released, true); +} + +test("report-only mode never calls a cleanup function for an eligible registry", async (t) => { + const { env, registration } = makeFixture(t); + addReleasedOwner(registration, env); + let cleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + env, + terminateProcessGroupImpl: async () => { cleanupCalls += 1; }, + terminateProcessTreeImpl: async () => { cleanupCalls += 1; } + }); + assert.equal(summary.mode, "report-only"); + assert.equal(summary.reaped, 0); + assert.equal(summary.results[0].reason, "eligible-but-apply-not-enabled"); + assert.equal(cleanupCalls, 0); +}); + +test("one live owner blocks cleanup even when every other owner is dead", async (t) => { + const { env, registration } = makeFixture(t); + const live = ownerEnv(env, "session-live", 5200); + const dead = ownerEnv(env, "session-dead", 5300); + registerBrokerOwner(registration, { env: live }); + registerBrokerOwner(registration, { env: dead }); + let cleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl(pids, options) { + return pids.filter((pid) => pid === 5200 && options.identities.some((identity) => identity.startsWith("5200@"))); + }, + terminateProcessTreeImpl: async () => { cleanupCalls += 1; } + }); + assert.equal(summary.results[0].reason, "live-owner"); + assert.equal(cleanupCalls, 0); +}); + +test("registered cleanup reclaims children before the broker and writes a mode-0600 receipt", async (t) => { + const { env, registration, childRecord } = makeFixture(t, { child: true }); + addReleasedOwner(registration, env); + const calls = []; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessGroupImpl: async (pgid, options) => { + calls.push(`group:${pgid}:${options.ownershipSnapshot.rootIdentity}`); + return { attempted: true, delivered: true, verified: true, degraded: false, targetIdentities: [childRecord.pidIdentity] }; + }, + terminateProcessTreeImpl: async (pid, options) => { + calls.push(`tree:${pid}:${options.expectedRootIdentity}`); + return { attempted: true, delivered: true, verified: true, degraded: false, targetIdentities: [options.expectedRootIdentity] }; + }, + attemptIdFactory: () => "attempt-success", + now: () => "2026-07-27T02:00:00.000Z" + }); + assert.equal(summary.reaped, 1); + assert.deepEqual(calls, [ + `group:4200:${childRecord.pidIdentity}`, + `tree:4100:${registration.broker.pidIdentity}` + ]); + const receiptPath = summary.results[0].receiptPath; + assert.ok(receiptPath); + assert.equal(fs.statSync(receiptPath).mode & 0o777, 0o600); + assert.equal(JSON.parse(fs.readFileSync(receiptPath, "utf8")).decision, "cleanup-verified"); +}); + +test("a contended registry lock closes the registration-to-signal race", async (t) => { + const { env, registration } = makeFixture(t); + addReleasedOwner(registration, env); + const lock = acquireBrokerRegistryLock(registration); + t.after(() => releaseBrokerRegistryLock(registration, lock)); + let cleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessTreeImpl: async () => { cleanupCalls += 1; } + }); + assert.equal(summary.results[0].reason, "registry-busy"); + assert.equal(cleanupCalls, 0); +}); + +test("the broker record is revalidated after the cleanup lock is acquired", async (t) => { + const { env, registration } = makeFixture(t); + addReleasedOwner(registration, env); + let cleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + acquireBrokerRegistryLockImpl(candidate) { + const lock = acquireBrokerRegistryLock(candidate); + fs.unlinkSync(path.join(candidate.registryDir, "broker.json")); + return lock; + }, + terminateProcessTreeImpl: async () => { cleanupCalls += 1; } + }); + assert.equal(summary.results[0].reason, "locked-broker-record-invalid"); + assert.equal(cleanupCalls, 0); +}); + +test("a reused or ambiguous child group blocks broker cleanup", async (t) => { + const { env, registration } = makeFixture(t, { child: true }); + addReleasedOwner(registration, env); + let brokerCleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: (pids) => pids, + terminateProcessGroupImpl: async () => ({ + attempted: false, + delivered: false, + verified: false, + degraded: true, + identityMismatch: true + }), + terminateProcessTreeImpl: async () => { brokerCleanupCalls += 1; }, + attemptIdFactory: () => "attempt-child-reused" + }); + assert.equal(summary.reaped, 0); + assert.equal(summary.results[0].reason, "cleanup-unverified"); + assert.equal(brokerCleanupCalls, 0); + assert.ok(summary.results[0].residualIdentities.length > 0); +}); + +test("malformed owner or child state is report-only", async (t) => { + const ownerFixture = makeFixture(t); + fs.mkdirSync(path.join(ownerFixture.registration.registryDir, "owners"), { recursive: true }); + fs.writeFileSync(path.join(ownerFixture.registration.registryDir, "owners", "bad.json"), "{}\n", "utf8"); + const ownerSummary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env: ownerFixture.env, + getLiveProcessPidsImpl: () => [] + }); + assert.equal(ownerSummary.results[0].reason, "malformed-registry"); + + const childFixture = makeFixture(t); + addReleasedOwner(childFixture.registration, childFixture.env); + fs.mkdirSync(path.join(childFixture.registration.registryDir, "children"), { recursive: true }); + fs.writeFileSync(path.join(childFixture.registration.registryDir, "children", "bad.json"), "{}\n", "utf8"); + const childSummary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env: childFixture.env, + getLiveProcessPidsImpl: () => [] + }); + assert.equal(childSummary.results[0].reason, "malformed-child-registry"); +}); + +test("missing and unregistered broker rows never reach a signal path", async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "registered-broker-reaper-missing-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = { CLAUDE_PLUGIN_DATA: root }; + const registryRoot = path.join(root, "state", "broker-ownership-v1"); + const missingBrokerDir = path.join(registryRoot, "a".repeat(64)); + fs.mkdirSync(missingBrokerDir, { recursive: true, mode: 0o700 }); + fs.chmodSync(registryRoot, 0o700); + fs.chmodSync(missingBrokerDir, 0o700); + let cleanupCalls = 0; + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + terminateProcessTreeImpl: async () => { cleanupCalls += 1; } + }); + assert.equal(summary.results[0].reason, "broker-record-invalid"); + assert.equal(cleanupCalls, 0); + + const emptyRoot = fs.mkdtempSync(path.join(os.tmpdir(), "registered-broker-reaper-empty-")); + t.after(() => fs.rmSync(emptyRoot, { recursive: true, force: true })); + const empty = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env: { CLAUDE_PLUGIN_DATA: emptyRoot } + }); + assert.equal(empty.scanned, 0); +}); + +test("an overly permissive registry root is report-only", async (t) => { + const { env, registration } = makeFixture(t); + addReleasedOwner(registration, env); + const registryRoot = path.dirname(registration.registryDir); + fs.chmodSync(registryRoot, 0o755); + const summary = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessTreeImpl: async () => { + throw new Error("invalid roots must not reach cleanup"); + } + }); + assert.equal(summary.reaped, 0); + assert.equal(summary.results[0].reason, "registry-root-invalid"); +}); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index f64af4539..d46ca2a29 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -8,9 +8,10 @@ import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; import { initGitRepo, makeTempDir as createTempDir, run } from "./helpers.mjs"; import { enqueueBackgroundTask, handleCancel, handleTaskWorker } from "../plugins/codex/scripts/codex-companion.mjs"; -import { cleanupSessionJobs } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; +import { cleanupSessionJobs, handleSessionEnd, handleSessionStart } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; import { isBrokerRequestAllowedDuringShutdown } from "../plugins/codex/scripts/app-server-broker.mjs"; +import { SESSION_OWNER_IDENTITY_ENV, SESSION_OWNER_PID_ENV } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; import { ensureBrokerSession, loadBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { readStoredJob } from "../plugins/codex/scripts/lib/job-control.mjs"; import { captureProcessOwnership, getProcessIdentity, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; @@ -64,6 +65,108 @@ test("broker rejects queued work after shutdown begins", () => { assert.equal(isBrokerRequestAllowedDuringShutdown(false, { id: 4, method: "thread/list" }), true); }); +test("SessionStart exports the stable process-group owner identity", () => { + const envFile = path.join(makeTempDir(), "session.env"); + const previousEnvFile = process.env.CLAUDE_ENV_FILE; + process.env.CLAUDE_ENV_FILE = envFile; + try { + handleSessionStart( + { + session_id: "session-owner-export", + transcript_path: "/tmp/transcript.jsonl", + cwd: ROOT + }, + { + pid: 7101, + platform: "darwin", + captureStableSessionOwnerImpl() { + return { + pid: 7100, + identity: "7100@Mon Jul 27 00:07:00 2026", + processGroupId: 7100 + }; + } + } + ); + } finally { + if (previousEnvFile == null) { + delete process.env.CLAUDE_ENV_FILE; + } else { + process.env.CLAUDE_ENV_FILE = previousEnvFile; + } + } + + const source = fs.readFileSync(envFile, "utf8"); + assert.match(source, /export CODEX_COMPANION_SESSION_ID='session-owner-export'/); + assert.match(source, new RegExp(`export ${SESSION_OWNER_PID_ENV}='7100'`)); + assert.match(source, new RegExp(`export ${SESSION_OWNER_IDENTITY_ENV}='7100@Mon Jul 27 00:07:00 2026'`)); +}); + +test("SessionEnd leaves a shared broker untouched while another registered owner is live", async () => { + const calls = []; + await handleSessionEnd( + { cwd: ROOT, session_id: "session-ending" }, + { + env: {}, + loadBrokerSessionImpl() { + return { + endpoint: "unix:/tmp/shared-broker.sock", + pid: 4100, + pidIdentity: "4100@Mon Jul 27 00:00:00 2026", + registry: { + registered: true, + version: 1, + brokerKey: "a".repeat(64), + registryRoot: "/tmp/registry", + registryDir: `/tmp/registry/${"a".repeat(64)}` + } + }; + }, + acquireBrokerRegistryLockImpl() { + calls.push("acquire-lock"); + return { acquired: true }; + }, + loadBrokerRegistrationImpl() { + calls.push("validate-registry"); + return { + registered: true, + version: 1, + brokerKey: "a".repeat(64), + registryRoot: "/tmp/registry", + registryDir: `/tmp/registry/${"a".repeat(64)}` + }; + }, + releaseBrokerOwnerImpl() { + calls.push("release-owner"); + return { released: true }; + }, + assessBrokerOwnersImpl() { + return { safeToShutdown: false, reason: "live-owner", liveOwners: [{ sessionId: "session-other" }] }; + }, + sendBrokerShutdownImpl() { + calls.push("shutdown-broker"); + }, + cleanupSessionJobsImpl() { + calls.push("cleanup-jobs"); + return { verified: true, failures: [] }; + }, + teardownBrokerSessionImpl() { + calls.push("teardown-broker"); + return { verified: true }; + }, + clearBrokerSessionImpl() { + calls.push("clear-broker"); + }, + releaseBrokerRegistryLockImpl() { + calls.push("release-lock"); + return { released: true }; + } + } + ); + + assert.deepEqual(calls, ["acquire-lock", "validate-registry", "release-owner", "release-lock", "cleanup-jobs"]); +}); + test("detached fixture keepalive self-expires when parent cleanup is skipped", async (t) => { if (process.platform === "win32") { t.skip("Unix detached fixture behavior is not available on Windows."); @@ -424,6 +527,88 @@ test("shared broker reclaims a post-snapshot helper before allowing replacement" assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 2); }); +test("automatic broker registration preserves a shared broker until its final owner releases", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + let ownerIdentity; + try { + ownerIdentity = getProcessIdentity(process.pid); + } catch (error) { + if (error?.code === "PROCESS_TABLE_UNAVAILABLE") { + t.skip(`process table unavailable: ${error.message}`); + return; + } + throw error; + } + assert.ok(ownerIdentity); + + const baseEnv = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: ownerIdentity + }; + const envA = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "registry-owner-a" }; + const envB = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "registry-owner-b" }; + const first = await ensureBrokerSession(repo, { env: envA }); + if (!first) { + t.skip("broker socket unavailable in this sandbox"); + return; + } + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envB, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "registry-owner-b" }) + }); + }); + + assert.equal(first.registry?.registered, true); + assert.equal(fs.statSync(path.join(first.registry.registryDir, "broker.json")).mode & 0o777, 0o600); + const second = await ensureBrokerSession(repo, { env: envB }); + assert.equal(second.pid, first.pid); + assert.equal(fs.readdirSync(path.join(first.registry.registryDir, "owners")).filter((name) => name.endsWith(".json")).length, 2); + + const client = await CodexAppServerClient.connect(repo, { env: envB }); + await client.request("thread/list", { cwd: repo }); + await client.close(); + await waitFor(() => { + const childrenDir = path.join(first.registry.registryDir, "children"); + return fs.existsSync(childrenDir) && fs.readdirSync(childrenDir).some((name) => name.endsWith(".json")); + }); + + const firstEnd = run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envA, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "registry-owner-a" }) + }); + assert.equal(firstEnd.status, 0, firstEnd.stderr); + assert.ok(loadBrokerSession(repo)); + assert.doesNotThrow(() => process.kill(first.pid, 0)); + + const finalEnd = run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envB, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "registry-owner-b" }) + }); + assert.equal(finalEnd.status, 0, finalEnd.stderr); + assert.equal(loadBrokerSession(repo), null); + await waitFor(() => { + try { + process.kill(first.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); +}); + test("direct app-server reclaims a post-snapshot helper after its child crashes", async (t) => { if (process.platform === "win32") { t.skip("Unix process groups are not available on Windows."); @@ -1247,7 +1432,7 @@ test("task --resume-last ignores running tasks from other Claude sessions", () = assert.match(resume.stderr, /No previous Codex task thread was found for this repository\./); }); -test("session start hook exports the Claude session id, transcript path, and plugin data dir", () => { +test("session start hook exports session context and a stable owner when available", () => { const repo = makeTempDir(); const envFile = path.join(makeTempDir(), "claude-env.sh"); fs.writeFileSync(envFile, "", "utf8"); @@ -1270,10 +1455,14 @@ test("session start hook exports the Claude session id, transcript path, and plu }); assert.equal(result.status, 0, result.stderr); - assert.equal( - fs.readFileSync(envFile, "utf8"), - `export CODEX_COMPANION_SESSION_ID='sess-current'\nexport CODEX_COMPANION_TRANSCRIPT_PATH='${transcriptPath}'\nexport CLAUDE_PLUGIN_DATA='${pluginDataDir}'\n` - ); + const exported = fs.readFileSync(envFile, "utf8"); + assert.match(exported, /export CODEX_COMPANION_SESSION_ID='sess-current'/); + assert.match(exported, new RegExp(`export CODEX_COMPANION_TRANSCRIPT_PATH='${transcriptPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}'`)); + assert.match(exported, new RegExp(`export CLAUDE_PLUGIN_DATA='${pluginDataDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}'`)); + if (process.platform !== "win32") { + assert.match(exported, /export CODEX_COMPANION_SESSION_OWNER_PID='\d+'/); + assert.match(exported, /export CODEX_COMPANION_SESSION_OWNER_IDENTITY='\d+@[^']+'/); + } }); test("write task output focuses on the Codex result without generic follow-up hints", () => { @@ -2710,10 +2899,15 @@ test("unverified cleanup preserves cancel, session, and broker ownership records assert.equal(state.jobs[0].pid, 123); assert.deepEqual(state.jobs[0].cleanupOutcome.survivorIdentities, ["123@old"]); + let sessionCleanupOptions; const sessionCleanup = await cleanupSessionJobs(workspace, "sess-current", { - terminateProcessTreeImpl: async () => cleanupOutcome + terminateProcessTreeImpl: async (_pid, options) => { + sessionCleanupOptions = options; + return cleanupOutcome; + } }); assert.equal(sessionCleanup.verified, false); + assert.equal(sessionCleanupOptions.priorCleanupDegraded, true); state = JSON.parse(fs.readFileSync(path.join(stateDir, "state.json"), "utf8")); assert.equal(state.jobs[0].pid, 123); assert.equal(fs.existsSync(jobFile), true); From 8fbd95872f2d079c3e1b74ff672f6dc9dc767be0 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Wed, 29 Jul 2026 23:25:13 -0700 Subject: [PATCH 20/29] [CC-5219] fix registered reaper review findings Attributed-To: Codex --- .../codex/scripts/lib/broker-ownership.mjs | 45 ++++++++------ .../scripts/registered-broker-reaper.mjs | 28 ++++++++- .../codex/scripts/session-lifecycle-hook.mjs | 2 +- tests/broker-ownership.test.mjs | 38 ++++++++++++ tests/registered-broker-reaper.test.mjs | 56 +++++++++++++++++ tests/runtime.test.mjs | 60 ++++++++++++++++--- 6 files changed, 201 insertions(+), 28 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-ownership.mjs b/plugins/codex/scripts/lib/broker-ownership.mjs index c220420d4..9266ae8d2 100644 --- a/plugins/codex/scripts/lib/broker-ownership.mjs +++ b/plugins/codex/scripts/lib/broker-ownership.mjs @@ -22,6 +22,10 @@ function isSafePid(value) { return Number.isSafeInteger(value) && value > 1; } +function isRegistryLockToken(value) { + return typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value); +} + function isPidIdentity(pid, identity) { return isSafePid(pid) && typeof identity === "string" && identity.startsWith(`${pid}@`) && identity.length > String(pid).length + 1; } @@ -137,8 +141,7 @@ function validRegistryLock(registration, lock) { validRegistrationReference(registration) && lock?.acquired === true && lock.brokerKey === registration.brokerKey && - typeof lock.token === "string" && - lock.token.length > 0 && + isRegistryLockToken(lock.token) && lock.path === path.join(registration.registryDir, REGISTRY_LOCK_DIR_NAME) ); } @@ -198,7 +201,7 @@ export function acquireBrokerRegistryLock( existingOwner?.version !== BROKER_OWNERSHIP_VERSION || existingOwner.kind !== "registry-lock" || existingOwner.brokerKey !== registration.brokerKey || - typeof existingOwner.token !== "string" || + !isRegistryLockToken(existingOwner.token) || !isSafePid(existingOwner.pid) ) { return { acquired: false, reason: "registry-lock-malformed", path: lockPath }; @@ -215,12 +218,14 @@ export function acquireBrokerRegistryLock( const staleRoot = path.join(registration.registryDir, "stale-locks"); ensurePrivateDir(staleRoot); + const stalePath = path.join(staleRoot, `${existingOwner.pid}-${existingOwner.token}`); try { - fs.renameSync(lockPath, path.join(staleRoot, `${existingOwner.pid}-${randomUUID()}`)); + fs.renameSync(lockPath, stalePath); } catch (error) { - if (error?.code !== "ENOENT") { - return { acquired: false, reason: "registry-lock-quarantine-failed", path: lockPath }; + if (error?.code === "ENOENT" || error?.code === "EEXIST" || error?.code === "ENOTEMPTY") { + continue; } + return { acquired: false, reason: "registry-lock-quarantine-failed", path: lockPath }; } } return { acquired: false, reason: "registry-lock-contention", path: lockPath }; @@ -592,8 +597,9 @@ export function loadBrokerChildren(registration) { export function releaseBrokerChild( registration, - { child, cleanupOutcome, now = () => new Date().toISOString() } = {} + options = {} ) { + const { child, cleanupOutcome, now = () => new Date().toISOString() } = options; if (!validRegistrationReference(registration)) { return { released: false, reason: "broker-registration-unavailable" }; } @@ -617,18 +623,21 @@ export function releaseBrokerChild( releasedAt: now() }; const releasePath = path.join(registration.registryDir, "child-releases", `${child.childKey}.json`); - if (fs.existsSync(releasePath)) { - const existing = readJson(releasePath); - if (!validChildReleaseRecord(existing, releasePath, registration, child)) { - const error = new Error(`Invalid existing broker child release at ${releasePath}.`); - error.code = "BROKER_OWNERSHIP_COLLISION"; - throw error; + const locked = withBrokerRegistryLock(registration, options, () => { + if (fs.existsSync(releasePath)) { + const existing = readJson(releasePath); + if (!validChildReleaseRecord(existing, releasePath, registration, child)) { + const error = new Error(`Invalid existing broker child release at ${releasePath}.`); + error.code = "BROKER_OWNERSHIP_COLLISION"; + throw error; + } + fs.chmodSync(releasePath, 0o600); + return { released: true, path: releasePath, release: existing }; } - fs.chmodSync(releasePath, 0o600); - return { released: true, path: releasePath, release: existing }; - } - createImmutableJson(releasePath, payload); - return { released: true, path: releasePath, release: payload }; + createImmutableJson(releasePath, payload); + return { released: true, path: releasePath, release: payload }; + }); + return locked.ok ? locked.value : { released: false, reason: locked.reason }; } export function publishBrokerReaperReceipt( diff --git a/plugins/codex/scripts/registered-broker-reaper.mjs b/plugins/codex/scripts/registered-broker-reaper.mjs index 12a3413f2..67c9a3d39 100644 --- a/plugins/codex/scripts/registered-broker-reaper.mjs +++ b/plugins/codex/scripts/registered-broker-reaper.mjs @@ -12,6 +12,7 @@ import { loadBrokerChildren, loadBrokerRegistration, publishBrokerReaperReceipt, + releaseBrokerChild, releaseBrokerRegistryLock, resolveBrokerOwnershipRoot } from "./lib/broker-ownership.mjs"; @@ -178,8 +179,33 @@ async function processRegistration(candidate, options) { killPollAttempts: options.killPollAttempts, pollIntervalMs: options.pollIntervalMs }); - outcomes.push({ target: "child", pid: child.pid, pidIdentity: child.pidIdentity, outcome }); if (outcome?.verified !== true) { + outcomes.push({ target: "child", pid: child.pid, pidIdentity: child.pidIdentity, outcome }); + cleanupVerified = false; + break; + } + let childRelease; + try { + childRelease = (options.releaseBrokerChildImpl ?? releaseBrokerChild)(lockedRegistration, { + child, + cleanupOutcome: outcome, + now: options.now, + registryLock + }); + } catch (error) { + childRelease = { released: false, reason: `child-release-error:${error.message}` }; + } + outcomes.push({ + target: "child", + pid: child.pid, + pidIdentity: child.pidIdentity, + outcome, + childRelease: { + released: childRelease?.released === true, + reason: childRelease?.reason ?? null + } + }); + if (childRelease?.released !== true) { cleanupVerified = false; break; } diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index b0d6877a2..e21f4c824 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -236,7 +236,7 @@ export async function handleSessionEnd(input, dependencies = {}) { } } - const brokerShutdownAllowed = !registry || registryAssessment?.safeToShutdown === true; + const brokerShutdownAllowed = registryAssessment?.safeToShutdown === true; let brokerCleanup = { verified: true }; let registryLockReleaseFailure = null; try { diff --git a/tests/broker-ownership.test.mjs b/tests/broker-ownership.test.mjs index da253b549..41379b9f6 100644 --- a/tests/broker-ownership.test.mjs +++ b/tests/broker-ownership.test.mjs @@ -102,6 +102,44 @@ test("a well-formed lock whose creator is absent is quarantined before retry", ( assert.equal(fs.existsSync(stale.path), false); }); +test("a stale-lock contender cannot quarantine a replacement live lock", (t) => { + const { registration } = makeFixture(t); + const stale = acquireBrokerRegistryLock(registration, { + pid: 5002, + now: () => "2026-07-27T00:00:32.000Z" + }); + assert.equal(stale.acquired, true); + + let replacement; + let livenessChecks = 0; + const contender = acquireBrokerRegistryLock(registration, { + pid: 5004, + now: () => "2026-07-27T00:00:34.000Z", + getLiveProcessPidsImpl(pids) { + livenessChecks += 1; + if (livenessChecks === 1) { + const staleRoot = path.join(registration.registryDir, "stale-locks"); + fs.mkdirSync(staleRoot, { recursive: true, mode: 0o700 }); + fs.chmodSync(staleRoot, 0o700); + fs.renameSync(stale.path, path.join(staleRoot, `${5002}-${stale.token}`)); + replacement = acquireBrokerRegistryLock(registration, { + pid: 5003, + now: () => "2026-07-27T00:00:33.000Z" + }); + assert.equal(replacement.acquired, true); + return []; + } + return pids.includes(5003) ? [5003] : []; + } + }); + + assert.equal(contender.acquired, false); + assert.equal(contender.reason, "registry-busy"); + const liveOwner = JSON.parse(fs.readFileSync(path.join(replacement.path, "owner.json"), "utf8")); + assert.equal(liveOwner.token, replacement.token); + assert.deepEqual(releaseBrokerRegistryLock(registration, replacement), { released: true }); +}); + test("registered broker with a live owner is not eligible for cleanup", (t) => { const { registration, env } = makeFixture(t); const liveOwner = ownerEnv(env, "session-live", 5100, "Mon Jul 27 00:01:00 2026"); diff --git a/tests/registered-broker-reaper.test.mjs b/tests/registered-broker-reaper.test.mjs index 1899d73c0..9adbc1ed0 100644 --- a/tests/registered-broker-reaper.test.mjs +++ b/tests/registered-broker-reaper.test.mjs @@ -6,6 +6,7 @@ import test from "node:test"; import { acquireBrokerRegistryLock, + loadBrokerChildren, publishBrokerChild, publishBrokerRegistration, registerBrokerOwner, @@ -139,6 +140,61 @@ test("registered cleanup reclaims children before the broker and writes a mode-0 assert.equal(JSON.parse(fs.readFileSync(receiptPath, "utf8")).decision, "cleanup-verified"); }); +test("each verified child is released before a later target can block convergence", async (t) => { + const { env, registration } = makeFixture(t, { child: true }); + publishBrokerChild(registration, { + ownershipSnapshot: snapshot(4300, 3), + now: () => "2026-07-27T01:03:00.000Z" + }); + addReleasedOwner(registration, env); + const orderedChildren = loadBrokerChildren(registration).children; + assert.equal(orderedChildren.length, 2); + const firstProcessedChild = orderedChildren[0]; + const blockedChild = orderedChildren[1]; + + let firstPassGroupCalls = 0; + let firstPassBrokerCalls = 0; + const firstPass = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessGroupImpl: async () => { + firstPassGroupCalls += 1; + if (firstPassGroupCalls === 1) { + return { attempted: true, delivered: true, verified: true, degraded: false, survivors: [], survivorIdentities: [] }; + } + return { attempted: true, delivered: true, verified: false, degraded: true, survivors: [blockedChild.pid], survivorIdentities: [blockedChild.pidIdentity] }; + }, + terminateProcessTreeImpl: async () => { firstPassBrokerCalls += 1; }, + attemptIdFactory: () => "attempt-partial-child-release" + }); + assert.equal(firstPass.reaped, 0); + assert.equal(firstPassGroupCalls, 2); + assert.equal(firstPassBrokerCalls, 0); + const afterFirstPass = loadBrokerChildren(registration); + assert.equal(afterFirstPass.valid, true); + assert.deepEqual(afterFirstPass.releasedChildren.map((child) => child.childKey), [firstProcessedChild.childKey]); + assert.deepEqual(afterFirstPass.children.map((child) => child.childKey), [blockedChild.childKey]); + + const secondPassCalls = []; + const secondPass = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessGroupImpl: async (pgid) => { + secondPassCalls.push(`group:${pgid}`); + return { attempted: true, delivered: true, verified: true, degraded: false, survivors: [], survivorIdentities: [] }; + }, + terminateProcessTreeImpl: async (pid) => { + secondPassCalls.push(`tree:${pid}`); + return { attempted: true, delivered: true, verified: true, degraded: false, survivors: [], survivorIdentities: [] }; + }, + attemptIdFactory: () => "attempt-converged-child-release" + }); + assert.equal(secondPass.reaped, 1); + assert.deepEqual(secondPassCalls, [`group:${blockedChild.processGroupId}`, `tree:${registration.broker.pid}`]); +}); + test("a contended registry lock closes the registration-to-signal race", async (t) => { const { env, registration } = makeFixture(t); addReleasedOwner(registration, env); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index d46ca2a29..266e308c6 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -12,7 +12,7 @@ import { cleanupSessionJobs, handleSessionEnd, handleSessionStart } from "../plu import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; import { isBrokerRequestAllowedDuringShutdown } from "../plugins/codex/scripts/app-server-broker.mjs"; import { SESSION_OWNER_IDENTITY_ENV, SESSION_OWNER_PID_ENV } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; -import { ensureBrokerSession, loadBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { clearBrokerSession, ensureBrokerSession, loadBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { readStoredJob } from "../plugins/codex/scripts/lib/job-control.mjs"; import { captureProcessOwnership, getProcessIdentity, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; import { hasCancelFlag, listJobs, loadState, resolveStateDir, saveState, upsertJob, writeCancelFlag, writeJobFile } from "../plugins/codex/scripts/lib/state.mjs"; @@ -39,20 +39,32 @@ function selfExpiringKeepaliveCode(ttlMs = DETACHED_FIXTURE_TTL_MS) { return `setTimeout(() => process.exit(0), ${ttlMs}); setInterval(() => {}, 1000)`; } -test.after(() => { +test.after(async () => { const cleanupFailures = []; for (const cwd of [ROOT, ...runtimeTempDirs]) { - if (!loadBrokerSession(cwd)) { + const brokerSession = loadBrokerSession(cwd); + if (!brokerSession) { continue; } - const cleanup = run(process.execPath, [SESSION_HOOK, "SessionEnd"], { - cwd, - input: JSON.stringify({ hook_event_name: "SessionEnd", cwd }) + await sendBrokerShutdown(brokerSession.endpoint).catch(() => {}); + const cleanup = await teardownBrokerSession({ + endpoint: brokerSession.endpoint, + pidFile: brokerSession.pidFile, + logFile: brokerSession.logFile, + sessionDir: brokerSession.sessionDir, + pid: brokerSession.pid, + pidIdentity: brokerSession.pidIdentity, + ownershipSnapshot: brokerSession.ownershipSnapshot, + requireVerifiedOwnership: brokerSession.ownershipCaptureFailed === true, + killProcess: terminateProcessTree }); - if (cleanup.status !== 0 || loadBrokerSession(cwd)) { - cleanupFailures.push({ cwd, status: cleanup.status, stderr: cleanup.stderr }); + if (cleanup?.verified === true) { + clearBrokerSession(cwd); + } + if (cleanup?.verified !== true || loadBrokerSession(cwd)) { + cleanupFailures.push({ cwd, cleanup }); } } @@ -167,6 +179,38 @@ test("SessionEnd leaves a shared broker untouched while another registered owner assert.deepEqual(calls, ["acquire-lock", "validate-registry", "release-owner", "release-lock", "cleanup-jobs"]); }); +test("SessionEnd leaves an unregistered or externally supplied broker report-only", async () => { + const calls = []; + await handleSessionEnd( + { cwd: ROOT, session_id: "session-unregistered" }, + { + loadBrokerSessionImpl() { + return { + endpoint: "unix:/tmp/unregistered-broker.sock", + pid: 4150, + pidIdentity: "4150@Mon Jul 27 00:00:50 2026" + }; + }, + sendBrokerShutdownImpl() { + calls.push("shutdown-broker"); + }, + teardownBrokerSessionImpl() { + calls.push("teardown-broker"); + return { verified: true }; + }, + clearBrokerSessionImpl() { + calls.push("clear-broker"); + }, + cleanupSessionJobsImpl() { + calls.push("cleanup-jobs"); + return { verified: true, failures: [] }; + } + } + ); + + assert.deepEqual(calls, ["cleanup-jobs"]); +}); + test("detached fixture keepalive self-expires when parent cleanup is skipped", async (t) => { if (process.platform === "win32") { t.skip("Unix detached fixture behavior is not available on Windows."); From 7f3094a3918e833e2a6c54c8deb7c3a8bbc17b73 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Wed, 29 Jul 2026 23:44:13 -0700 Subject: [PATCH 21/29] [CC-5219] close terminal ownership gaps Attributed-To: Codex --- .../codex/scripts/lib/broker-lifecycle.mjs | 8 ++++ plugins/codex/scripts/lib/process.mjs | 7 ++- tests/process.test.mjs | 48 ++++++++++++++++++- tests/runtime.test.mjs | 18 ++++++- 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 2af47066f..fcfb96860 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -113,6 +113,14 @@ async function isBrokerEndpointReady(endpoint) { } export async function ensureBrokerSession(cwd, options = {}) { + // Automatic brokers are detached and outlive the process that spawned them. + // Until Windows has a durable, reuse-resistant process identity for the + // registry, use the attached direct app-server lifecycle instead of + // publishing an unregistered broker that SessionEnd must refuse to signal. + if ((options.platform ?? process.platform) === "win32") { + return null; + } + const existing = loadBrokerSession(cwd); if (existing && (await isBrokerEndpointReady(existing.endpoint))) { if (existing.registry?.registered === true) { diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 9ea63e155..70680d2a6 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -707,11 +707,13 @@ export async function terminateProcessGroup(pgid, options = {}) { for (const record of recordsFromOwnershipSnapshot(ownershipSnapshot)) { tracked.set(record.identity, record); } + let groupMembersFound = false; let groupSelectionFound = false; for (const record of processes.values()) { if (record.processGroupId !== pgid || !isRunningProcess(record)) { continue; } + groupMembersFound = true; const snapshotRecord = recordsFromOwnershipSnapshot(ownershipSnapshot).find((candidate) => candidate.pid === record.pid); if (ownershipSnapshot && snapshotRecord && snapshotRecord.identity !== record.identity) { continue; @@ -752,11 +754,12 @@ export async function terminateProcessGroup(pgid, options = {}) { } const root = processes.get(pgid); const rootIdentityMatches = !root || root.identity === ownershipSnapshot?.rootIdentity; + const ownedGroupAccountedFor = groupSelectionFound || !groupMembersFound; return normalizeProcessCleanupOutcome({ attempted: true, delivered, - verified: live.length === 0 && (!ownershipEstablished || (rootIdentityMatches && groupSelectionFound)), - degraded: ownershipEstablished && (!rootIdentityMatches || !groupSelectionFound), + verified: live.length === 0 && (!ownershipEstablished || (rootIdentityMatches && ownedGroupAccountedFor)), + degraded: ownershipEstablished && (!rootIdentityMatches || !ownedGroupAccountedFor), escalated, method: "process-group", targets: [...tracked.values()].map((record) => record.pid), diff --git a/tests/process.test.mjs b/tests/process.test.mjs index cfc494044..d0576a97e 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -821,6 +821,50 @@ test("terminateProcessGroup reclaims orphaned members of a dead leader's group", assert.deepEqual(outcome.targets, [202]); }); +test("terminateProcessGroup converges when an owned group is already absent", async () => { + const signals = []; + const ownershipSnapshot = { + rootPid: 200, + rootIdentity: "200@Mon Jul 27 00:00:00 2026", + processGroupId: 200, + members: [ + { + pid: 200, + parentPid: 1, + processGroupId: 200, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "200@Mon Jul 27 00:00:00 2026", + depth: 0 + } + ] + }; + const outcome = await terminateProcessGroup(200, { + platform: "darwin", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + } + }); + + assert.deepEqual(signals, []); + assert.equal(outcome.verified, true); + assert.equal(outcome.degraded, false); + assert.deepEqual(outcome.survivors, []); +}); + test("terminateProcessGroup hunts an observed regrouped helper after its root exits", async () => { const signals = []; const alive = new Set([200]); @@ -871,8 +915,8 @@ test("terminateProcessGroup hunts an observed regrouped helper after its root ex }); assert.deepEqual(signals, [[-200, "SIGTERM"]]); - assert.equal(outcome.verified, false); - assert.equal(outcome.degraded, true); + assert.equal(outcome.verified, true); + assert.equal(outcome.degraded, false); assert.deepEqual(outcome.survivors, []); }); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 266e308c6..40f048c6e 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -489,7 +489,7 @@ test("fake app-server crash reclaims an observed regrouped helper without replac await new Promise((resolve) => child.once("exit", resolve)); const outcome = await terminateProcessGroup(child.pid, { ownershipSnapshot, env }); - assert.equal(outcome.verified, false); + assert.equal(outcome.verified, true); assert.deepEqual(outcome.survivors, []); assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 1); }); @@ -653,6 +653,22 @@ test("automatic broker registration preserves a shared broker until its final ow }); }); +test("automatic broker creation stays disabled where registered ownership is unsupported", async () => { + const repo = makeTempDir(); + let endpointFactoryCalled = false; + const session = await ensureBrokerSession(repo, { + platform: "win32", + createBrokerEndpoint() { + endpointFactoryCalled = true; + throw new Error("Windows automatic broker creation must stop before endpoint creation."); + } + }); + + assert.equal(session, null); + assert.equal(endpointFactoryCalled, false); + assert.equal(loadBrokerSession(repo), null); +}); + test("direct app-server reclaims a post-snapshot helper after its child crashes", async (t) => { if (process.platform === "win32") { t.skip("Unix process groups are not available on Windows."); From 58c1ea315807fb493540d4b50ed9d4cc2fa25ef4 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Thu, 30 Jul 2026 01:52:48 -0700 Subject: [PATCH 22/29] [CC-5589] make automatic broker launch transactional Attributed-To: Codex --- plugins/codex/scripts/app-server-broker.mjs | 69 +- plugins/codex/scripts/lib/app-server.mjs | 4 +- .../codex/scripts/lib/broker-lifecycle.mjs | 612 ++++++++++++++-- .../codex/scripts/lib/broker-ownership.mjs | 87 ++- plugins/codex/scripts/lib/codex.mjs | 4 +- plugins/codex/scripts/lib/process.mjs | 13 + tests/broker-ownership.test.mjs | 68 ++ tests/process.test.mjs | 22 +- tests/runtime.test.mjs | 656 +++++++++++++++++- 9 files changed, 1439 insertions(+), 96 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 742d8035d..023e9c09d 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -16,6 +16,8 @@ const DEFAULT_CHILD_IDLE_MS = 5 * 60 * 1000; const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]); const BROKER_CLEANUP_UNVERIFIED_RPC_CODE = -32002; const BROKER_SHUTDOWN_RPC_CODE = -32003; +const BROKER_NOT_ACTIVATED_RPC_CODE = -32004; +const BROKER_ACTIVATION_ACK = "activated"; export function isBrokerRequestAllowedDuringShutdown(shuttingDown, message) { return !shuttingDown || message?.method === "broker/shutdown"; @@ -80,7 +82,8 @@ async function main() { } const { options } = parseArgs(argv, { - valueOptions: ["cwd", "pid-file", "endpoint"] + valueOptions: ["cwd", "pid-file", "endpoint"], + booleanOptions: ["require-activation-stdin"] }); if (!options.endpoint) { @@ -91,6 +94,7 @@ async function main() { const endpoint = String(options.endpoint); const listenTarget = parseBrokerEndpoint(endpoint); const pidFile = options["pid-file"] ? path.resolve(options["pid-file"]) : null; + const activationRequired = options["require-activation-stdin"] === true; writePidFile(pidFile); const childIdleMs = resolveChildIdleMs(); @@ -102,6 +106,8 @@ async function main() { let inFlightRequests = 0; let shutdownPromise = null; let shuttingDown = false; + let activated = !activationRequired; + let activationAbortStarted = false; let activeRequestSocket = null; let activeStreamSocket = null; let activeStreamThreadIds = null; @@ -408,6 +414,50 @@ async function main() { return shutdownPromise; } + function setupActivationGate(server) { + let buffer = ""; + const abort = () => { + if (activated || activationAbortStarted) { + return; + } + activationAbortStarted = true; + void shutdown(server) + .catch(() => {}) + .finally(() => process.exit(1)); + }; + + process.stdin.setEncoding("utf8"); + process.stdin.on("data", (chunk) => { + if (activated || activationAbortStarted) { + return; + } + buffer += chunk; + const newlineIndex = buffer.indexOf("\n"); + if (newlineIndex === -1) { + return; + } + const command = buffer.slice(0, newlineIndex).trim(); + if (command !== "activate") { + abort(); + return; + } + activated = true; + try { + fs.writeSync(3, `${BROKER_ACTIVATION_ACK}\n`); + fs.closeSync(3); + } catch { + activated = false; + abort(); + } + }); + process.stdin.on("end", abort); + process.stdin.on("error", abort); + process.stdin.resume(); + if (process.stdin.readableEnded) { + abort(); + } + } + const server = net.createServer((socket) => { if (shuttingDown) { socket.destroy(); @@ -445,6 +495,17 @@ async function main() { continue; } + if (activationRequired && !activated && message.method !== "broker/shutdown") { + if (message.id !== undefined) { + send(socket, { + id: message.id, + error: buildJsonRpcError(BROKER_NOT_ACTIVATED_RPC_CODE, "Shared Codex broker is not activated.") + }); + } + socket.end(); + continue; + } + if (message.id !== undefined && message.method === "initialize") { send(socket, { id: message.id, @@ -582,7 +643,11 @@ async function main() { process.exit(0); }); - server.listen(listenTarget.path); + server.listen(listenTarget.path, () => { + if (activationRequired) { + setupActivationGate(server); + } + }); } if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index 912d0097f..31f7c9eba 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -13,7 +13,7 @@ import process from "node:process"; import { spawn } from "node:child_process"; import readline from "node:readline"; import { parseBrokerEndpoint } from "./broker-endpoint.mjs"; -import { ensureBrokerSession, loadBrokerSession } from "./broker-lifecycle.mjs"; +import { ensureBrokerSession, loadReusableBrokerSession } from "./broker-lifecycle.mjs"; import { captureProcessOwnership, normalizeProcessCleanupOutcome, terminateProcessGroup, terminateProcessTree } from "./process.mjs"; const PLUGIN_MANIFEST_URL = new URL("../../.claude-plugin/plugin.json", import.meta.url); @@ -457,7 +457,7 @@ export class CodexAppServerClient { if (!options.disableBroker) { brokerEndpoint = options.brokerEndpoint ?? options.env?.[BROKER_ENDPOINT_ENV] ?? process.env[BROKER_ENDPOINT_ENV] ?? null; if (!brokerEndpoint && options.reuseExistingBroker) { - brokerEndpoint = loadBrokerSession(cwd)?.endpoint ?? null; + brokerEndpoint = loadReusableBrokerSession(cwd, options.env ?? process.env)?.endpoint ?? null; } if (!brokerEndpoint && !options.reuseExistingBroker) { const brokerSession = await ensureBrokerSession(cwd, { env: options.env }); diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index fcfb96860..c897e1a3c 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -4,15 +4,124 @@ import os from "node:os"; import path from "node:path"; import process from "node:process"; import { spawn } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; import { fileURLToPath } from "node:url"; -import { publishBrokerRegistration, registerBrokerOwner } from "./broker-ownership.mjs"; +import { + acquireBrokerRegistryLock, + assessBrokerOwners, + hasLiveBrokerOwnerIdentity, + loadBrokerRegistration, + publishRegisteredBroker, + registerBrokerOwner, + releaseBrokerOwner, + releaseBrokerRegistryLock, + resolveBrokerOwnershipRoot +} from "./broker-ownership.mjs"; import { createBrokerEndpoint, parseBrokerEndpoint } from "./broker-endpoint.mjs"; -import { captureProcessOwnership } from "./process.mjs"; +import { captureProcessOwnership, getProcessIdentity, terminateProcessTree } from "./process.mjs"; import { resolveStateDir } from "./state.mjs"; export const PID_FILE_ENV = "CODEX_COMPANION_APP_SERVER_PID_FILE"; export const LOG_FILE_ENV = "CODEX_COMPANION_APP_SERVER_LOG_FILE"; const BROKER_STATE_FILE = "broker.json"; +const BROKER_ACTIVATION_ACK = "activated"; +const BROKER_LAUNCH_LOCK_HOST = "127.0.0.1"; +const BROKER_LAUNCH_LOCK_MIN_PORT = 49152; +const BROKER_LAUNCH_LOCK_PORT_COUNT = 16384; + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function brokerLaunchLockIdentity(cwd) { + return createHash("sha256").update(resolveBrokerStateFile(cwd)).digest("hex"); +} + +export function brokerLaunchLockPort(cwd) { + const identity = brokerLaunchLockIdentity(cwd); + const digest = Buffer.from(identity, "hex"); + return BROKER_LAUNCH_LOCK_MIN_PORT + (digest.readUInt16BE(0) % BROKER_LAUNCH_LOCK_PORT_COUNT); +} + +function tryListenForBrokerLaunch(port, greeting) { + return new Promise((resolve, reject) => { + const server = net.createServer((socket) => socket.end(`${greeting}\n`)); + const onError = (error) => { + if (error?.code === "EADDRINUSE") { + resolve(null); + return; + } + reject(error); + }; + server.once("error", onError); + server.listen({ host: BROKER_LAUNCH_LOCK_HOST, port, exclusive: true }, () => { + server.removeListener("error", onError); + resolve(server); + }); + }); +} + +function probeBrokerLaunchPort(port, expectedGreeting, timeoutMs = 150) { + return new Promise((resolve) => { + const socket = net.createConnection({ host: BROKER_LAUNCH_LOCK_HOST, port }); + let connected = false; + let buffer = ""; + let settled = false; + const finish = (state) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + socket.destroy(); + resolve(state); + }; + const timer = setTimeout(() => finish(connected ? "foreign" : "transient"), timeoutMs); + socket.setEncoding("utf8"); + socket.on("connect", () => { + connected = true; + }); + socket.on("data", (chunk) => { + buffer += chunk; + if (buffer.includes("\n")) { + finish(buffer.trim() === expectedGreeting ? "matching" : "foreign"); + } + }); + socket.on("end", () => finish(buffer.trim() === expectedGreeting ? "matching" : "foreign")); + socket.on("error", () => finish("transient")); + }); +} + +export async function acquireBrokerLaunchLock(cwd, options = {}) { + const identity = brokerLaunchLockIdentity(cwd); + const greeting = `codex-broker-launch-v1:${identity}`; + const port = brokerLaunchLockPort(cwd); + const timeoutMs = options.timeoutMs ?? 10_000; + const deadline = Date.now() + timeoutMs; + + while (Date.now() < deadline) { + const server = await tryListenForBrokerLaunch(port, greeting); + if (server) { + return { + port, + async release() { + await new Promise((resolve) => server.close(resolve)); + } + }; + } + const observed = await probeBrokerLaunchPort(port, greeting); + if (observed === "foreign") { + const error = new Error("The deterministic Codex broker launch lock is occupied by another local service."); + error.code = "BROKER_LAUNCH_LOCK_UNAVAILABLE"; + throw error; + } + await delay(25); + } + + const error = new Error("Timed out waiting for another Codex broker launch to finish."); + error.code = "BROKER_LAUNCH_LOCK_TIMEOUT"; + throw error; +} export function createBrokerSessionDir(prefix = "cxc-") { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); @@ -60,15 +169,119 @@ export async function sendBrokerShutdown(endpoint) { export function spawnBrokerProcess({ scriptPath, cwd, endpoint, pidFile, logFile, env = process.env }) { const logFd = fs.openSync(logFile, "a"); - const child = spawn(process.execPath, [scriptPath, "serve", "--endpoint", endpoint, "--cwd", cwd, "--pid-file", pidFile], { - cwd, - env, - detached: true, - stdio: ["ignore", logFd, logFd] + try { + const child = spawn( + process.execPath, + [scriptPath, "serve", "--endpoint", endpoint, "--cwd", cwd, "--pid-file", pidFile, "--require-activation-stdin"], + { + cwd, + env, + detached: true, + stdio: ["pipe", logFd, logFd, "pipe"] + } + ); + child.on("error", () => {}); + return child; + } finally { + fs.closeSync(logFd); + } +} + +export function activateBrokerProcess(child, timeoutMs = 2000) { + return new Promise((resolve, reject) => { + const status = child?.stdio?.[3]; + if (!child?.stdin || !status) { + const error = new Error("Broker activation channels are unavailable."); + error.code = "BROKER_ACTIVATION_FAILED"; + reject(error); + return; + } + + let buffer = ""; + let settled = false; + const ignoreStdinError = () => {}; + const cleanup = () => { + clearTimeout(timer); + status.removeListener("data", onData); + status.removeListener("error", onStatusError); + child.removeListener("error", onChildError); + child.removeListener("exit", onChildExit); + child.stdin.removeListener("error", onStdinError); + }; + const finish = (error = null) => { + if (settled) { + return; + } + settled = true; + cleanup(); + status.destroy(); + child.stdin.on("error", ignoreStdinError); + if (error) { + reject(error); + return; + } + child.unref(); + resolve(); + }; + const activationError = (message) => { + const error = new Error(message); + error.code = "BROKER_ACTIVATION_FAILED"; + return error; + }; + const onData = (chunk) => { + buffer += chunk; + if (buffer.includes("\n")) { + finish(buffer.trim() === BROKER_ACTIVATION_ACK + ? null + : activationError(`Unexpected broker activation response: ${buffer.trim() || "empty"}.`)); + } + }; + const onStatusError = (error) => finish(activationError(`Broker activation response failed: ${error.message}`)); + const onChildError = (error) => finish(activationError(`Broker activation process failed: ${error.message}`)); + const onChildExit = () => finish(activationError("Broker exited before activation completed.")); + const onStdinError = (error) => finish(activationError(`Broker activation request failed: ${error.message}`)); + const timer = setTimeout( + () => finish(activationError("Timed out waiting for broker activation acknowledgement.")), + timeoutMs + ); + + status.setEncoding("utf8"); + status.on("data", onData); + status.once("error", onStatusError); + child.once("error", onChildError); + child.once("exit", onChildExit); + child.stdin.once("error", onStdinError); + child.stdin.end("activate\n"); + }); +} + +function closeBrokerActivationChannels(child) { + if (child?.stdin && !child.stdin.destroyed) { + child.stdin.once("error", () => {}); + child.stdin.end(); + } + child?.stdio?.[3]?.destroy(); +} + +async function waitForBrokerProcessExit(child, timeoutMs = 1000) { + if (!child || child.exitCode !== null || child.signalCode !== null) { + return true; + } + return new Promise((resolve) => { + let settled = false; + const finish = (exited) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + child.removeListener("exit", onExit); + resolve(exited); + }; + const onExit = () => finish(true); + const timer = setTimeout(() => finish(false), timeoutMs); + child.once("exit", onExit); }); - child.unref(); - fs.closeSync(logFd); - return child; } function resolveBrokerStateFile(cwd) { @@ -91,7 +304,20 @@ export function loadBrokerSession(cwd) { export function saveBrokerSession(cwd, session) { const stateDir = resolveStateDir(cwd); fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync(resolveBrokerStateFile(cwd), `${JSON.stringify(session, null, 2)}\n`, "utf8"); + const stateFile = resolveBrokerStateFile(cwd); + const temporaryStateFile = `${stateFile}.${process.pid}.${randomUUID()}.tmp`; + try { + fs.writeFileSync(temporaryStateFile, `${JSON.stringify(session, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600 + }); + fs.renameSync(temporaryStateFile, stateFile); + } finally { + if (fs.existsSync(temporaryStateFile)) { + fs.unlinkSync(temporaryStateFile); + } + } } export function clearBrokerSession(cwd) { @@ -101,6 +327,49 @@ export function clearBrokerSession(cwd) { } } +export function loadReusableBrokerSession(cwd, env = process.env) { + if (!resolveBrokerOwnershipRoot(env) || !hasLiveBrokerOwnerIdentity(env)) { + return null; + } + const session = loadBrokerSession(cwd); + if ( + session?.activationPending === true || + session?.activationFailed === true || + session?.registry?.registered !== true || + typeof session.pidIdentity !== "string" + ) { + return null; + } + try { + if (getProcessIdentity(session.pid, { cwd, env }) !== session.pidIdentity) { + return null; + } + } catch { + return null; + } + const registration = loadBrokerRegistration({ + endpoint: session.endpoint, + brokerIdentity: session.pidIdentity, + env + }); + if ( + registration.registered !== true || + registration.brokerKey !== session.registry.brokerKey || + registration.registryDir !== session.registry.registryDir + ) { + return null; + } + try { + const target = parseBrokerEndpoint(session.endpoint); + if (target.kind !== "unix" || !fs.lstatSync(target.path).isSocket()) { + return null; + } + } catch { + return null; + } + return { ...session, registry: registration }; +} + async function isBrokerEndpointReady(endpoint) { if (!endpoint) { return false; @@ -112,30 +381,98 @@ async function isBrokerEndpointReady(endpoint) { } } -export async function ensureBrokerSession(cwd, options = {}) { - // Automatic brokers are detached and outlive the process that spawned them. - // Until Windows has a durable, reuse-resistant process identity for the - // registry, use the attached direct app-server lifecycle instead of - // publishing an unregistered broker that SessionEnd must refuse to signal. - if ((options.platform ?? process.platform) === "win32") { - return null; +async function rollbackNewBrokerSession(cwd, child, session, options) { + closeBrokerActivationChannels(child); + const exited = await waitForBrokerProcessExit(child, options.rollbackExitTimeoutMs ?? 1000); + let cleanup; + try { + cleanup = await teardownBrokerSession({ + endpoint: session.endpoint, + pidFile: session.pidFile, + logFile: session.logFile, + sessionDir: session.sessionDir, + pid: session.pid, + pidIdentity: session.pidIdentity, + ownershipSnapshot: session.ownershipSnapshot, + requireVerifiedOwnership: session.ownershipCaptureFailed === true, + killProcess: exited ? null : (options.killProcess ?? terminateProcessTree) + }); + } finally { + child?.unref?.(); } - const existing = loadBrokerSession(cwd); - if (existing && (await isBrokerEndpointReady(existing.endpoint))) { - if (existing.registry?.registered === true) { - const owner = registerBrokerOwner(existing.registry, { env: options.env ?? process.env }); - if (owner.registered !== true) { - const error = new Error(`Unable to register this session as a shared Codex broker owner (${owner.reason ?? "unknown"}).`); - error.code = "BROKER_OWNER_REGISTRATION_FAILED"; + if (cleanup?.verified === true) { + clearBrokerSession(cwd); + return cleanup; + } + + try { + saveBrokerSession(cwd, { ...session, activationFailed: true }); + } catch { + // Preserve the cleanup outcome as the primary failure if even the + // report-only recovery record cannot be written. + } + return cleanup; +} + +async function cleanupExistingBrokerSession(cwd, existing, options) { + const env = options.env ?? process.env; + const acquireRegistryLock = options.acquireBrokerRegistryLockImpl ?? acquireBrokerRegistryLock; + const releaseRegistryLock = options.releaseBrokerRegistryLockImpl ?? releaseBrokerRegistryLock; + const registryLock = acquireRegistryLock(existing.registry); + if (registryLock?.acquired !== true) { + const error = new Error(`Broker cleanup eligibility could not be locked (${registryLock?.reason ?? "unknown"}).`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + + try { + const loadRegistration = options.loadBrokerRegistrationImpl ?? loadBrokerRegistration; + const registration = loadRegistration({ + endpoint: existing.endpoint, + brokerIdentity: existing.pidIdentity, + env + }); + if ( + registration.registered !== true || + registration.brokerKey !== existing.registry.brokerKey || + registration.registryDir !== existing.registry.registryDir + ) { + const error = new Error("The existing Codex broker registration is invalid; cleanup remains report-only."); + error.code = "BROKER_REGISTRATION_REQUIRED"; + throw error; + } + + const readProcessIdentity = options.getProcessIdentityImpl ?? getProcessIdentity; + let currentIdentity; + try { + currentIdentity = readProcessIdentity(existing.pid, { cwd, env }); + } catch (cause) { + const error = new Error("Broker liveness could not be verified; cleanup remains report-only."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + error.cause = cause; + throw error; + } + if (currentIdentity && currentIdentity !== existing.pidIdentity) { + const error = new Error("The saved broker PID has been reused; refusing to signal it or start a replacement."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + if (currentIdentity === existing.pidIdentity) { + const assessOwners = options.assessBrokerOwnersImpl ?? assessBrokerOwners; + const assessment = assessOwners(registration); + if (assessment?.safeToShutdown !== true) { + if (assessment?.reason === "live-owner") { + return { cleaned: false, blockedByLiveOwner: true }; + } + const error = new Error(`Broker ownership is ambiguous (${assessment?.reason ?? "unknown"}); cleanup remains report-only.`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; throw error; } } - return existing; - } - if (existing) { - const cleanup = await teardownBrokerSession({ + const teardownBroker = options.teardownBrokerSessionImpl ?? teardownBrokerSession; + const cleanup = await teardownBroker({ endpoint: existing.endpoint ?? null, pidFile: existing.pidFile ?? null, logFile: existing.logFile ?? null, @@ -144,7 +481,7 @@ export async function ensureBrokerSession(cwd, options = {}) { pidIdentity: existing.pidIdentity ?? null, ownershipSnapshot: existing.ownershipSnapshot ?? null, requireVerifiedOwnership: existing.ownershipCaptureFailed === true, - killProcess: options.killProcess ?? null + killProcess: options.killProcess ?? terminateProcessTree }); if (cleanup?.verified !== true) { const error = new Error("Broker cleanup is unverified; refusing to start another broker session."); @@ -152,6 +489,99 @@ export async function ensureBrokerSession(cwd, options = {}) { throw error; } clearBrokerSession(cwd); + return { cleaned: true, cleanup }; + } finally { + const released = releaseRegistryLock(existing.registry, registryLock); + if (released?.released !== true) { + const error = new Error(`Broker registry lock release failed (${released?.reason ?? "unknown"}).`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + } +} + +async function ensureBrokerSessionLocked(cwd, options = {}) { + const env = options.env ?? process.env; + // Automatic brokers are detached and outlive the process that spawned them. + // Until Windows has a durable, reuse-resistant process identity for the + // registry, use the attached direct app-server lifecycle instead of + // publishing an unregistered broker that SessionEnd must refuse to signal. + const stateFileExists = fs.existsSync(resolveBrokerStateFile(cwd)); + const existing = loadBrokerSession(cwd); + if (stateFileExists && !existing) { + return null; + } + if (existing && existing.registry?.registered !== true) { + return null; + } + let existingProcessIdentity = null; + if (existing && existing.activationFailed !== true) { + try { + const readProcessIdentity = options.getProcessIdentityImpl ?? getProcessIdentity; + existingProcessIdentity = readProcessIdentity(existing.pid, { cwd, env }); + } catch (cause) { + const error = new Error("Existing broker liveness could not be verified; refusing reuse or replacement."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + error.cause = cause; + throw error; + } + } + if ( + existing && + existing.activationPending !== true && + existing.activationFailed !== true && + existingProcessIdentity === existing.pidIdentity && + (await isBrokerEndpointReady(existing.endpoint)) + ) { + const acquireRegistryLock = options.acquireBrokerRegistryLockImpl ?? acquireBrokerRegistryLock; + const releaseRegistryLock = options.releaseBrokerRegistryLockImpl ?? releaseBrokerRegistryLock; + const registryLock = acquireRegistryLock(existing.registry); + if (registryLock?.acquired !== true) { + return null; + } + try { + const loadRegistration = options.loadBrokerRegistrationImpl ?? loadBrokerRegistration; + const registration = loadRegistration({ + endpoint: existing.endpoint, + brokerIdentity: existing.pidIdentity, + env + }); + if ( + registration.registered !== true || + registration.brokerKey !== existing.registry.brokerKey || + registration.registryDir !== existing.registry.registryDir || + !hasLiveBrokerOwnerIdentity(env) + ) { + return null; + } + const readProcessIdentity = options.getProcessIdentityImpl ?? getProcessIdentity; + if ( + readProcessIdentity(existing.pid, { cwd, env }) !== existing.pidIdentity || + !(await isBrokerEndpointReady(existing.endpoint)) + ) { + return null; + } + const registerOwner = options.registerBrokerOwnerImpl ?? registerBrokerOwner; + const owner = registerOwner(registration, { env, registryLock }); + if (owner.registered !== true) { + return null; + } + return { ...existing, registry: registration }; + } finally { + const released = releaseRegistryLock(existing.registry, registryLock); + if (released?.released !== true) { + const error = new Error(`Broker registry lock release failed (${released?.reason ?? "unknown"}).`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + } + } + + if (existing) { + const recovery = await cleanupExistingBrokerSession(cwd, existing, options); + if (recovery.blockedByLiveOwner) { + return null; + } } const sessionDir = createBrokerSessionDir(); @@ -163,7 +593,8 @@ export async function ensureBrokerSession(cwd, options = {}) { options.scriptPath ?? fileURLToPath(new URL("../app-server-broker.mjs", import.meta.url)); - const child = spawnBrokerProcess({ + const spawnBroker = options.spawnBrokerProcessImpl ?? spawnBrokerProcess; + const child = spawnBroker({ scriptPath, cwd, endpoint, @@ -187,57 +618,108 @@ export async function ensureBrokerSession(cwd, options = {}) { } } + const session = { + endpoint, + pidFile, + logFile, + sessionDir, + pid: child.pid ?? null, + pidIdentity: ownershipSnapshot?.rootIdentity ?? null, + ownershipSnapshot, + ownershipCaptureFailed, + registry: null + }; const ready = await waitForBrokerEndpoint(endpoint, options.timeoutMs ?? 2000); if (!ready) { - const cleanup = await teardownBrokerSession({ - endpoint, - pidFile, - logFile, - sessionDir, - pid: child.pid ?? null, - pidIdentity: ownershipSnapshot?.rootIdentity ?? null, - ownershipSnapshot, - requireVerifiedOwnership: ownershipCaptureFailed, - killProcess: options.killProcess ?? null - }); + const cleanup = await rollbackNewBrokerSession(cwd, child, session, options); if (cleanup?.verified !== true) { - return null; + const error = new Error("Failed broker startup cleanup is unverified; refusing to create or hide another process."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; } return null; } - let registry = null; + let fallbackToDirect = false; + let ownerRegistered = false; try { - const candidate = publishBrokerRegistration({ + const publishRegistration = options.publishRegisteredBrokerImpl ?? publishRegisteredBroker; + const candidate = publishRegistration({ cwd, endpoint, pid: child.pid ?? null, ownershipSnapshot, - env: options.env ?? process.env + env }); - if (candidate.registered === true) { - const owner = registerBrokerOwner(candidate, { env: options.env ?? process.env }); - if (owner.registered === true) { - registry = candidate; - } + if (candidate.registered !== true) { + fallbackToDirect = [ + "broker-identity-unavailable", + "plugin-data-unavailable", + "session-owner-not-live", + "session-owner-unavailable" + ].includes(candidate.reason); + throw new Error(`Broker registration is unavailable (${candidate.reason ?? "unknown"}).`); } + session.registry = candidate; + ownerRegistered = true; + const saveSession = options.saveBrokerSessionImpl ?? saveBrokerSession; + session.activationPending = true; + saveSession(cwd, session); + + const activateBroker = options.activateBrokerProcessImpl ?? activateBrokerProcess; + await activateBroker(child, options.activationTimeoutMs ?? 2000); + session.activationPending = false; + saveSession(cwd, session); + return session; } catch (error) { - process.stderr.write(`Warning: unable to publish Codex broker ownership: ${error.message}. Broker remains unregistered.\n`); + if (ownerRegistered) { + const releaseOwner = options.releaseBrokerOwnerImpl ?? releaseBrokerOwner; + try { + const released = releaseOwner(session.registry, { env }); + if (released.released !== true) { + process.stderr.write(`Warning: unable to release rolled-back Codex broker owner (${released.reason ?? "unknown"}).\n`); + } + } catch (releaseError) { + process.stderr.write(`Warning: unable to release rolled-back Codex broker owner: ${releaseError.message}.\n`); + } + } + const cleanup = await rollbackNewBrokerSession(cwd, child, session, options); + if (cleanup?.verified !== true) { + const cleanupError = new Error(`Broker launch failed and exact rollback is unverified: ${error.message}`); + cleanupError.code = "BROKER_CLEANUP_UNVERIFIED"; + cleanupError.cause = error; + throw cleanupError; + } + if (fallbackToDirect) { + return null; + } + const transactionError = new Error(`Broker launch transaction failed: ${error.message}`); + transactionError.code = "BROKER_REGISTRATION_FAILED"; + transactionError.cause = error; + throw transactionError; } +} - const session = { - endpoint, - pidFile, - logFile, - sessionDir, - pid: child.pid ?? null, - pidIdentity: ownershipSnapshot?.rootIdentity ?? null, - ownershipSnapshot, - ownershipCaptureFailed, - registry - }; - saveBrokerSession(cwd, session); - return session; +export async function ensureBrokerSession(cwd, options = {}) { + if ((options.platform ?? process.platform) === "win32") { + return null; + } + const env = options.env ?? process.env; + if (!resolveBrokerOwnershipRoot(env) || !hasLiveBrokerOwnerIdentity(env)) { + return null; + } + const acquireLaunchLock = options.acquireBrokerLaunchLockImpl ?? acquireBrokerLaunchLock; + let launchLock; + try { + launchLock = await acquireLaunchLock(cwd, { timeoutMs: options.launchLockTimeoutMs ?? 10_000 }); + } catch { + return null; + } + try { + return await ensureBrokerSessionLocked(cwd, options); + } finally { + await launchLock.release(); + } } export async function teardownBrokerSession({ diff --git a/plugins/codex/scripts/lib/broker-ownership.mjs b/plugins/codex/scripts/lib/broker-ownership.mjs index 9266ae8d2..f2ced454b 100644 --- a/plugins/codex/scripts/lib/broker-ownership.mjs +++ b/plugins/codex/scripts/lib/broker-ownership.mjs @@ -3,7 +3,7 @@ import fs from "node:fs"; import path from "node:path"; import process from "node:process"; -import { getLiveProcessPids } from "./process.mjs"; +import { getLiveProcessPids, hasLiveProcessIdentity } from "./process.mjs"; export const BROKER_OWNERSHIP_VERSION = 1; export const SESSION_OWNER_PID_ENV = "CODEX_COMPANION_SESSION_OWNER_PID"; @@ -370,6 +370,91 @@ function ownerFromEnv(env) { return { ownerKey, sessionId, pid, identity }; } +export function hasLiveBrokerOwnerIdentity(env = process.env, options = {}) { + const owner = ownerFromEnv(env); + if (!owner) { + return false; + } + const hasLiveIdentity = options.hasLiveProcessIdentityImpl ?? hasLiveProcessIdentity; + try { + return hasLiveIdentity(owner.pid, owner.identity); + } catch { + return false; + } +} + +export function publishRegisteredBroker({ + cwd, + endpoint, + pid, + ownershipSnapshot, + env = process.env, + now = () => new Date().toISOString(), + hasLiveProcessIdentityImpl = hasLiveProcessIdentity +}) { + const registryRoot = resolveBrokerOwnershipRoot(env); + if (!registryRoot) { + return { registered: false, reason: "plugin-data-unavailable" }; + } + const owner = ownerFromEnv(env); + if (!owner) { + return { registered: false, reason: "session-owner-unavailable" }; + } + if (!hasLiveBrokerOwnerIdentity(env, { hasLiveProcessIdentityImpl })) { + return { registered: false, reason: "session-owner-not-live" }; + } + if ( + typeof endpoint !== "string" || + !endpoint || + !isSafePid(pid) || + ownershipSnapshot?.rootPid !== pid || + !isPidIdentity(pid, ownershipSnapshot?.rootIdentity) || + !isSafePid(ownershipSnapshot?.processGroupId) + ) { + return { registered: false, reason: "broker-identity-unavailable" }; + } + + const brokerKey = brokerOwnershipKey(endpoint, ownershipSnapshot.rootIdentity); + const registration = registrationReference(registryRoot, brokerKey); + const createdAt = now(); + const broker = { + version: BROKER_OWNERSHIP_VERSION, + kind: "broker", + brokerKey, + endpoint, + pid, + pidIdentity: ownershipSnapshot.rootIdentity, + processGroupId: ownershipSnapshot.processGroupId, + workspaceHash: sha256(path.resolve(cwd)), + createdAt + }; + const ownerRecord = { + version: BROKER_OWNERSHIP_VERSION, + kind: "owner", + brokerKey, + ownerKey: owner.ownerKey, + sessionId: owner.sessionId, + pid: owner.pid, + pidIdentity: owner.identity, + registeredAt: createdAt + }; + + ensurePrivateDir(registryRoot); + const preparedDir = path.join(registryRoot, `.${brokerKey}.${process.pid}.${randomUUID()}.prepared`); + fs.mkdirSync(preparedDir, { mode: 0o700 }); + try { + createImmutableJson(path.join(preparedDir, "broker.json"), broker); + createImmutableJson(path.join(preparedDir, "owners", `${owner.ownerKey}.json`), ownerRecord); + fs.renameSync(preparedDir, registration.registryDir); + requirePrivateDirectory(registration.registryDir); + return { ...registration, broker, ownerKey: owner.ownerKey }; + } finally { + if (fs.existsSync(preparedDir)) { + fs.rmSync(preparedDir, { recursive: true, force: true }); + } + } +} + export function registerBrokerOwner(registration, options = {}) { const { env = process.env, now = () => new Date().toISOString() } = options; if (!validRegistrationReference(registration)) { diff --git a/plugins/codex/scripts/lib/codex.mjs b/plugins/codex/scripts/lib/codex.mjs index fead00cc4..de474a4a1 100644 --- a/plugins/codex/scripts/lib/codex.mjs +++ b/plugins/codex/scripts/lib/codex.mjs @@ -41,7 +41,7 @@ import path from "node:path"; import { readJsonFile } from "./fs.mjs"; import { BROKER_BUSY_RPC_CODE, BROKER_ENDPOINT_ENV, CodexAppServerClient } from "./app-server.mjs"; -import { loadBrokerSession } from "./broker-lifecycle.mjs"; +import { loadReusableBrokerSession } from "./broker-lifecycle.mjs"; import { binaryAvailable } from "./process.mjs"; const SERVICE_NAME = "claude_code_codex_plugin"; @@ -904,7 +904,7 @@ export function getCodexAvailability(cwd) { } export function getSessionRuntimeStatus(env = process.env, cwd = process.cwd()) { - const endpoint = env?.[BROKER_ENDPOINT_ENV] ?? loadBrokerSession(cwd)?.endpoint ?? null; + const endpoint = env?.[BROKER_ENDPOINT_ENV] ?? loadReusableBrokerSession(cwd, env)?.endpoint ?? null; if (endpoint) { return { mode: "shared", diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 70680d2a6..df3fc0d81 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -228,6 +228,19 @@ export function getProcessIdentity(pid, options = {}) { return readUnixProcessTable(options.runCommandImpl ?? runCommand, options).get(pid)?.identity ?? null; } +export function hasLiveProcessIdentity(pid, identity, options = {}) { + if ( + !Number.isFinite(pid) || + typeof identity !== "string" || + !identity || + (options.platform ?? process.platform) === "win32" + ) { + return false; + } + const current = readUnixProcessTable(options.runCommandImpl ?? runCommand, options).get(pid); + return isRunningProcess(current) && current.identity === identity; +} + export function getLiveProcessPids(pids, options = {}) { const candidates = [...new Set((pids ?? []).filter((pid) => Number.isFinite(pid)))]; const expectedIdentities = identitiesByPid(options.identities); diff --git a/tests/broker-ownership.test.mjs b/tests/broker-ownership.test.mjs index 41379b9f6..a13b9640f 100644 --- a/tests/broker-ownership.test.mjs +++ b/tests/broker-ownership.test.mjs @@ -10,6 +10,7 @@ import { loadBrokerChildren, publishBrokerChild, publishBrokerRegistration, + publishRegisteredBroker, registerBrokerOwner, releaseBrokerChild, releaseBrokerOwner, @@ -58,6 +59,73 @@ function ownerEnv(env, sessionId, pid, startedAt) { }; } +test("initial broker and owner records become visible atomically", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-atomic-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-atomic", 5050, "Mon Jul 27 00:00:45 2026"); + const brokerIdentity = "4100@Mon Jul 27 00:00:00 2026"; + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: brokerIdentity, + processGroupId: 4100, + members: [] + }, + env, + now: () => "2026-07-27T00:00:00.000Z", + hasLiveProcessIdentityImpl: (pid, identity) => pid === 5050 && identity === "5050@Mon Jul 27 00:00:45 2026" + }); + + assert.equal(registration.registered, true); + assert.equal(fs.existsSync(path.join(registration.registryDir, "broker.json")), true); + assert.equal(fs.readdirSync(path.join(registration.registryDir, "owners")).length, 1); + assert.equal(fs.readdirSync(registration.registryRoot).some((name) => name.endsWith(".prepared")), false); +}); + +test("missing owner identity prevents any broker registry publication", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-no-owner-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: "4100@Mon Jul 27 00:00:00 2026", + processGroupId: 4100, + members: [] + }, + env: { CLAUDE_PLUGIN_DATA: root } + }); + + assert.deepEqual(registration, { registered: false, reason: "session-owner-unavailable" }); + assert.equal(fs.existsSync(path.join(root, "state", "broker-ownership-v1")), false); +}); + +test("stale owner identity prevents any broker registry publication", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-stale-owner-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: "4100@Mon Jul 27 00:00:00 2026", + processGroupId: 4100, + members: [] + }, + env: ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-stale", 5050, "Mon Jul 27 00:00:45 2026"), + hasLiveProcessIdentityImpl: () => false + }); + + assert.deepEqual(registration, { registered: false, reason: "session-owner-not-live" }); + assert.equal(fs.existsSync(path.join(root, "state", "broker-ownership-v1")), false); +}); + test("owner publication fails closed while cleanup holds the registry lock", (t) => { const { registration, env } = makeFixture(t); const lock = acquireBrokerRegistryLock(registration, { diff --git a/tests/process.test.mjs b/tests/process.test.mjs index d0576a97e..1ec5d6a3d 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -1,7 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { captureStableSessionOwner, getLiveProcessPids, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; +import { captureStableSessionOwner, getLiveProcessPids, hasLiveProcessIdentity, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; test("captureStableSessionOwner records the hook process-group leader", () => { const owner = captureStableSessionOwner(7101, { @@ -48,6 +48,26 @@ test("captureStableSessionOwner refuses a hook that is its own process-group lea assert.equal(owner, null); }); +test("hasLiveProcessIdentity excludes a matching zombie process", () => { + const identity = "7300@Mon Jul 27 00:09:00 2026"; + const live = hasLiveProcessIdentity(7300, identity, { + platform: "darwin", + runCommandImpl(command, args) { + return { + command, + args, + status: 0, + signal: null, + stdout: "7300 1 7300 Z Mon Jul 27 00:09:00 2026\n", + stderr: "", + error: null + }; + } + }); + + assert.equal(live, false); +}); + test("terminateProcessTree uses taskkill on Windows", async () => { let captured = null; const outcome = await terminateProcessTree(1234, { diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 40f048c6e..073ba9d3a 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -1,4 +1,5 @@ import fs from "node:fs"; +import net from "node:net"; import path from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; @@ -11,8 +12,8 @@ import { enqueueBackgroundTask, handleCancel, handleTaskWorker } from "../plugin import { cleanupSessionJobs, handleSessionEnd, handleSessionStart } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; import { isBrokerRequestAllowedDuringShutdown } from "../plugins/codex/scripts/app-server-broker.mjs"; -import { SESSION_OWNER_IDENTITY_ENV, SESSION_OWNER_PID_ENV } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; -import { clearBrokerSession, ensureBrokerSession, loadBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { acquireBrokerRegistryLock, loadBrokerRegistration, registerBrokerOwner, releaseBrokerRegistryLock, SESSION_OWNER_IDENTITY_ENV, SESSION_OWNER_PID_ENV } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; +import { acquireBrokerLaunchLock, activateBrokerProcess, brokerLaunchLockPort, clearBrokerSession, ensureBrokerSession, loadBrokerSession, loadReusableBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession, waitForBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { readStoredJob } from "../plugins/codex/scripts/lib/job-control.mjs"; import { captureProcessOwnership, getProcessIdentity, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; import { hasCancelFlag, listJobs, loadState, resolveStateDir, saveState, upsertJob, writeCancelFlag, writeJobFile } from "../plugins/codex/scripts/lib/state.mjs"; @@ -23,11 +24,13 @@ const PLUGIN_ROOT = path.join(ROOT, "plugins", "codex"); const SCRIPT = path.join(PLUGIN_ROOT, "scripts", "codex-companion.mjs"); const STOP_HOOK = path.join(PLUGIN_ROOT, "scripts", "stop-review-gate-hook.mjs"); const SESSION_HOOK = path.join(PLUGIN_ROOT, "scripts", "session-lifecycle-hook.mjs"); +const BROKER_SCRIPT = path.join(PLUGIN_ROOT, "scripts", "app-server-broker.mjs"); const DETACHED_FIXTURE_TTL_MS = 5 * 60 * 1000; const SELF_EXPIRING_KEEPALIVE = selfExpiringKeepaliveCode(); const runtimeTempDirs = new Set(); const runtimePluginDataDir = createTempDir("codex-plugin-runtime-state-"); process.env.CLAUDE_PLUGIN_DATA = runtimePluginDataDir; +let brokerOwnerSequence = 0; function makeTempDir(prefix) { const tempDir = createTempDir(prefix); @@ -35,6 +38,16 @@ function makeTempDir(prefix) { return tempDir; } +function withBrokerOwner(env, label) { + brokerOwnerSequence += 1; + return { + ...env, + CODEX_COMPANION_SESSION_ID: `runtime-${label}-${brokerOwnerSequence}`, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: getProcessIdentity(process.pid) + }; +} + function selfExpiringKeepaliveCode(ttlMs = DETACHED_FIXTURE_TTL_MS) { return `setTimeout(() => process.exit(0), ${ttlMs}); setInterval(() => {}, 1000)`; } @@ -499,10 +512,10 @@ test("shared broker reclaims a post-snapshot helper before allowing replacement" const binDir = makeTempDir(); const fakeStatePath = path.join(binDir, "fake-codex-state.json"); installFakeCodex(binDir, "crash-with-post-snapshot-helper"); - const env = { + const env = withBrokerOwner({ ...buildEnv(binDir), CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" - }; + }, "post-snapshot-helper"); const brokerSession = await ensureBrokerSession(repo, { env }); if (!brokerSession) { t.skip("broker socket unavailable in this sandbox"); @@ -653,6 +666,595 @@ test("automatic broker registration preserves a shared broker until its final ow }); }); +test("existing broker reuse holds the registry lock through owner publication", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const baseEnv = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: getProcessIdentity(process.pid) + }; + const envA = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "locked-reuse-a" }; + const envB = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "locked-reuse-b" }; + const first = await ensureBrokerSession(repo, { env: envA }); + let heldLock = null; + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envB, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "locked-reuse-b" }) + }); + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envA, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "locked-reuse-a" }) + }); + }); + + const second = await ensureBrokerSession(repo, { + env: envB, + acquireBrokerRegistryLockImpl(registration) { + heldLock = acquireBrokerRegistryLock(registration); + return heldLock; + }, + loadBrokerRegistrationImpl(args) { + assert.equal(heldLock?.acquired, true); + return loadBrokerRegistration(args); + }, + registerBrokerOwnerImpl(registration, options) { + assert.equal(options.registryLock, heldLock); + return registerBrokerOwner(registration, options); + }, + releaseBrokerRegistryLockImpl(registration, lock) { + assert.equal(lock, heldLock); + const released = releaseBrokerRegistryLock(registration, lock); + heldLock = null; + return released; + } + }); + + assert.equal(second.pid, first.pid); + assert.equal(heldLock, null); +}); + +test("deterministic broker launch lock never splits across fallback resources", async (t) => { + const repo = makeTempDir(); + const port = brokerLaunchLockPort(repo); + const foreignServer = net.createServer((socket) => socket.end("foreign-service\n")); + await new Promise((resolve, reject) => { + foreignServer.once("error", reject); + foreignServer.listen({ host: "127.0.0.1", port, exclusive: true }, resolve); + }); + t.after(async () => { + if (foreignServer.listening) { + await new Promise((resolve) => foreignServer.close(resolve)); + } + }); + + await assert.rejects( + acquireBrokerLaunchLock(repo, { timeoutMs: 250 }), + (error) => error?.code === "BROKER_LAUNCH_LOCK_UNAVAILABLE" + ); + await new Promise((resolve) => foreignServer.close(resolve)); + + const first = await acquireBrokerLaunchLock(repo, { timeoutMs: 250 }); + assert.equal(first.port, port); + try { + await assert.rejects( + acquireBrokerLaunchLock(repo, { timeoutMs: 75 }), + (error) => error?.code === "BROKER_LAUNCH_LOCK_TIMEOUT" + ); + } finally { + await first.release(); + } + const next = await acquireBrokerLaunchLock(repo, { timeoutMs: 250 }); + assert.equal(next.port, port); + await next.release(); +}); + +test("automatic broker falls back to direct mode when its launch lock cannot bind", async () => { + const repo = makeTempDir(); + const env = withBrokerOwner(process.env, "launch-lock-bind-denied"); + let endpointCreated = false; + + const session = await ensureBrokerSession(repo, { + env, + async acquireBrokerLaunchLockImpl() { + const error = new Error("loopback bind denied"); + error.code = "EACCES"; + throw error; + }, + createBrokerEndpoint() { + endpointCreated = true; + return "unix:/unused.sock"; + } + }); + + assert.equal(session, null); + assert.equal(endpointCreated, false); +}); + +test("automatic and explicit reuse paths refuse a live unregistered endpoint", async (t) => { + if (process.platform === "win32") { + t.skip("Unix broker sockets are required for this contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const socketPath = path.join("/private/tmp", `cxc-unregistered-${process.pid}-${Date.now()}.sock`); + const endpoint = `unix:${socketPath}`; + let brokerConnections = 0; + const server = net.createServer((socket) => { + brokerConnections += 1; + socket.end(); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, resolve); + }); + saveBrokerSession(repo, { endpoint, registry: null }); + t.after(async () => { + await new Promise((resolve) => server.close(resolve)); + clearBrokerSession(repo); + if (fs.existsSync(socketPath)) { + fs.unlinkSync(socketPath); + } + }); + + const directClient = await CodexAppServerClient.connect(repo, { + reuseExistingBroker: true, + env: buildEnv(binDir) + }); + await directClient.close(); + assert.equal(brokerConnections, 0); + + assert.equal( + await ensureBrokerSession(repo, { + env: withBrokerOwner(process.env, "unregistered-refusal") + }), + null + ); + assert.equal(loadBrokerSession(repo)?.endpoint, endpoint); + assert.equal(brokerConnections, 0); +}); + +test("invalid legacy broker state falls back to direct mode without deleting evidence", async () => { + const repo = makeTempDir(); + const stateDir = resolveStateDir(repo); + fs.mkdirSync(stateDir, { recursive: true }); + const statePath = path.join(stateDir, "broker.json"); + fs.writeFileSync(statePath, "{truncated\n", "utf8"); + + const session = await ensureBrokerSession(repo, { + env: withBrokerOwner(process.env, "invalid-legacy-state") + }); + + assert.equal(session, null); + assert.equal(fs.readFileSync(statePath, "utf8"), "{truncated\n"); + fs.unlinkSync(statePath); +}); + +test("automatic broker rolls back when the broker registration cannot be published", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + const socketPath = path.join("/private/tmp", `cxc-publish-failure-${process.pid}-${Date.now()}.sock`); + installFakeCodex(binDir, "review-ok"); + const ownerIdentity = getProcessIdentity(process.pid); + const env = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + CODEX_COMPANION_SESSION_ID: "publish-failure-owner", + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: ownerIdentity + }; + let brokerPid = null; + + await assert.rejects( + ensureBrokerSession(repo, { + env, + createBrokerEndpoint: () => `unix:${socketPath}`, + captureProcessOwnershipImpl(pid, options) { + brokerPid = pid; + return captureProcessOwnership(pid, options); + }, + publishRegisteredBrokerImpl() { + throw new Error("injected broker registration failure"); + } + }), + (error) => error?.code === "BROKER_REGISTRATION_FAILED" + ); + + assert.ok(Number.isFinite(brokerPid)); + await waitFor(() => { + try { + process.kill(brokerPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + assert.equal(loadBrokerSession(repo), null); + assert.equal(fs.existsSync(socketPath), false); +}); + +test("automatic broker rolls back when owner, state, or activation publication fails", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const ownerIdentity = getProcessIdentity(process.pid); + for (const failure of ["owner", "state", "activation"]) { + const repo = makeTempDir(); + const binDir = makeTempDir(); + const socketPath = path.join("/private/tmp", `cxc-${failure}-failure-${process.pid}-${Date.now()}.sock`); + installFakeCodex(binDir, "review-ok"); + const env = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + CODEX_COMPANION_SESSION_ID: `${failure}-failure-owner`, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: ownerIdentity + }; + let brokerPid = null; + + await assert.rejects( + ensureBrokerSession(repo, { + env, + createBrokerEndpoint: () => `unix:${socketPath}`, + captureProcessOwnershipImpl(pid, options) { + brokerPid = pid; + return captureProcessOwnership(pid, options); + }, + publishRegisteredBrokerImpl: failure === "owner" + ? () => ({ registered: false, reason: "injected-owner-failure" }) + : undefined, + saveBrokerSessionImpl: failure === "state" + ? () => { + throw new Error("injected state publication failure"); + } + : undefined, + activateBrokerProcessImpl: failure === "activation" + ? async () => { + throw new Error("injected activation failure"); + } + : undefined + }), + (error) => error?.code === "BROKER_REGISTRATION_FAILED" + ); + + assert.ok(Number.isFinite(brokerPid)); + await waitFor(() => { + try { + process.kill(brokerPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + assert.equal(loadBrokerSession(repo), null); + assert.equal(fs.existsSync(socketPath), false); + } +}); + +test("a failed activation recovery marker is never reusable", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir + }, "failed-activation-marker"); + const unverifiedCleanup = async () => ({ + attempted: true, + delivered: false, + verified: false, + degraded: true, + survivors: [] + }); + + await assert.rejects( + ensureBrokerSession(repo, { + env, + rollbackExitTimeoutMs: 0, + async activateBrokerProcessImpl(child) { + await activateBrokerProcess(child); + throw new Error("injected post-activation publication failure"); + }, + killProcess: unverifiedCleanup + }), + (error) => error?.code === "BROKER_CLEANUP_UNVERIFIED" + ); + + const failedSession = loadBrokerSession(repo); + t.after(async () => { + await sendBrokerShutdown(failedSession?.endpoint).catch(() => {}); + if (Number.isFinite(failedSession?.pid)) { + await terminateProcessTree(failedSession.pid, { + expectedRootIdentity: failedSession.pidIdentity, + ownershipSnapshot: failedSession.ownershipSnapshot + }).catch(() => {}); + } + clearBrokerSession(repo); + }); + assert.equal(failedSession?.activationFailed, true); + assert.equal(loadReusableBrokerSession(repo, env), null); + assert.doesNotThrow(() => process.kill(failedSession.pid, 0)); + + await assert.rejects( + ensureBrokerSession(repo, { env, killProcess: unverifiedCleanup }), + (error) => error?.code === "BROKER_CLEANUP_UNVERIFIED" + ); + assert.equal(loadBrokerSession(repo)?.pid, failedSession.pid); + assert.equal(loadBrokerSession(repo)?.activationFailed, true); +}); + +test("a broker remains non-reusable until activation is acknowledged", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir + }, "pending-activation"); + let releaseActivation; + const activationGate = new Promise((resolve) => { + releaseActivation = resolve; + }); + let session; + const launch = ensureBrokerSession(repo, { + env, + async activateBrokerProcessImpl(child) { + await activationGate; + await activateBrokerProcess(child); + } + }); + t.after(async () => { + releaseActivation?.(); + session ??= await launch.catch(() => null); + await sendBrokerShutdown(session?.endpoint).catch(() => {}); + if (Number.isFinite(session?.pid)) { + await terminateProcessTree(session.pid, { + expectedRootIdentity: session.pidIdentity, + ownershipSnapshot: session.ownershipSnapshot + }).catch(() => {}); + } + clearBrokerSession(repo); + }); + + await waitFor(() => loadBrokerSession(repo)?.activationPending === true); + assert.equal(loadReusableBrokerSession(repo, env), null); + + releaseActivation(); + session = await launch; + assert.equal(loadBrokerSession(repo)?.activationPending, false); + assert.equal(loadReusableBrokerSession(repo, env)?.pid, session.pid); +}); + +test("a registered broker with a missing socket is not reusable", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir + }, "missing-socket"); + const session = await ensureBrokerSession(repo, { env }); + t.after(async () => { + if (Number.isFinite(session?.pid)) { + await terminateProcessTree(session.pid, { + expectedRootIdentity: session.pidIdentity, + ownershipSnapshot: session.ownershipSnapshot + }).catch(() => {}); + } + clearBrokerSession(repo); + }); + + assert.equal(loadReusableBrokerSession(repo, env)?.pid, session.pid); + fs.unlinkSync(session.endpoint.slice("unix:".length)); + assert.equal(loadReusableBrokerSession(repo, env), null); +}); + +test("automatic broker falls back without leaving a process when session ownership is unavailable", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + const socketPath = path.join("/private/tmp", `cxc-owner-unavailable-${process.pid}-${Date.now()}.sock`); + installFakeCodex(binDir, "review-ok"); + let endpointFactoryCalled = false; + let brokerPid = null; + + const session = await ensureBrokerSession(repo, { + env: buildEnv(binDir), + createBrokerEndpoint: () => { + endpointFactoryCalled = true; + return `unix:${socketPath}`; + }, + captureProcessOwnershipImpl(pid, options) { + brokerPid = pid; + return captureProcessOwnership(pid, options); + } + }); + + assert.equal(session, null); + assert.equal(endpointFactoryCalled, false); + assert.equal(brokerPid, null); + assert.equal(loadBrokerSession(repo), null); + assert.equal(fs.existsSync(socketPath), false); +}); + +test("concurrent automatic broker launches converge on one registered process", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const ownerIdentity = getProcessIdentity(process.pid); + const baseEnv = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: ownerIdentity + }; + const snapshots = []; + const capture = (pid, options) => { + const snapshot = captureProcessOwnership(pid, options); + snapshots.push(snapshot); + return snapshot; + }; + t.after(async () => { + for (const snapshot of snapshots) { + await terminateProcessTree(snapshot.rootPid, { + expectedRootIdentity: snapshot.rootIdentity, + ownershipSnapshot: snapshot + }).catch(() => {}); + } + }); + + const [first, second] = await Promise.all([ + ensureBrokerSession(repo, { + env: { ...baseEnv, CODEX_COMPANION_SESSION_ID: "concurrent-owner-a" }, + captureProcessOwnershipImpl: capture + }), + ensureBrokerSession(repo, { + env: { ...baseEnv, CODEX_COMPANION_SESSION_ID: "concurrent-owner-b" }, + captureProcessOwnershipImpl: capture + }) + ]); + + assert.equal(first.pid, second.pid); + assert.equal(snapshots.length, 1); + assert.equal(loadBrokerSession(repo)?.pid, first.pid); + assert.equal(fs.readdirSync(path.join(first.registry.registryDir, "owners")).filter((name) => name.endsWith(".json")).length, 2); +}); + +test("an unreachable registered broker is never terminated while an owner is live", async (t) => { + if (process.platform === "win32") { + t.skip("Unix broker sockets are required for this contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const ownerIdentity = getProcessIdentity(process.pid); + const baseEnv = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: ownerIdentity + }; + const first = await ensureBrokerSession(repo, { + env: { ...baseEnv, CODEX_COMPANION_SESSION_ID: "unreachable-live-owner-a" } + }); + t.after(async () => { + await terminateProcessTree(first.pid, { + expectedRootIdentity: first.pidIdentity, + ownershipSnapshot: first.ownershipSnapshot + }).catch(() => {}); + clearBrokerSession(repo); + }); + + const socketPath = first.endpoint.slice("unix:".length); + fs.unlinkSync(socketPath); + const second = await ensureBrokerSession(repo, { + env: { ...baseEnv, CODEX_COMPANION_SESSION_ID: "unreachable-live-owner-b" } + }); + + assert.equal(second, null); + assert.doesNotThrow(() => process.kill(first.pid, 0)); + assert.equal(loadBrokerSession(repo)?.pid, first.pid); +}); + +test("pre-activation broker exits when its launcher pipe closes", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process groups are required for this contract."); + return; + } + + const repo = makeTempDir(); + const sessionDir = makeTempDir("cxc-launcher-exit-"); + const socketPath = path.join("/private/tmp", `cxc-launcher-exit-${process.pid}-${Date.now()}.sock`); + const endpoint = `unix:${socketPath}`; + const pidFile = path.join(sessionDir, "broker.pid"); + const child = spawn( + process.execPath, + [BROKER_SCRIPT, "serve", "--endpoint", endpoint, "--cwd", repo, "--pid-file", pidFile, "--require-activation-stdin"], + { cwd: repo, detached: true, stdio: ["pipe", "ignore", "ignore"] } + ); + const ownershipSnapshot = captureProcessOwnership(child.pid, { cwd: repo }); + t.after(async () => { + child.stdin?.destroy(); + await terminateProcessTree(child.pid, { + expectedRootIdentity: ownershipSnapshot.rootIdentity, + ownershipSnapshot + }).catch(() => {}); + }); + + assert.equal(await waitForBrokerEndpoint(endpoint, 2000), true); + const preActivationResponse = await new Promise((resolve, reject) => { + const socket = net.createConnection({ path: socketPath }); + let buffer = ""; + socket.setEncoding("utf8"); + socket.on("connect", () => { + socket.write(`${JSON.stringify({ id: 1, method: "initialize", params: {} })}\n`); + }); + socket.on("data", (chunk) => { + buffer += chunk; + }); + socket.on("end", () => resolve(JSON.parse(buffer.trim()))); + socket.on("error", reject); + }); + assert.equal(preActivationResponse.error?.code, -32004); + child.stdin.end(); + await waitFor(() => { + try { + process.kill(child.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }, { timeoutMs: 2000 }); + assert.equal(fs.existsSync(socketPath), false); + assert.equal(fs.existsSync(pidFile), false); +}); + test("automatic broker creation stays disabled where registered ownership is unsupported", async () => { const repo = makeTempDir(); let endpointFactoryCalled = false; @@ -720,7 +1322,7 @@ test("broker shutdown completes without starting a second app-server", async (t) const binDir = makeTempDir(); const fakeStatePath = path.join(binDir, "fake-codex-state.json"); installFakeCodex(binDir, "with-resistant-helper"); - const env = buildEnv(binDir); + const env = withBrokerOwner(buildEnv(binDir), "broker-shutdown"); const brokerSocketPath = path.join("/private/tmp", `cxc-p1c-${process.pid}-${Date.now()}.sock`); const brokerSession = await ensureBrokerSession(repo, { env, @@ -1727,7 +2329,7 @@ test("task using the shared broker still completes when Codex spawns subagents", run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); - const env = buildEnv(binDir); + const env = withBrokerOwner(buildEnv(binDir), "subagent-task"); const review = run("node", [SCRIPT, "review"], { cwd: repo, env @@ -3107,7 +3709,7 @@ test("cancel sends turn interrupt to the shared app-server before killing a brok run("git", ["add", "README.md"], { cwd: repo }); run("git", ["commit", "-m", "init"], { cwd: repo }); - const env = buildEnv(binDir); + const env = withBrokerOwner(buildEnv(binDir), "cancel-interrupt"); const launched = run("node", [SCRIPT, "task", "--background", "--json", "investigate the flaky worker timeout"], { cwd: repo, env @@ -3504,7 +4106,7 @@ test("commands lazily start and reuse one shared app-server after first use", as run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); - const env = buildEnv(binDir); + const env = withBrokerOwner(buildEnv(binDir), "lazy-reuse"); const review = run("node", [SCRIPT, "review"], { cwd: repo, @@ -3543,10 +4145,10 @@ test("shared broker clears a disconnected stream after its request rejects", asy const fakeStatePath = path.join(binDir, "fake-codex-state.json"); installSlowRejectFakeCodex(binDir); - const env = { + const env = withBrokerOwner({ ...buildEnv(binDir), CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" - }; + }, "disconnected-stream"); t.after(() => { run("node", [SESSION_HOOK, "SessionEnd"], { cwd: repo, @@ -3601,10 +4203,10 @@ test("shared broker cleans up an app-server whose initialization returns an RPC const fakeStatePath = path.join(binDir, "fake-codex-state.json"); installInitializeErrorFakeCodex(binDir); - const env = { + const env = withBrokerOwner({ ...buildEnv(binDir), CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" - }; + }, "initialize-error"); const brokerSession = await ensureBrokerSession(repo, { env }); t.after(() => { if (brokerSession) { @@ -3679,10 +4281,10 @@ test("shared broker releases its idle app-server child and restarts it on demand run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); - const env = { + const env = withBrokerOwner({ ...buildEnv(binDir), CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" - }; + }, "idle-child"); const review = run("node", [SCRIPT, "review"], { cwd: repo, @@ -3787,10 +4389,10 @@ test("shared broker keeps active work alive after its client disconnects", async run("git", ["add", "README.md"], { cwd: repo }); run("git", ["commit", "-m", "init"], { cwd: repo }); - const env = { + const env = withBrokerOwner({ ...buildEnv(binDir), CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" - }; + }, "active-work"); t.after(() => { run("node", [SESSION_HOOK, "SessionEnd"], { cwd: repo, @@ -3845,7 +4447,7 @@ test("setup reuses an existing shared app-server without starting another one", run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); - const env = buildEnv(binDir); + const env = withBrokerOwner(buildEnv(binDir), "setup-reuse"); const review = run("node", [SCRIPT, "review"], { cwd: repo, @@ -3888,9 +4490,10 @@ test("status reports shared session runtime when a lazy broker is active", () => run("git", ["commit", "-m", "init"], { cwd: repo }); fs.writeFileSync(path.join(repo, "README.md"), "hello again\n"); + const env = withBrokerOwner(buildEnv(binDir), "status-runtime"); const review = run("node", [SCRIPT, "review"], { cwd: repo, - env: buildEnv(binDir) + env }); assert.equal(review.status, 0, review.stderr); @@ -3900,14 +4503,21 @@ test("status reports shared session runtime when a lazy broker is active", () => const result = run("node", [SCRIPT, "status"], { cwd: repo, - env: buildEnv(binDir) + env }); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /Session runtime: shared session/); + + const unownedResult = run("node", [SCRIPT, "status"], { + cwd: repo, + env: buildEnv(binDir) + }); + assert.equal(unownedResult.status, 0, unownedResult.stderr); + assert.match(unownedResult.stdout, /Session runtime: direct startup/); }); -test("setup and status honor --cwd when reading shared session runtime", () => { +test("setup and status ignore an unregistered saved runtime", () => { const targetWorkspace = makeTempDir(); const invocationWorkspace = makeTempDir(); @@ -3919,13 +4529,13 @@ test("setup and status honor --cwd when reading shared session runtime", () => { cwd: invocationWorkspace }); assert.equal(status.status, 0, status.stderr); - assert.match(status.stdout, /Session runtime: shared session/); + assert.match(status.stdout, /Session runtime: direct startup/); const setup = run("node", [SCRIPT, "setup", "--cwd", targetWorkspace, "--json"], { cwd: invocationWorkspace }); assert.equal(setup.status, 0, setup.stderr); const payload = JSON.parse(setup.stdout); - assert.equal(payload.sessionRuntime.mode, "shared"); - assert.equal(payload.sessionRuntime.endpoint, "unix:/tmp/fake-broker.sock"); + assert.equal(payload.sessionRuntime.mode, "direct"); + assert.equal(payload.sessionRuntime.endpoint, null); }); From 328bdd56f909166d8c37a8f8cf03a1083e653689 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Thu, 30 Jul 2026 02:24:51 -0700 Subject: [PATCH 23/29] [CC-5589] close broker ownership activation races Attributed-To: Codex --- plugins/codex/scripts/app-server-broker.mjs | 71 ++++--- plugins/codex/scripts/app-server-child.mjs | 84 ++++++++ .../scripts/lib/app-server-protocol.d.ts | 3 + plugins/codex/scripts/lib/app-server.mjs | 26 ++- .../codex/scripts/lib/broker-lifecycle.mjs | 34 +++- .../codex/scripts/lib/broker-ownership.mjs | 50 ++++- plugins/codex/scripts/lib/codex.mjs | 15 +- plugins/codex/scripts/lib/process.mjs | 21 +- .../codex/scripts/session-lifecycle-hook.mjs | 18 +- tests/broker-ownership.test.mjs | 84 +++++++- tests/process.test.mjs | 9 +- tests/runtime.test.mjs | 189 +++++++++++++++--- 12 files changed, 529 insertions(+), 75 deletions(-) create mode 100644 plugins/codex/scripts/app-server-child.mjs diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 023e9c09d..7d0a85e07 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -7,7 +7,7 @@ import process from "node:process"; import { fileURLToPath } from "node:url"; import { parseArgs } from "./lib/args.mjs"; -import { BROKER_BUSY_RPC_CODE, CodexAppServerClient } from "./lib/app-server.mjs"; +import { BROKER_BUSY_RPC_CODE, BROKER_OWNERSHIP_RPC_CODE, BROKER_STREAM_COMPLETED_METHOD, CodexAppServerClient } from "./lib/app-server.mjs"; import { loadBrokerRegistration, publishBrokerChild, releaseBrokerChild } from "./lib/broker-ownership.mjs"; import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs"; import { getLiveProcessPids, getProcessIdentity } from "./lib/process.mjs"; @@ -122,21 +122,20 @@ async function main() { const sockets = new Set(); function getBrokerRegistration() { - if (brokerRegistration?.registered === true) { - return brokerRegistration; - } try { const brokerIdentity = getProcessIdentity(process.pid); if (brokerIdentity) { const candidate = loadBrokerRegistration({ endpoint, brokerIdentity }); if (candidate.registered === true) { brokerRegistration = candidate; + return brokerRegistration; } } } catch { // Missing registry evidence keeps the child outside automated cleanup. } - return brokerRegistration; + brokerRegistration = null; + return null; } function cancelChildIdleClose() { @@ -319,32 +318,37 @@ async function main() { ensureCleanupSafeToSpawn(); } if (!appClientStartPromise) { - appClientStartPromise = CodexAppServerClient.connect(cwd, { disableBroker: true }) + let childRegistration = null; + let registration = null; + appClientStartPromise = CodexAppServerClient.connect(cwd, { + disableBroker: true, + gatedBrokerChild: true, + async beforeAppServerActivation(ownershipSnapshot) { + registration = getBrokerRegistration(); + if (registration?.registered !== true) { + const error = new Error("Shared Codex broker ownership registration is unavailable."); + error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + throw error; + } + childRegistration = publishBrokerChild(registration, { ownershipSnapshot }); + if (childRegistration.registered !== true) { + const error = new Error(`Unable to register shared Codex app-server ownership (${childRegistration.reason ?? "unknown"}).`); + error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + throw error; + } + } + }) .then(async (client) => { - const registration = getBrokerRegistration(); - let childRegistration = null; - if (registration?.registered === true) { - try { - childRegistration = client.ownershipSnapshot - ? publishBrokerChild(registration, { ownershipSnapshot: client.ownershipSnapshot }) - : { registered: false, reason: "child-identity-unavailable" }; - } catch (error) { - await client.close().catch(() => {}); - throw error; - } - if (childRegistration.registered !== true) { - await client.close().catch(() => {}); - const error = new Error(`Unable to register shared Codex app-server ownership (${childRegistration.reason ?? "unknown"}).`); - error.rpcCode = BROKER_SHUTDOWN_RPC_CODE; - throw error; - } + if (registration?.registered !== true || childRegistration?.registered !== true) { + await client.close().catch(() => {}); + const error = new Error("Shared Codex app-server activation lost its durable ownership registration."); + error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + throw error; } let registryChild = null; appClient = client; - if (registration?.registered === true) { - registryChild = childRegistration.child; - appClientRegistryChild = registryChild; - } + registryChild = childRegistration.child; + appClientRegistryChild = registryChild; client.setNotificationHandler(routeNotification); const childPid = client.proc?.pid ?? null; client.setExitHandler(() => { @@ -520,6 +524,19 @@ async function main() { continue; } + if (message.method === BROKER_STREAM_COMPLETED_METHOD && message.id === undefined) { + const threadId = message.params?.threadId ?? null; + if ( + activeStreamRunning && + activeStreamSocket === socket && + (!threadId || !activeStreamThreadIds || activeStreamThreadIds.has(threadId)) + ) { + clearStreamState(); + scheduleChildIdleClose(); + } + continue; + } + if (message.id !== undefined && message.method === "broker/shutdown") { shuttingDown = true; send(socket, { id: message.id, result: {} }); diff --git a/plugins/codex/scripts/app-server-child.mjs b/plugins/codex/scripts/app-server-child.mjs new file mode 100644 index 000000000..486544615 --- /dev/null +++ b/plugins/codex/scripts/app-server-child.mjs @@ -0,0 +1,84 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import process from "node:process"; +import { spawn } from "node:child_process"; + +const activation = fs.createReadStream(null, { fd: 3, autoClose: true }); +let activated = false; +let activationBuffer = ""; +let child = null; + +function exitWithoutChild(code = 1) { + if (child || process.exitCode != null) { + return; + } + process.exitCode = code; + process.stdin.resume(); +} + +function startAppServer() { + if (activated || child) { + return; + } + activated = true; + child = spawn("codex", ["app-server"], { + cwd: process.cwd(), + env: process.env, + detached: false, + stdio: ["pipe", "pipe", "pipe"], + shell: process.platform === "win32" ? (process.env.SHELL || true) : false, + windowsHide: true + }); + child.stdout.pipe(process.stdout); + child.stderr.pipe(process.stderr); + process.stdin.pipe(child.stdin); + child.on("error", (error) => { + process.stderr.write(`Unable to start codex app-server: ${error.message}\n`); + process.exit(1); + }); + // `close` follows both process exit and stdio closure, so the wrapper does + // not truncate the JSONL stream while forwarding the child's final bytes. + child.on("close", (code, signal) => { + process.exit(Number.isInteger(code) ? code : signal ? 1 : 0); + }); +} + +activation.setEncoding("utf8"); +activation.on("data", (chunk) => { + if (activated) { + return; + } + activationBuffer += chunk; + const newlineIndex = activationBuffer.indexOf("\n"); + if (newlineIndex === -1) { + return; + } + if (activationBuffer.slice(0, newlineIndex).trim() !== "activate") { + exitWithoutChild(1); + return; + } + startAppServer(); +}); +activation.on("end", () => { + if (!activated) { + exitWithoutChild(1); + } +}); +activation.on("error", () => { + if (!activated) { + exitWithoutChild(1); + } +}); + +process.stdin.on("end", () => { + if (!activated) { + exitWithoutChild(1); + return; + } + child?.stdin.end(); +}); +process.stdin.on("error", () => { + child?.stdin.destroy(); +}); +process.stdin.resume(); diff --git a/plugins/codex/scripts/lib/app-server-protocol.d.ts b/plugins/codex/scripts/lib/app-server-protocol.d.ts index f61a4588e..bbf57c1b5 100644 --- a/plugins/codex/scripts/lib/app-server-protocol.d.ts +++ b/plugins/codex/scripts/lib/app-server-protocol.d.ts @@ -54,6 +54,8 @@ export interface CodexAppServerClientOptions { brokerEndpoint?: string; disableBroker?: boolean; reuseExistingBroker?: boolean; + gatedBrokerChild?: boolean; + beforeAppServerActivation?: (ownershipSnapshot: unknown) => void | Promise; } export interface AppServerMethodMap { @@ -66,6 +68,7 @@ export interface AppServerMethodMap { "review/start": { params: ReviewStartParams; result: ReviewStartResponse }; "turn/start": { params: TurnStartParams; result: TurnStartResponse }; "turn/interrupt": { params: TurnInterruptParams; result: TurnInterruptResponse }; + "broker/stream-completed": { params: { threadId: string; turnId: string | null }; result: Record }; } export type AppServerMethod = keyof AppServerMethodMap; diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index 31f7c9eba..b5ce78198 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -12,6 +12,7 @@ import net from "node:net"; import process from "node:process"; import { spawn } from "node:child_process"; import readline from "node:readline"; +import { fileURLToPath } from "node:url"; import { parseBrokerEndpoint } from "./broker-endpoint.mjs"; import { ensureBrokerSession, loadReusableBrokerSession } from "./broker-lifecycle.mjs"; import { captureProcessOwnership, normalizeProcessCleanupOutcome, terminateProcessGroup, terminateProcessTree } from "./process.mjs"; @@ -19,9 +20,12 @@ import { captureProcessOwnership, normalizeProcessCleanupOutcome, terminateProce const PLUGIN_MANIFEST_URL = new URL("../../.claude-plugin/plugin.json", import.meta.url); const PLUGIN_MANIFEST = JSON.parse(fs.readFileSync(PLUGIN_MANIFEST_URL, "utf8")); const DEFAULT_CLOSE_WAIT_MS = 2000; +const GATED_APP_SERVER_CHILD = fileURLToPath(new URL("../app-server-child.mjs", import.meta.url)); export const BROKER_ENDPOINT_ENV = "CODEX_COMPANION_APP_SERVER_ENDPOINT"; export const BROKER_BUSY_RPC_CODE = -32001; +export const BROKER_OWNERSHIP_RPC_CODE = -32005; +export const BROKER_STREAM_COMPLETED_METHOD = "broker/stream-completed"; /** @type {ClientInfo} */ const DEFAULT_CLIENT_INFO = { @@ -194,11 +198,12 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { } async initialize() { - this.proc = spawn("codex", ["app-server"], { + const gated = this.options.gatedBrokerChild === true && process.platform !== "win32"; + this.proc = spawn(gated ? process.execPath : "codex", gated ? [GATED_APP_SERVER_CHILD] : ["app-server"], { cwd: this.cwd, env: this.options.env ?? process.env, detached: process.platform !== "win32", - stdio: ["pipe", "pipe", "pipe"], + stdio: gated ? ["pipe", "pipe", "pipe", "pipe"] : ["pipe", "pipe", "pipe"], shell: process.platform === "win32" ? (process.env.SHELL || true) : false, windowsHide: true }); @@ -232,10 +237,6 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { this.handleLine(line); }); - await this.request("initialize", { - clientInfo: this.options.clientInfo ?? DEFAULT_CLIENT_INFO, - capabilities: this.options.capabilities ?? DEFAULT_CAPABILITIES - }); try { const captureOwnership = this.options.captureProcessOwnershipImpl ?? captureProcessOwnership; this.ownershipSnapshot = @@ -254,6 +255,18 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { this.identityCaptureFailed = true; throw new Error("Unable to capture codex app-server process identity."); } + if (gated) { + await this.options.beforeAppServerActivation?.(this.ownershipSnapshot); + const activationControl = this.proc.stdio?.[3]; + if (!activationControl) { + throw new Error("Codex app-server activation control is unavailable."); + } + activationControl.end("activate\n"); + } + await this.request("initialize", { + clientInfo: this.options.clientInfo ?? DEFAULT_CLIENT_INFO, + capabilities: this.options.capabilities ?? DEFAULT_CAPABILITIES + }); this.notify("initialized", {}); } @@ -315,6 +328,7 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { } if (this.proc) { + this.proc.stdio?.[3]?.end?.(); try { this.proc.stdin.end(); } catch { diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index c897e1a3c..73c331146 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -642,6 +642,7 @@ async function ensureBrokerSessionLocked(cwd, options = {}) { let fallbackToDirect = false; let ownerRegistered = false; + let transactionRegistryLock = null; try { const publishRegistration = options.publishRegisteredBrokerImpl ?? publishRegisteredBroker; const candidate = publishRegistration({ @@ -649,18 +650,25 @@ async function ensureBrokerSessionLocked(cwd, options = {}) { endpoint, pid: child.pid ?? null, ownershipSnapshot, - env + env, + retainRegistryLock: true }); if (candidate.registered !== true) { fallbackToDirect = [ "broker-identity-unavailable", + "broker-not-live", "plugin-data-unavailable", "session-owner-not-live", "session-owner-unavailable" ].includes(candidate.reason); throw new Error(`Broker registration is unavailable (${candidate.reason ?? "unknown"}).`); } - session.registry = candidate; + if (candidate.registryLock?.acquired !== true) { + throw new Error("Broker registration did not retain the launch transaction lock."); + } + transactionRegistryLock = candidate.registryLock; + const { registryLock: _registryLock, ...registration } = candidate; + session.registry = registration; ownerRegistered = true; const saveSession = options.saveBrokerSessionImpl ?? saveBrokerSession; session.activationPending = true; @@ -672,10 +680,17 @@ async function ensureBrokerSessionLocked(cwd, options = {}) { saveSession(cwd, session); return session; } catch (error) { - if (ownerRegistered) { + // Keep the initial owner and the publication-time registry lock intact + // until exact rollback has converged. Otherwise a reaper can classify the + // half-launched broker as abandoned and race the launcher's own teardown. + const cleanup = await rollbackNewBrokerSession(cwd, child, session, options); + if (ownerRegistered && cleanup?.verified === true) { const releaseOwner = options.releaseBrokerOwnerImpl ?? releaseBrokerOwner; try { - const released = releaseOwner(session.registry, { env }); + const released = releaseOwner(session.registry, { + env, + ...(transactionRegistryLock ? { registryLock: transactionRegistryLock } : {}) + }); if (released.released !== true) { process.stderr.write(`Warning: unable to release rolled-back Codex broker owner (${released.reason ?? "unknown"}).\n`); } @@ -683,7 +698,6 @@ async function ensureBrokerSessionLocked(cwd, options = {}) { process.stderr.write(`Warning: unable to release rolled-back Codex broker owner: ${releaseError.message}.\n`); } } - const cleanup = await rollbackNewBrokerSession(cwd, child, session, options); if (cleanup?.verified !== true) { const cleanupError = new Error(`Broker launch failed and exact rollback is unverified: ${error.message}`); cleanupError.code = "BROKER_CLEANUP_UNVERIFIED"; @@ -697,6 +711,16 @@ async function ensureBrokerSessionLocked(cwd, options = {}) { transactionError.code = "BROKER_REGISTRATION_FAILED"; transactionError.cause = error; throw transactionError; + } finally { + if (transactionRegistryLock && session.registry) { + const releaseRegistryLock = options.releaseBrokerRegistryLockImpl ?? releaseBrokerRegistryLock; + const released = releaseRegistryLock(session.registry, transactionRegistryLock); + if (released?.released !== true) { + const lockError = new Error(`Broker launch transaction lock release failed (${released?.reason ?? "unknown"}).`); + lockError.code = "BROKER_CLEANUP_UNVERIFIED"; + throw lockError; + } + } } } diff --git a/plugins/codex/scripts/lib/broker-ownership.mjs b/plugins/codex/scripts/lib/broker-ownership.mjs index f2ced454b..841199061 100644 --- a/plugins/codex/scripts/lib/broker-ownership.mjs +++ b/plugins/codex/scripts/lib/broker-ownership.mjs @@ -146,6 +146,30 @@ function validRegistryLock(registration, lock) { ); } +function createPreparedRegistryLock(registration, preparedRegistryDir, options = {}) { + const pid = options.pid ?? process.pid; + if (!isSafePid(pid)) { + return { acquired: false, reason: "registry-lock-owner-invalid" }; + } + const token = randomUUID(); + const preparedLockPath = path.join(preparedRegistryDir, REGISTRY_LOCK_DIR_NAME); + ensurePrivateDir(preparedLockPath); + createImmutableJson(path.join(preparedLockPath, "owner.json"), { + version: BROKER_OWNERSHIP_VERSION, + kind: "registry-lock", + brokerKey: registration.brokerKey, + token, + pid, + acquiredAt: (options.now ?? (() => new Date().toISOString()))() + }); + return { + acquired: true, + brokerKey: registration.brokerKey, + token, + path: path.join(registration.registryDir, REGISTRY_LOCK_DIR_NAME) + }; +} + export function acquireBrokerRegistryLock( registration, { @@ -390,7 +414,8 @@ export function publishRegisteredBroker({ ownershipSnapshot, env = process.env, now = () => new Date().toISOString(), - hasLiveProcessIdentityImpl = hasLiveProcessIdentity + hasLiveProcessIdentityImpl = hasLiveProcessIdentity, + retainRegistryLock = false }) { const registryRoot = resolveBrokerOwnershipRoot(env); if (!registryRoot) { @@ -445,9 +470,30 @@ export function publishRegisteredBroker({ try { createImmutableJson(path.join(preparedDir, "broker.json"), broker); createImmutableJson(path.join(preparedDir, "owners", `${owner.ownerKey}.json`), ownerRecord); + const registryLock = retainRegistryLock + ? createPreparedRegistryLock(registration, preparedDir, { now }) + : null; + if (retainRegistryLock && registryLock?.acquired !== true) { + return { registered: false, reason: registryLock?.reason ?? "registry-lock-unavailable" }; + } + // Publication is the point at which the reaper can first observe this + // broker. Revalidate both identities after all prepared bytes exist and + // immediately before the atomic rename so the visible initial state never + // begins with a dead sole owner or a reused broker PID. + if (!hasLiveProcessIdentityImpl(owner.pid, owner.identity)) { + return { registered: false, reason: "session-owner-not-live" }; + } + if (!hasLiveProcessIdentityImpl(pid, ownershipSnapshot.rootIdentity)) { + return { registered: false, reason: "broker-not-live" }; + } fs.renameSync(preparedDir, registration.registryDir); requirePrivateDirectory(registration.registryDir); - return { ...registration, broker, ownerKey: owner.ownerKey }; + return { + ...registration, + broker, + ownerKey: owner.ownerKey, + ...(registryLock ? { registryLock } : {}) + }; } finally { if (fs.existsSync(preparedDir)) { fs.rmSync(preparedDir, { recursive: true, force: true }); diff --git a/plugins/codex/scripts/lib/codex.mjs b/plugins/codex/scripts/lib/codex.mjs index de474a4a1..5c9171c7b 100644 --- a/plugins/codex/scripts/lib/codex.mjs +++ b/plugins/codex/scripts/lib/codex.mjs @@ -20,6 +20,7 @@ * rejectCompletion: (error: unknown) => void, * finalTurn: Turn | null, * completed: boolean, + * inferredCompletion: boolean, * finalAnswerSeen: boolean, * pendingCollaborations: Set, * activeSubagentTurns: Set, @@ -40,7 +41,7 @@ import os from "node:os"; import path from "node:path"; import { readJsonFile } from "./fs.mjs"; -import { BROKER_BUSY_RPC_CODE, BROKER_ENDPOINT_ENV, CodexAppServerClient } from "./app-server.mjs"; +import { BROKER_BUSY_RPC_CODE, BROKER_ENDPOINT_ENV, BROKER_OWNERSHIP_RPC_CODE, BROKER_STREAM_COMPLETED_METHOD, CodexAppServerClient } from "./app-server.mjs"; import { loadReusableBrokerSession } from "./broker-lifecycle.mjs"; import { binaryAvailable } from "./process.mjs"; @@ -321,6 +322,7 @@ function createTurnCaptureState(threadId, options = {}) { rejectCompletion, finalTurn: null, completed: false, + inferredCompletion: false, finalAnswerSeen: false, pendingCollaborations: new Set(), activeSubagentTurns: new Set(), @@ -364,6 +366,7 @@ function completeTurn(state, turn = null, options = {}) { } if (options.inferred) { + state.inferredCompletion = true; emitProgress(state.onProgress, "Turn completion inferred after the main thread finished and subagent work drained.", "finalizing"); } @@ -603,7 +606,14 @@ async function captureTurn(client, threadId, startRequest, options = {}) { completeTurn(state, response.turn); } - return await state.completion; + const completedState = await state.completion; + if (completedState.inferredCompletion && client.transport === "broker") { + client.notify(BROKER_STREAM_COMPLETED_METHOD, { + threadId: completedState.threadId, + turnId: completedState.turnId + }); + } + return completedState; } finally { clearCompletionTimer(state); client.setNotificationHandler(previousHandler ?? null); @@ -621,6 +631,7 @@ async function withAppServer(cwd, fn) { const brokerRequested = client?.transport === "broker" || Boolean(process.env[BROKER_ENDPOINT_ENV]); const shouldRetryDirect = (client?.transport === "broker" && error?.rpcCode === BROKER_BUSY_RPC_CODE) || + (client?.transport === "broker" && error?.rpcCode === BROKER_OWNERSHIP_RPC_CODE) || (brokerRequested && (error?.code === "ENOENT" || error?.code === "ECONNREFUSED")); if (client) { diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index df3fc0d81..8a0b95b6e 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -430,11 +430,30 @@ function warnProcessCleanup(message, options) { function degradedDirectChildKill(pid, options, killImpl, reason) { const ownershipSnapshot = options.ownershipSnapshot ?? null; + const ownerHoldsLiveHandle = options.ownerHoldsLiveHandle === true; const canSignalOwnedGroup = - options.ownerHoldsLiveHandle === true && + ownerHoldsLiveHandle && ownershipSnapshot?.rootPid === pid && ownershipSnapshot?.processGroupId === pid && ownershipSnapshot?.rootIdentity === (options.expectedRootIdentity ?? ownershipSnapshot?.rootIdentity); + if (!ownerHoldsLiveHandle) { + warnProcessCleanup( + `Unable to verify Unix process cleanup for PID ${pid}; deferred signalling until process enumeration recovers (${String(reason).replace(/\s+/g, " ").trim()}). Surviving PIDs: ${pid}.`, + options + ); + return normalizeProcessCleanupOutcome({ + attempted: true, + delivered: false, + verified: false, + escalated: false, + degraded: true, + method: "deferred", + targets: [], + targetIdentities: options.expectedRootIdentity ? [options.expectedRootIdentity] : [], + survivors: [pid], + survivorIdentities: options.expectedRootIdentity ? [options.expectedRootIdentity] : [] + }); + } const directKillImpl = options.directKillImpl ?? ((signal) => killImpl(pid, signal)); let delivered = false; try { diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index e21f4c824..ba436744a 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -26,6 +26,7 @@ import { import { loadState, resolveStateFile, saveState, writeCancelFlag, writeJobFile } from "./lib/state.mjs"; import { TRANSCRIPT_PATH_ENV } from "./lib/claude-session-transfer.mjs"; import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; +import { runRegisteredBrokerReaper } from "./registered-broker-reaper.mjs"; export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; const PLUGIN_DATA_ENV = "CLAUDE_PLUGIN_DATA"; @@ -136,7 +137,7 @@ export async function cleanupSessionJobs(cwd, sessionId, dependencies = {}) { return { verified: retainedJobs.length === 0, failures: [] }; } -export function handleSessionStart(input, dependencies = {}) { +export async function handleSessionStart(input, dependencies = {}) { appendEnvVar(SESSION_ID_ENV, input.session_id); appendEnvVar(TRANSCRIPT_PATH_ENV, input.transcript_path); appendEnvVar(PLUGIN_DATA_ENV, process.env[PLUGIN_DATA_ENV]); @@ -157,6 +158,19 @@ export function handleSessionStart(input, dependencies = {}) { } catch { // Missing owner identity leaves this session in the report-only class. } + // A killed session cannot run SessionEnd. Reap only immutable registered + // ownership on the next harness start so crash residue converges without a + // process-name sweep, cron job, or manual cleanup. + try { + const reapRegistered = dependencies.runRegisteredBrokerReaperImpl ?? runRegisteredBrokerReaper; + await reapRegistered({ + mode: "apply-registered", + env: dependencies.env ?? process.env + }); + } catch (error) { + const warn = dependencies.warnImpl ?? ((message) => process.stderr.write(`${message}\n`)); + warn(`Registered Codex broker cleanup remains report-only: ${error instanceof Error ? error.message : String(error)}`); + } } export async function handleSessionEnd(input, dependencies = {}) { @@ -291,7 +305,7 @@ async function main() { const eventName = process.argv[2] ?? input.hook_event_name ?? ""; if (eventName === "SessionStart") { - handleSessionStart(input); + await handleSessionStart(input); return; } diff --git a/tests/broker-ownership.test.mjs b/tests/broker-ownership.test.mjs index a13b9640f..57ea67ca2 100644 --- a/tests/broker-ownership.test.mjs +++ b/tests/broker-ownership.test.mjs @@ -76,7 +76,9 @@ test("initial broker and owner records become visible atomically", (t) => { }, env, now: () => "2026-07-27T00:00:00.000Z", - hasLiveProcessIdentityImpl: (pid, identity) => pid === 5050 && identity === "5050@Mon Jul 27 00:00:45 2026" + hasLiveProcessIdentityImpl: (pid, identity) => + (pid === 5050 && identity === "5050@Mon Jul 27 00:00:45 2026") || + (pid === 4100 && identity === brokerIdentity) }); assert.equal(registration.registered, true); @@ -85,6 +87,32 @@ test("initial broker and owner records become visible atomically", (t) => { assert.equal(fs.readdirSync(registration.registryRoot).some((name) => name.endsWith(".prepared")), false); }); +test("initial publication can make its transaction lock visible atomically", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-atomic-lock-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const brokerIdentity = "4100@Mon Jul 27 00:00:00 2026"; + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: brokerIdentity, + processGroupId: 4100, + members: [] + }, + env: ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-atomic-lock", 5050, "Mon Jul 27 00:00:45 2026"), + retainRegistryLock: true, + hasLiveProcessIdentityImpl: () => true + }); + + assert.equal(registration.registered, true); + assert.equal(registration.registryLock?.acquired, true); + assert.equal(fs.existsSync(registration.registryLock.path), true); + assert.equal(acquireBrokerRegistryLock(registration).acquired, false); + assert.equal(releaseBrokerRegistryLock(registration, registration.registryLock).released, true); +}); + test("missing owner identity prevents any broker registry publication", (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-no-owner-")); t.after(() => fs.rmSync(root, { recursive: true, force: true })); @@ -126,6 +154,60 @@ test("stale owner identity prevents any broker registry publication", (t) => { assert.equal(fs.existsSync(path.join(root, "state", "broker-ownership-v1")), false); }); +test("initial publication revalidates the sole owner after preparing registry bytes", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-owner-revalidation-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-race", 5050, "Mon Jul 27 00:00:45 2026"); + let ownerChecks = 0; + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: "4100@Mon Jul 27 00:00:00 2026", + processGroupId: 4100, + members: [] + }, + env, + hasLiveProcessIdentityImpl(pid) { + if (pid === 5050) { + ownerChecks += 1; + return ownerChecks === 1; + } + return true; + } + }); + + assert.deepEqual(registration, { registered: false, reason: "session-owner-not-live" }); + assert.equal(fs.existsSync(path.join(root, "state", "broker-ownership-v1")), true); + assert.deepEqual(fs.readdirSync(path.join(root, "state", "broker-ownership-v1")), []); +}); + +test("initial publication revalidates the broker identity immediately before visibility", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-process-revalidation-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const env = ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-race", 5050, "Mon Jul 27 00:00:45 2026"); + const registration = publishRegisteredBroker({ + cwd: root, + endpoint: `unix:${path.join(root, "broker.sock")}`, + pid: 4100, + ownershipSnapshot: { + rootPid: 4100, + rootIdentity: "4100@Mon Jul 27 00:00:00 2026", + processGroupId: 4100, + members: [] + }, + env, + hasLiveProcessIdentityImpl(pid) { + return pid === 5050; + } + }); + + assert.deepEqual(registration, { registered: false, reason: "broker-not-live" }); + assert.deepEqual(fs.readdirSync(path.join(root, "state", "broker-ownership-v1")), []); +}); + test("owner publication fails closed while cleanup holds the registry lock", (t) => { const { registration, env } = makeFixture(t); const lock = acquireBrokerRegistryLock(registration, { diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 1ec5d6a3d..0225f12cd 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -315,7 +315,7 @@ test("terminateProcessTree parses a captured Linux procps process table", async assert.deepEqual(outcome.targets, [42002, 42001]); }); -test("terminateProcessTree falls back to a direct child kill when Unix process enumeration fails", async () => { +test("terminateProcessTree defers persisted cleanup when Unix process enumeration fails", async () => { const signals = []; const warnings = []; const outcome = await terminateProcessTree(1234, { @@ -342,12 +342,13 @@ test("terminateProcessTree falls back to a direct child kill when Unix process e } }); - assert.deepEqual(signals, [[1234, "SIGKILL"]]); + assert.deepEqual(signals, []); assert.equal(outcome.verified, false); assert.equal(outcome.degraded, true); - assert.deepEqual(outcome.survivors, []); + assert.equal(outcome.method, "deferred"); + assert.deepEqual(outcome.survivors, [1234]); assert.equal(warnings.length, 1); - assert.match(warnings[0], /direct-child kill fallback.*none known/i); + assert.match(warnings[0], /deferred signalling.*1234/i); }); test("terminateProcessTree preserves an owned detached group during degraded live-handle cleanup", async () => { diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 073ba9d3a..787a31476 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -12,10 +12,10 @@ import { enqueueBackgroundTask, handleCancel, handleTaskWorker } from "../plugin import { cleanupSessionJobs, handleSessionEnd, handleSessionStart } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; import { isBrokerRequestAllowedDuringShutdown } from "../plugins/codex/scripts/app-server-broker.mjs"; -import { acquireBrokerRegistryLock, loadBrokerRegistration, registerBrokerOwner, releaseBrokerRegistryLock, SESSION_OWNER_IDENTITY_ENV, SESSION_OWNER_PID_ENV } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; +import { acquireBrokerRegistryLock, loadBrokerChildren, loadBrokerRegistration, publishRegisteredBroker, registerBrokerOwner, releaseBrokerOwner, releaseBrokerRegistryLock, SESSION_OWNER_IDENTITY_ENV, SESSION_OWNER_PID_ENV } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; import { acquireBrokerLaunchLock, activateBrokerProcess, brokerLaunchLockPort, clearBrokerSession, ensureBrokerSession, loadBrokerSession, loadReusableBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession, waitForBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { readStoredJob } from "../plugins/codex/scripts/lib/job-control.mjs"; -import { captureProcessOwnership, getProcessIdentity, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; +import { captureProcessOwnership, getProcessIdentity, hasLiveProcessIdentity, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; import { hasCancelFlag, listJobs, loadState, resolveStateDir, saveState, upsertJob, writeCancelFlag, writeJobFile } from "../plugins/codex/scripts/lib/state.mjs"; import { runTrackedJob } from "../plugins/codex/scripts/lib/tracked-jobs.mjs"; @@ -90,12 +90,13 @@ test("broker rejects queued work after shutdown begins", () => { assert.equal(isBrokerRequestAllowedDuringShutdown(false, { id: 4, method: "thread/list" }), true); }); -test("SessionStart exports the stable process-group owner identity", () => { +test("SessionStart exports the stable process-group owner identity and runs registered cleanup", async () => { const envFile = path.join(makeTempDir(), "session.env"); const previousEnvFile = process.env.CLAUDE_ENV_FILE; process.env.CLAUDE_ENV_FILE = envFile; try { - handleSessionStart( + let reaperMode = null; + await handleSessionStart( { session_id: "session-owner-export", transcript_path: "/tmp/transcript.jsonl", @@ -110,9 +111,14 @@ test("SessionStart exports the stable process-group owner identity", () => { identity: "7100@Mon Jul 27 00:07:00 2026", processGroupId: 7100 }; + }, + async runRegisteredBrokerReaperImpl(options) { + reaperMode = options.mode; + return { mode: options.mode, scanned: 0, reaped: 0, reported: 0, results: [] }; } } ); + assert.equal(reaperMode, "apply-registered"); } finally { if (previousEnvFile == null) { delete process.env.CLAUDE_ENV_FILE; @@ -951,6 +957,54 @@ test("automatic broker rolls back when owner, state, or activation publication f } }); +test("automatic broker rollback remains locked until exact cleanup converges", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir + }, "locked-rollback"); + let registration = null; + let brokerPid = null; + let releaseObserved = false; + + await assert.rejects( + ensureBrokerSession(repo, { + env, + rollbackExitTimeoutMs: 0, + captureProcessOwnershipImpl(pid, options) { + brokerPid = pid; + return captureProcessOwnership(pid, options); + }, + publishRegisteredBrokerImpl(options) { + registration = publishRegisteredBroker(options); + return registration; + }, + saveBrokerSessionImpl() { + throw new Error("injected state publication failure"); + }, + releaseBrokerOwnerImpl(lockedRegistration, options) { + assert.equal(options.registryLock?.acquired, true); + assert.equal(fs.existsSync(options.registryLock.path), true); + assert.equal(acquireBrokerRegistryLock(lockedRegistration).acquired, false); + assert.equal(hasLiveProcessIdentity(brokerPid, registration.broker.pidIdentity), false); + releaseObserved = true; + return releaseBrokerOwner(lockedRegistration, options); + } + }), + (error) => error?.code === "BROKER_REGISTRATION_FAILED" + ); + + assert.equal(releaseObserved, true); + assert.equal(fs.existsSync(registration.registryLock.path), false); +}); + test("a failed activation recovery marker is never reusable", async (t) => { if (process.platform === "win32") { t.skip("Unix process identities are required for the registered broker contract."); @@ -1000,10 +1054,7 @@ test("a failed activation recovery marker is never reusable", async (t) => { assert.equal(loadReusableBrokerSession(repo, env), null); assert.doesNotThrow(() => process.kill(failedSession.pid, 0)); - await assert.rejects( - ensureBrokerSession(repo, { env, killProcess: unverifiedCleanup }), - (error) => error?.code === "BROKER_CLEANUP_UNVERIFIED" - ); + assert.equal(await ensureBrokerSession(repo, { env, killProcess: unverifiedCleanup }), null); assert.equal(loadBrokerSession(repo)?.pid, failedSession.pid); assert.equal(loadBrokerSession(repo)?.activationFailed, true); }); @@ -1255,6 +1306,43 @@ test("pre-activation broker exits when its launcher pipe closes", async (t) => { assert.equal(fs.existsSync(pidFile), false); }); +test("an unregistered broker refuses to activate a detached app-server child", async (t) => { + if (process.platform === "win32") { + t.skip("Unix broker sockets are required for this contract."); + return; + } + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + const socketPath = path.join("/private/tmp", `cxc-unregistered-child-${process.pid}-${Date.now()}.sock`); + const endpoint = `unix:${socketPath}`; + installFakeCodex(binDir, "with-helper-child"); + const env = buildEnv(binDir); + const broker = spawn(process.execPath, [BROKER_SCRIPT, "serve", "--endpoint", endpoint, "--cwd", repo], { + cwd: repo, + env, + detached: true, + stdio: ["ignore", "ignore", "ignore"] + }); + const brokerOwnership = captureProcessOwnership(broker.pid, { cwd: repo, env }); + t.after(async () => { + await sendBrokerShutdown(endpoint).catch(() => {}); + await terminateProcessTree(broker.pid, { + expectedRootIdentity: brokerOwnership.rootIdentity, + ownershipSnapshot: brokerOwnership + }).catch(() => {}); + }); + assert.equal(await waitForBrokerEndpoint(endpoint, 2000), true); + + const client = await CodexAppServerClient.connect(repo, { brokerEndpoint: endpoint, env }); + await assert.rejects( + client.request("thread/start", { cwd: repo, ephemeral: true }), + (error) => error?.rpcCode === -32005 + ); + await client.close(); + assert.equal(fs.existsSync(fakeStatePath), false); +}); + test("automatic broker creation stays disabled where registered ownership is unsupported", async () => { const repo = makeTempDir(); let endpointFactoryCalled = false; @@ -2349,6 +2437,43 @@ test("task using the shared broker still completes when Codex spawns subagents", assert.equal(result.stdout, "Handled the requested task.\nTask prompt accepted.\n"); }); +test("inferred shared turn completion releases broker stream state and its idle child", async (t) => { + if (process.platform === "win32") { + return; + } + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "with-subagent-no-main-turn-completed"); + initGitRepo(repo); + fs.writeFileSync(path.join(repo, "README.md"), "hello\n"); + run("git", ["add", "README.md"], { cwd: repo }); + run("git", ["commit", "-m", "init"], { cwd: repo }); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "100" + }, "inferred-completion"); + + const result = run("node", [SCRIPT, "task", "challenge the current design"], { + cwd: repo, + env + }); + assert.equal(result.status, 0, result.stderr); + const brokerSession = loadBrokerSession(repo); + assert.equal(brokerSession?.registry?.registered, true); + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + }); + + await waitFor(() => { + const children = loadBrokerChildren(brokerSession.registry); + return children.valid === true && children.children.length === 0 && children.releasedChildren.length === 1; + }, { timeoutMs: 2500, intervalMs: 25 }); +}); + test("task --background enqueues a detached worker and exposes per-job status", async () => { const repo = makeTempDir(); const binDir = makeTempDir(); @@ -4340,7 +4465,7 @@ test("shared broker releases its idle app-server child and restarts it on demand assert.equal(fakeState.helperPids.length, 2); }); -test("identity capture failure cleans the owned app-server tree and reports unverified cleanup", async (t) => { +test("identity capture failure prevents app-server activation and reports unverified cleanup", async () => { const repo = makeTempDir(); const binDir = makeTempDir(); const fakeStatePath = path.join(binDir, "fake-codex-state.json"); @@ -4358,23 +4483,37 @@ test("identity capture failure cleans the owned app-server tree and reports unve (error) => error.cleanupOutcome?.verified === false && error.cleanupOutcome?.degraded === true ); - const helperPid = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).helperPids[0]; - function isRunning(pid) { - const result = run("/bin/ps", ["-o", "stat=", "-p", String(pid)]); - return result.status === 0 && !result.stdout.trim().startsWith("Z"); + assert.equal(fs.existsSync(fakeStatePath), false); +}); + +test("a broker-owned app-server cannot activate before durable child publication", async () => { + if (process.platform === "win32") { + return; } - await waitFor(() => { - return !isRunning(helperPid); - }); - const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); - assert.equal(state.appServerStarts, 1); - t.after(() => { - try { - process.kill(helperPid, "SIGKILL"); - } catch { - // Ignore the helper after cleanup. - } - }); + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "with-helper-child"); + const env = buildEnv(binDir); + let wrapperPid = null; + + await assert.rejects( + CodexAppServerClient.connect(repo, { + disableBroker: true, + gatedBrokerChild: true, + env, + beforeAppServerActivation(ownershipSnapshot) { + wrapperPid = ownershipSnapshot.rootPid; + assert.equal(fs.existsSync(fakeStatePath), false); + throw new Error("injected durable child publication failure"); + } + }), + /durable child publication failure/ + ); + + assert.ok(Number.isFinite(wrapperPid)); + assert.equal(fs.existsSync(fakeStatePath), false); + await waitFor(() => getProcessIdentity(wrapperPid) === null); }); test("shared broker keeps active work alive after its client disconnects", async (t) => { From 4e8884be6e0d27ce3859559f48daaf7e5abcacaf Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Thu, 30 Jul 2026 10:01:26 -0700 Subject: [PATCH 24/29] [CC-5589] close helper ownership and registry retirement gaps Attributed-To: Codex --- plugins/codex/scripts/app-server-broker.mjs | 27 +- plugins/codex/scripts/app-server-child.mjs | 5 +- .../scripts/lib/app-server-protocol.d.ts | 1 + plugins/codex/scripts/lib/app-server.mjs | 53 ++++ .../codex/scripts/lib/broker-ownership.mjs | 254 +++++++++++++++++- plugins/codex/scripts/lib/process.mjs | 86 ++++-- .../scripts/registered-broker-reaper.mjs | 34 ++- tests/broker-ownership.test.mjs | 78 ++++++ tests/fake-codex-fixture.mjs | 15 +- tests/process.test.mjs | 100 ++++++- tests/registered-broker-reaper.test.mjs | 16 ++ tests/runtime.test.mjs | 4 +- 12 files changed, 632 insertions(+), 41 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 7d0a85e07..4b15edbf1 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -8,7 +8,12 @@ import { fileURLToPath } from "node:url"; import { parseArgs } from "./lib/args.mjs"; import { BROKER_BUSY_RPC_CODE, BROKER_OWNERSHIP_RPC_CODE, BROKER_STREAM_COMPLETED_METHOD, CodexAppServerClient } from "./lib/app-server.mjs"; -import { loadBrokerRegistration, publishBrokerChild, releaseBrokerChild } from "./lib/broker-ownership.mjs"; +import { + loadBrokerRegistration, + publishBrokerChild, + publishBrokerChildObservation, + releaseBrokerChild +} from "./lib/broker-ownership.mjs"; import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs"; import { getLiveProcessPids, getProcessIdentity } from "./lib/process.mjs"; @@ -336,6 +341,26 @@ async function main() { error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; throw error; } + }, + async afterAppServerOwnershipRefresh(ownershipSnapshot) { + if (registration?.registered !== true || childRegistration?.registered !== true) { + const error = new Error("Shared Codex app-server ownership registration was lost before helper observation."); + error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + throw error; + } + const observation = publishBrokerChildObservation(registration, { + child: childRegistration.child, + ownershipSnapshot + }); + if (observation.observed !== true) { + const error = new Error(`Unable to publish shared Codex app-server helper ownership (${observation.reason ?? "unknown"}).`); + error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + throw error; + } + childRegistration = { + ...childRegistration, + child: observation.child + }; } }) .then(async (client) => { diff --git a/plugins/codex/scripts/app-server-child.mjs b/plugins/codex/scripts/app-server-child.mjs index 486544615..650c9c416 100644 --- a/plugins/codex/scripts/app-server-child.mjs +++ b/plugins/codex/scripts/app-server-child.mjs @@ -81,4 +81,7 @@ process.stdin.on("end", () => { process.stdin.on("error", () => { child?.stdin.destroy(); }); -process.stdin.resume(); +// Keep protocol bytes buffered until activation has created the real app +// server and its stdin pipe. Flowing stdin here can discard an initialize +// request that races the activation-control pipe. +process.stdin.pause(); diff --git a/plugins/codex/scripts/lib/app-server-protocol.d.ts b/plugins/codex/scripts/lib/app-server-protocol.d.ts index bbf57c1b5..0e61c92e7 100644 --- a/plugins/codex/scripts/lib/app-server-protocol.d.ts +++ b/plugins/codex/scripts/lib/app-server-protocol.d.ts @@ -56,6 +56,7 @@ export interface CodexAppServerClientOptions { reuseExistingBroker?: boolean; gatedBrokerChild?: boolean; beforeAppServerActivation?: (ownershipSnapshot: unknown) => void | Promise; + afterAppServerOwnershipRefresh?: (ownershipSnapshot: unknown) => void | Promise; } export interface AppServerMethodMap { diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index b5ce78198..cb88c0ef4 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -270,6 +270,59 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { this.notify("initialized", {}); } + mergeOwnershipSnapshot(observation) { + if ( + !observation?.rootIdentity || + observation.rootIdentity !== this.procIdentity || + observation.rootPid !== this.ownershipSnapshot?.rootPid + ) { + throw new Error("Codex app-server ownership identity changed while refreshing its helper tree."); + } + const members = new Map(); + for (const snapshot of [this.ownershipSnapshot, observation]) { + for (const member of snapshot?.members ?? []) { + members.set(member.identity, member); + } + } + this.ownershipSnapshot = { + ...observation, + members: [...members.values()] + }; + return this.ownershipSnapshot; + } + + async refreshOwnership() { + if (process.platform === "win32" || !this.procIdentity || !this.proc?.pid) { + return this.ownershipSnapshot ?? null; + } + const captureOwnership = this.options.captureProcessOwnershipImpl ?? captureProcessOwnership; + const observation = captureOwnership(this.proc.pid, { + cwd: this.cwd, + env: this.options.env ?? process.env + }); + const ownershipSnapshot = this.mergeOwnershipSnapshot(observation); + await this.options.afterAppServerOwnershipRefresh?.(ownershipSnapshot); + return ownershipSnapshot; + } + + async request(method, params) { + let result; + let requestError; + try { + result = await super.request(method, params); + } catch (error) { + requestError = error; + } + // Helpers may be created only after the gated wrapper is activated. Take + // and durably publish a fresh identity snapshot at every request boundary + // before the broker exposes the response or error to its caller. + await this.refreshOwnership(); + if (requestError) { + throw requestError; + } + return result; + } + startUnexpectedExitCleanup(pid) { if ( this.unexpectedExitCleanupPromise || diff --git a/plugins/codex/scripts/lib/broker-ownership.mjs b/plugins/codex/scripts/lib/broker-ownership.mjs index 841199061..c1f78f26e 100644 --- a/plugins/codex/scripts/lib/broker-ownership.mjs +++ b/plugins/codex/scripts/lib/broker-ownership.mjs @@ -13,6 +13,7 @@ const PLUGIN_DATA_ENV = "CLAUDE_PLUGIN_DATA"; const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; const REGISTRY_DIR_NAME = "broker-ownership-v1"; const REGISTRY_LOCK_DIR_NAME = "registry.lock"; +const TERMINAL_FILE_NAME = "terminal.json"; function sha256(value) { return createHash("sha256").update(String(value)).digest("hex"); @@ -346,6 +347,41 @@ export function loadBrokerRegistration({ endpoint, brokerIdentity, env = process } } +function validBrokerTerminal(record, filePath, registration) { + return Boolean( + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "terminal" && + record.brokerKey === registration.brokerKey && + record.pidIdentity === registration.broker?.pidIdentity && + record.decision === "cleanup-verified" && + typeof record.attemptId === "string" && + /^[a-zA-Z0-9._-]{1,128}$/.test(record.attemptId) && + record.receipt === `${record.attemptId}.json` && + typeof record.retiredAt === "string" && + record.retiredAt.length > 0 && + filePath === path.join(registration.registryDir, TERMINAL_FILE_NAME) + ); +} + +export function loadBrokerTerminal(registration) { + if (!validRegistrationReference(registration)) { + return { terminal: false, reason: "broker-registration-unavailable" }; + } + const terminalPath = path.join(registration.registryDir, TERMINAL_FILE_NAME); + if (!fs.existsSync(terminalPath)) { + return { terminal: false, reason: "terminal-absent", path: terminalPath }; + } + try { + const record = readJson(terminalPath); + if (!validBrokerTerminal(record, terminalPath, registration)) { + return { terminal: false, reason: "terminal-invalid", path: terminalPath }; + } + return { terminal: true, reason: "cleanup-verified", path: terminalPath, record }; + } catch { + return { terminal: false, reason: "terminal-invalid", path: terminalPath }; + } +} + export function publishBrokerRegistration({ cwd, endpoint, pid, ownershipSnapshot, env = process.env, now = () => new Date().toISOString() }) { const registryRoot = resolveBrokerOwnershipRoot(env); if (!registryRoot) { @@ -624,6 +660,7 @@ function validSnapshotMember(record) { Number.isSafeInteger(record.parentPid) && record.parentPid >= 0 && isSafePid(record.processGroupId) && + (record.sessionId == null || isSafePid(record.sessionId)) && typeof record.state === "string" && record.state.length > 0 && typeof record.startedAt === "string" && @@ -648,13 +685,67 @@ function validChildRecord(record, filePath, registration) { snapshot?.rootPid === record.pid && snapshot.rootIdentity === record.pidIdentity && snapshot.processGroupId === record.processGroupId && + (snapshot.sessionId == null || + (isSafePid(snapshot.sessionId) && + snapshot.sessionId === snapshot.rootPid)) && Array.isArray(snapshot.members) && snapshot.members.length > 0 && snapshot.members.every(validSnapshotMember) && - snapshot.members.some((member) => member.pid === record.pid && member.identity === record.pidIdentity) + snapshot.members.some( + (member) => + member.pid === record.pid && + member.identity === record.pidIdentity && + (snapshot.sessionId == null || member.sessionId === snapshot.sessionId) + ) ); } +function validChildObservation(record, filePath, registration, child) { + const snapshot = record?.ownershipSnapshot; + return Boolean( + child && + record?.version === BROKER_OWNERSHIP_VERSION && + record.kind === "child-observation" && + record.brokerKey === registration.brokerKey && + record.childKey === child.childKey && + typeof record.observationKey === "string" && + record.observationKey === sha256(JSON.stringify(snapshot)) && + path.basename(filePath) === `${record.observationKey}.json` && + snapshot?.rootPid === child.pid && + snapshot.rootIdentity === child.pidIdentity && + snapshot.processGroupId === child.processGroupId && + (snapshot.sessionId == null || + (isSafePid(snapshot.sessionId) && snapshot.sessionId === snapshot.rootPid)) && + Array.isArray(snapshot.members) && + snapshot.members.length > 0 && + snapshot.members.every(validSnapshotMember) && + snapshot.members.some( + (member) => + member.pid === child.pid && + member.identity === child.pidIdentity && + (snapshot.sessionId == null || member.sessionId === snapshot.sessionId) + ) && + typeof record.observedAt === "string" && + record.observedAt.length > 0 + ); +} + +function mergeChildOwnership(child, observations) { + const members = new Map(); + for (const snapshot of [child.ownershipSnapshot, ...observations.map((observation) => observation.ownershipSnapshot)]) { + for (const member of snapshot.members) { + members.set(member.identity, member); + } + } + return { + ...child, + ownershipSnapshot: { + ...child.ownershipSnapshot, + members: [...members.values()] + } + }; +} + function validChildReleaseRecord(record, filePath, registration, child) { return Boolean( child && @@ -702,7 +793,28 @@ export function loadBrokerChildren(registration) { malformed.push(filePath); } } - const childrenByKey = new Map(children.map((child) => [child.childKey, child])); + const observedChildren = []; + for (const child of children) { + const observationDir = path.join(registration.registryDir, "child-observations", child.childKey); + const observations = []; + if (fs.existsSync(observationDir)) { + try { + requirePrivateDirectory(observationDir); + for (const filePath of listJsonFiles(observationDir)) { + const observation = readJson(filePath); + if (!validChildObservation(observation, filePath, registration, child)) { + malformed.push(filePath); + } else { + observations.push(observation); + } + } + } catch { + malformed.push(observationDir); + } + } + observedChildren.push(mergeChildOwnership(child, observations)); + } + const childrenByKey = new Map(observedChildren.map((child) => [child.childKey, child])); for (const filePath of childReleaseFiles) { try { const record = readJson(filePath); @@ -720,8 +832,8 @@ export function loadBrokerChildren(registration) { : { valid: true, reason: "valid-child-registry", - children: children.filter((child) => !releases.has(child.childKey)), - releasedChildren: children.filter((child) => releases.has(child.childKey)), + children: observedChildren.filter((child) => !releases.has(child.childKey)), + releasedChildren: observedChildren.filter((child) => releases.has(child.childKey)), malformed: [] }; } @@ -735,8 +847,16 @@ export function releaseBrokerChild( return { released: false, reason: "broker-registration-unavailable" }; } const childPath = path.join(registration.registryDir, "children", `${child?.childKey}.json`); + let persistedChild; + try { + persistedChild = readJson(childPath); + } catch { + return { released: false, reason: "child-cleanup-unverified" }; + } if ( - !validChildRecord(child, childPath, registration) || + !validChildRecord(persistedChild, childPath, registration) || + child?.childKey !== persistedChild.childKey || + child?.pidIdentity !== persistedChild.pidIdentity || cleanupOutcome?.verified !== true || (cleanupOutcome.survivors?.length ?? 0) > 0 || (cleanupOutcome.survivorIdentities?.length ?? 0) > 0 @@ -802,6 +922,58 @@ export function publishBrokerReaperReceipt( return { published: true, path: receiptPath, receipt: payload }; } +export function publishBrokerTerminal( + registration, + { attemptId, receiptPath, retiredAt = new Date().toISOString(), registryLock } +) { + if (!validRegistrationReference(registration)) { + return { terminal: false, reason: "broker-registration-unavailable" }; + } + const expectedReceiptPath = path.join(registration.registryDir, "receipts", `${attemptId}.json`); + if ( + typeof attemptId !== "string" || + !/^[a-zA-Z0-9._-]{1,128}$/.test(attemptId) || + receiptPath !== expectedReceiptPath || + typeof retiredAt !== "string" || + retiredAt.length === 0 + ) { + return { terminal: false, reason: "terminal-invalid" }; + } + let receipt; + try { + receipt = readJson(receiptPath); + } catch { + return { terminal: false, reason: "terminal-receipt-unavailable" }; + } + if ( + receipt?.version !== BROKER_OWNERSHIP_VERSION || + receipt.kind !== "reaper-receipt" || + receipt.brokerKey !== registration.brokerKey || + receipt.attemptId !== attemptId || + receipt.decision !== "cleanup-verified" || + !Array.isArray(receipt.residualIdentities) || + receipt.residualIdentities.length !== 0 + ) { + return { terminal: false, reason: "terminal-receipt-invalid" }; + } + const terminalPath = path.join(registration.registryDir, TERMINAL_FILE_NAME); + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "terminal", + brokerKey: registration.brokerKey, + pidIdentity: registration.broker.pidIdentity, + decision: "cleanup-verified", + attemptId, + receipt: path.basename(receiptPath), + retiredAt + }; + const locked = withBrokerRegistryLock(registration, { registryLock }, () => { + createImmutableJson(terminalPath, payload); + return { terminal: true, reason: "cleanup-verified", path: terminalPath, record: payload }; + }); + return locked.ok ? locked.value : { terminal: false, reason: locked.reason }; +} + export function assessBrokerOwners(registration, { getLiveProcessPidsImpl = getLiveProcessPids } = {}) { if (!validRegistrationReference(registration)) { return { safeToShutdown: false, reason: "broker-registration-unavailable", liveOwners: [], deadOwners: [], releasedOwners: [], malformed: [] }; @@ -909,7 +1081,13 @@ export function publishBrokerChild(registration, options = {}) { } const pid = ownershipSnapshot?.rootPid; const identity = ownershipSnapshot?.rootIdentity; - if (!isPidIdentity(pid, identity) || !isSafePid(ownershipSnapshot?.processGroupId) || !Array.isArray(ownershipSnapshot?.members)) { + if ( + !isPidIdentity(pid, identity) || + !isSafePid(ownershipSnapshot?.processGroupId) || + (ownershipSnapshot?.sessionId != null && + (!isSafePid(ownershipSnapshot.sessionId) || ownershipSnapshot.sessionId !== pid)) || + !Array.isArray(ownershipSnapshot?.members) + ) { return { registered: false, reason: "child-identity-unavailable" }; } const locked = withBrokerRegistryLock(registration, options, () => { @@ -949,3 +1127,67 @@ export function publishBrokerChild(registration, options = {}) { }); return locked.ok ? locked.value : { registered: false, reason: locked.reason }; } + +export function publishBrokerChildObservation(registration, options = {}) { + const { + child, + ownershipSnapshot, + now = () => new Date().toISOString() + } = options; + if (!validRegistrationReference(registration)) { + return { observed: false, reason: "broker-registration-unavailable" }; + } + const childPath = path.join(registration.registryDir, "children", `${child?.childKey}.json`); + const locked = withBrokerRegistryLock(registration, options, () => { + const persistedChild = readJson(childPath); + if ( + !validChildRecord(persistedChild, childPath, registration) || + child?.childKey !== persistedChild.childKey || + child?.pidIdentity !== persistedChild.pidIdentity + ) { + return { observed: false, reason: "child-registration-invalid" }; + } + const observationKey = sha256(JSON.stringify(ownershipSnapshot)); + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "child-observation", + brokerKey: registration.brokerKey, + childKey: persistedChild.childKey, + observationKey, + ownershipSnapshot, + observedAt: now() + }; + const observationPath = path.join( + registration.registryDir, + "child-observations", + persistedChild.childKey, + `${observationKey}.json` + ); + if (fs.existsSync(observationPath)) { + const existing = readJson(observationPath); + if (!validChildObservation(existing, observationPath, registration, persistedChild)) { + return { observed: false, reason: "child-observation-invalid" }; + } + return { + observed: true, + reason: "child-already-observed", + path: observationPath, + observation: existing, + child: mergeChildOwnership(persistedChild, [existing]) + }; + } + if (!validChildObservation(payload, observationPath, registration, persistedChild)) { + return { observed: false, reason: "child-observation-invalid" }; + } + createImmutableJson(observationPath, payload); + const observedChild = mergeChildOwnership(persistedChild, [payload]); + return { + observed: true, + reason: "child-observed", + path: observationPath, + observation: payload, + child: observedChild + }; + }); + return locked.ok ? locked.value : { observed: false, reason: locked.reason }; +} diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 8a0b95b6e..a6a905fa7 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -54,7 +54,7 @@ function looksLikeMissingProcessMessage(text) { return /not found|no running instance|cannot find|does not exist|no such process/i.test(text); } -const UNIX_PROCESS_TABLE_ARGS = ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]; +const UNIX_PROCESS_TABLE_ARGS = ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]; const UNIX_PS_COMMAND = "/bin/ps"; const UNIX_PS_PATH_COMMAND = "ps"; @@ -86,22 +86,39 @@ function readUnixProcessTable(runCommandImpl, options = {}) { if (!trimmedLine) { continue; } - const match = trimmedLine.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/); - if (!match) { + const match = trimmedLine.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/); + const legacyMatch = match ? null : trimmedLine.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/); + const parsed = match ?? legacyMatch; + if (!parsed) { throw createProcessTableError(`Unable to parse Unix process table line: ${trimmedLine}`); } - const pid = Number(match[1]); - const parentPid = Number(match[2]); - const processGroupId = Number(match[3]); - const state = match[4]; - const startedAt = match[5].trim(); - if (!Number.isSafeInteger(pid) || !Number.isSafeInteger(parentPid) || !Number.isSafeInteger(processGroupId) || !startedAt) { + const pid = Number(parsed[1]); + const parentPid = Number(parsed[2]); + const processGroupId = Number(parsed[3]); + const parsedSessionId = match ? Number(parsed[4]) : null; + // Darwin's `sess` is the security audit session, not the POSIX process + // session used for containment. Never treat it as a child-tree boundary. + const sessionId = + (options.platform ?? process.platform) !== "darwin" && + Number.isSafeInteger(parsedSessionId) && + parsedSessionId > 1 + ? parsedSessionId + : null; + const state = parsed[match ? 5 : 4]; + const startedAt = parsed[match ? 6 : 5].trim(); + if ( + !Number.isSafeInteger(pid) || + !Number.isSafeInteger(parentPid) || + !Number.isSafeInteger(processGroupId) || + !startedAt + ) { throw createProcessTableError(`Unable to parse Unix process table line: ${trimmedLine}`); } processes.set(pid, { pid, parentPid, processGroupId, + sessionId, state, startedAt, identity: `${pid}@${startedAt}` @@ -157,6 +174,7 @@ export function captureProcessOwnership(pid, options = {}) { rootPid: root.pid, rootIdentity: root.identity, processGroupId: root.processGroupId, + sessionId: root.sessionId, members: collectProcessTree(pid, processes).map((record) => ({ ...record })) }; } @@ -716,6 +734,13 @@ export async function terminateProcessGroup(pgid, options = {}) { const tracked = new Map(); const ownershipSnapshot = options.ownershipSnapshot ?? null; const ownershipEstablished = Boolean(ownershipSnapshot); + const ownershipSessionId = + Number.isSafeInteger(ownershipSnapshot?.sessionId) && + ownershipSnapshot.sessionId > 1 && + ownershipSnapshot.rootPid === ownershipSnapshot.sessionId + ? ownershipSnapshot.sessionId + : null; + const cleanupMethod = ownershipSessionId ? "process-session" : "process-group"; // If a live process holds this group id but is not the root we recorded, the // id has been reused and nothing in the group is ours. Refuse before // admitting any member: members of an unrelated group are absent from the @@ -731,7 +756,20 @@ export async function terminateProcessGroup(pgid, options = {}) { verified: false, degraded: true, identityMismatch: true, - method: "process-group", + method: cleanupMethod, + survivors: recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.pid), + survivorIdentities: recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.identity) + }); + } + const sessionLeader = ownershipSessionId ? processes.get(ownershipSessionId) : null; + if (ownershipSessionId && sessionLeader && sessionLeader.identity !== ownershipSnapshot.rootIdentity) { + return normalizeProcessCleanupOutcome({ + attempted: false, + delivered: false, + verified: false, + degraded: true, + identityMismatch: true, + method: cleanupMethod, survivors: recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.pid), survivorIdentities: recordsFromOwnershipSnapshot(ownershipSnapshot).map((record) => record.identity) }); @@ -739,21 +777,25 @@ export async function terminateProcessGroup(pgid, options = {}) { for (const record of recordsFromOwnershipSnapshot(ownershipSnapshot)) { tracked.set(record.identity, record); } - let groupMembersFound = false; - let groupSelectionFound = false; + let ownershipScopeMembersFound = false; + let ownershipSelectionFound = false; for (const record of processes.values()) { - if (record.processGroupId !== pgid || !isRunningProcess(record)) { + const inOwnedScope = + record.processGroupId === pgid || + (ownershipSessionId !== null && record.sessionId === ownershipSessionId); + if (!inOwnedScope || !isRunningProcess(record)) { continue; } - groupMembersFound = true; + ownershipScopeMembersFound = true; const snapshotRecord = recordsFromOwnershipSnapshot(ownershipSnapshot).find((candidate) => candidate.pid === record.pid); if (ownershipSnapshot && snapshotRecord && snapshotRecord.identity !== record.identity) { continue; } - groupSelectionFound = true; - if (record.processGroupId === pgid && isRunningProcess(record)) { - tracked.set(record.identity, record); - } + ownershipSelectionFound = true; + tracked.set(record.identity, { + ...record, + depth: Number.isSafeInteger(record.depth) ? record.depth : record.pid === pgid ? 0 : 1 + }); } if (tracked.size === 0) { return normalizeProcessCleanupOutcome({ @@ -786,14 +828,14 @@ export async function terminateProcessGroup(pgid, options = {}) { } const root = processes.get(pgid); const rootIdentityMatches = !root || root.identity === ownershipSnapshot?.rootIdentity; - const ownedGroupAccountedFor = groupSelectionFound || !groupMembersFound; + const ownedScopeAccountedFor = ownershipSelectionFound || !ownershipScopeMembersFound; return normalizeProcessCleanupOutcome({ attempted: true, delivered, - verified: live.length === 0 && (!ownershipEstablished || (rootIdentityMatches && ownedGroupAccountedFor)), - degraded: ownershipEstablished && (!rootIdentityMatches || !ownedGroupAccountedFor), + verified: live.length === 0 && (!ownershipEstablished || (rootIdentityMatches && ownedScopeAccountedFor)), + degraded: ownershipEstablished && (!rootIdentityMatches || !ownedScopeAccountedFor), escalated, - method: "process-group", + method: cleanupMethod, targets: [...tracked.values()].map((record) => record.pid), targetIdentities: [...tracked.values()].map((record) => record.identity), survivors: live.map((record) => record.pid), diff --git a/plugins/codex/scripts/registered-broker-reaper.mjs b/plugins/codex/scripts/registered-broker-reaper.mjs index 67c9a3d39..53777c1cf 100644 --- a/plugins/codex/scripts/registered-broker-reaper.mjs +++ b/plugins/codex/scripts/registered-broker-reaper.mjs @@ -11,7 +11,9 @@ import { assessBrokerOwners, loadBrokerChildren, loadBrokerRegistration, + loadBrokerTerminal, publishBrokerReaperReceipt, + publishBrokerTerminal, releaseBrokerChild, releaseBrokerRegistryLock, resolveBrokerOwnershipRoot @@ -67,7 +69,15 @@ function listRegistrationCandidates(env) { ) { candidates.push({ valid: false, brokerKey: entry.name, registryDir, reason: "broker-record-invalid" }); } else { - candidates.push({ valid: true, registration }); + const terminal = loadBrokerTerminal(registration); + if (terminal.terminal === true) { + continue; + } + if (terminal.reason !== "terminal-absent") { + candidates.push({ valid: false, brokerKey: entry.name, registryDir, reason: terminal.reason }); + } else { + candidates.push({ valid: true, registration }); + } } } catch { candidates.push({ valid: false, brokerKey: entry.name, registryDir, reason: "broker-record-invalid" }); @@ -245,14 +255,30 @@ async function processRegistration(candidate, options) { residualIdentities, createdAt: (options.now ?? (() => new Date().toISOString()))() }); + const terminal = + decision === "cleanup-verified" && receipt?.published === true + ? (options.publishBrokerTerminalImpl ?? publishBrokerTerminal)(lockedRegistration, { + attemptId, + receiptPath: receipt.path, + retiredAt: (options.now ?? (() => new Date().toISOString()))(), + registryLock + }) + : null; + const terminalRecorded = decision !== "cleanup-verified" || terminal?.terminal === true; result = { - status: decision === "cleanup-verified" ? "reaped" : "report-only", + status: decision === "cleanup-verified" && receipt?.published === true && terminalRecorded ? "reaped" : "report-only", brokerKey: lockedRegistration.brokerKey, pid: lockedRegistration.broker.pid, - reason: receipt?.published === true ? decision : `${decision}-receipt-unavailable`, + reason: + receipt?.published !== true + ? `${decision}-receipt-unavailable` + : terminalRecorded + ? decision + : `${decision}-${terminal?.reason ?? "terminal-unavailable"}`, outcomes, residualIdentities, - receiptPath: receipt?.published === true ? receipt.path : null + receiptPath: receipt?.published === true ? receipt.path : null, + terminalPath: terminal?.terminal === true ? terminal.path : null }; } } diff --git a/tests/broker-ownership.test.mjs b/tests/broker-ownership.test.mjs index 57ea67ca2..fbbcd77ab 100644 --- a/tests/broker-ownership.test.mjs +++ b/tests/broker-ownership.test.mjs @@ -9,6 +9,7 @@ import { assessBrokerOwners, loadBrokerChildren, publishBrokerChild, + publishBrokerChildObservation, publishBrokerRegistration, publishRegisteredBroker, registerBrokerOwner, @@ -454,3 +455,80 @@ test("broker child ownership is immutable and identity keyed", (t) => { assert.equal(children.children.length, 0); assert.equal(children.releasedChildren.length, 1); }); + +test("post-activation child observations extend durable cleanup ownership", (t) => { + const { registration } = makeFixture(t); + const rootIdentity = "6200@Mon Jul 27 00:08:00 2026"; + const child = publishBrokerChild(registration, { + ownershipSnapshot: { + rootPid: 6200, + rootIdentity, + processGroupId: 6200, + members: [ + { + pid: 6200, + parentPid: 4100, + processGroupId: 6200, + state: "S", + startedAt: "Mon Jul 27 00:08:00 2026", + identity: rootIdentity, + depth: 0 + } + ] + } + }); + const helperIdentity = "6202@Mon Jul 27 00:08:02 2026"; + const observed = publishBrokerChildObservation(registration, { + child: child.child, + ownershipSnapshot: { + rootPid: 6200, + rootIdentity, + processGroupId: 6200, + members: [ + { + pid: 6200, + parentPid: 4100, + processGroupId: 6200, + state: "S", + startedAt: "Mon Jul 27 00:08:00 2026", + identity: rootIdentity, + depth: 0 + }, + { + pid: 6202, + parentPid: 6201, + processGroupId: 6202, + state: "S", + startedAt: "Mon Jul 27 00:08:02 2026", + identity: helperIdentity, + depth: 2 + } + ] + }, + now: () => "2026-07-27T00:08:03.000Z" + }); + + assert.equal(observed.observed, true); + assert.equal(fs.statSync(observed.path).mode & 0o777, 0o600); + const repeated = publishBrokerChildObservation(registration, { + child: child.child, + ownershipSnapshot: observed.observation.ownershipSnapshot, + now: () => "2026-07-27T00:08:04.000Z" + }); + assert.equal(repeated.observed, true); + assert.equal(repeated.reason, "child-already-observed"); + assert.equal(repeated.path, observed.path); + const loaded = loadBrokerChildren(registration); + assert.equal(loaded.valid, true); + assert.deepEqual(loaded.children[0].ownershipSnapshot.members.map((member) => member.identity), [ + rootIdentity, + helperIdentity + ]); + + const released = releaseBrokerChild(registration, { + child: loaded.children[0], + cleanupOutcome: { verified: true, survivors: [], survivorIdentities: [] } + }); + assert.equal(released.released, true); + assert.equal(loadBrokerChildren(registration).releasedChildren[0].ownershipSnapshot.members.length, 2); +}); diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index 6f69e352d..2cc0ad795 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -288,7 +288,7 @@ if (BEHAVIOR === "with-helper-child" || BEHAVIOR === "slow-task-with-helper-chil bootState.helperPids = [...(bootState.helperPids || []), helper.pid]; } } -if (BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "crash-with-post-snapshot-helper") { +if (BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "crash-with-post-snapshot-helper" || BEHAVIOR === "crash-with-post-activation-regrouped-helper") { bootState.appServerPids = [...(bootState.appServerPids || []), process.pid]; } saveState(bootState); @@ -306,6 +306,14 @@ rl.on("line", (line) => { switch (message.method) { case "initialize": state.capabilities = message.params.capabilities || null; + if (BEHAVIOR === "crash-with-post-activation-regrouped-helper") { + const helper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { + detached: process.platform !== "win32", + stdio: "ignore" + }); + helper.unref(); + state.helperPids = [...(state.helperPids || []), helper.pid]; + } saveState(state); send({ id: message.id, result: { userAgent: "fake-codex-app-server" } }); if (BEHAVIOR === "crash-with-regrouped-helper") { @@ -324,6 +332,11 @@ rl.on("line", (line) => { setTimeout(() => process.exit(1), 100); }, 250); } + if (BEHAVIOR === "crash-with-post-activation-regrouped-helper") { + // Leave enough time for the broker to persist the post-response + // ownership observation before simulating the later crash. + setTimeout(() => process.exit(1), 500); + } break; case "initialized": diff --git a/tests/process.test.mjs b/tests/process.test.mjs index 0225f12cd..916c073d9 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -1,7 +1,36 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { captureStableSessionOwner, getLiveProcessPids, hasLiveProcessIdentity, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; +import { + captureProcessOwnership, + captureStableSessionOwner, + getLiveProcessPids, + hasLiveProcessIdentity, + terminateProcessGroup, + terminateProcessTree +} from "../plugins/codex/scripts/lib/process.mjs"; + +test("captureProcessOwnership never treats a Darwin audit session as process containment", () => { + const snapshot = captureProcessOwnership(7300, { + platform: "darwin", + runCommandImpl(command, args) { + assert.equal(command, "/bin/ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]); + return { + command, + args, + status: 0, + signal: null, + stdout: "7300 1 7300 44001 S Mon Jul 27 00:09:00 2026\n", + stderr: "", + error: null + }; + } + }); + + assert.equal(snapshot.sessionId, null); + assert.equal(snapshot.members[0].sessionId, null); +}); test("captureStableSessionOwner records the hook process-group leader", () => { const owner = captureStableSessionOwner(7101, { @@ -148,7 +177,7 @@ test("terminateProcessTree terminates Unix descendant groups deepest-first", asy expectedRootIdentity: "1234@Mon Jul 27 00:00:00 2026", runCommandImpl(command, args) { assert.equal(command, "/bin/ps"); - assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]); const stdout = [...alive] .map((pid) => `${pid} ${parents.get(pid)} ${pid} S Mon Jul 27 00:00:0${pid - 1234} 2026`) .join("\n"); @@ -289,7 +318,7 @@ test("terminateProcessTree parses a captured Linux procps process table", async expectedRootIdentity: "42001@Mon Jul 27 12:34:56 2026", runCommandImpl(command, args) { assert.equal(command, "/bin/ps"); - assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]); const stdout = [...alive] .map((pid) => sample.split("\n").find((line) => line.startsWith(String(pid)))) .filter(Boolean) @@ -326,7 +355,7 @@ test("terminateProcessTree defers persisted cleanup when Unix process enumeratio }, runCommandImpl(command, args) { assert.equal(command, "/bin/ps"); - assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,stat=,lstart="]); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]); return { command, args, @@ -1039,6 +1068,69 @@ test("terminateProcessGroup reclaims a post-snapshot member of the owned group", assert.deepEqual(outcome.survivors, []); }); +test("terminateProcessGroup reclaims a post-activation group in the owned Unix session", async () => { + const signals = []; + const alive = new Set([100, 300]); + const ownershipSnapshot = { + rootPid: 100, + rootIdentity: "100@Mon Jul 27 00:00:00 2026", + processGroupId: 100, + sessionId: 100, + members: [ + { + pid: 100, + parentPid: 1, + processGroupId: 100, + sessionId: 100, + state: "S", + startedAt: "Mon Jul 27 00:00:00 2026", + identity: "100@Mon Jul 27 00:00:00 2026", + depth: 0 + } + ] + }; + const outcome = await terminateProcessGroup(100, { + platform: "linux", + ownershipSnapshot, + pollIntervalMs: 0, + runCommandImpl(command, args) { + assert.equal(command, "/bin/ps"); + assert.deepEqual(args, ["-axo", "pid=,ppid=,pgid=,sess=,stat=,lstart="]); + const rows = []; + if (alive.has(100)) { + rows.push("100 1 100 100 S Mon Jul 27 00:00:00 2026"); + } + if (alive.has(300)) { + rows.push("300 1 300 100 S Mon Jul 27 00:00:02 2026"); + } + return { + command, + args, + status: 0, + signal: null, + stdout: rows.length > 0 ? `${rows.join("\n")}\n` : "", + stderr: "", + error: null + }; + }, + killImpl(pid, signal) { + signals.push([pid, signal]); + const groupId = Math.abs(pid); + for (const candidate of [...alive]) { + if (candidate === groupId) { + alive.delete(candidate); + } + } + } + }); + + assert.deepEqual(signals, [[-300, "SIGTERM"], [-100, "SIGTERM"]]); + assert.equal(outcome.verified, true); + assert.equal(outcome.degraded, false); + assert.equal(outcome.method, "process-session"); + assert.deepEqual(outcome.survivors, []); +}); + test("terminateProcessGroup signals nothing when the group id was reused by another leader", async () => { const signals = []; const ownershipSnapshot = { diff --git a/tests/registered-broker-reaper.test.mjs b/tests/registered-broker-reaper.test.mjs index 9adbc1ed0..bf946a568 100644 --- a/tests/registered-broker-reaper.test.mjs +++ b/tests/registered-broker-reaper.test.mjs @@ -138,6 +138,22 @@ test("registered cleanup reclaims children before the broker and writes a mode-0 assert.ok(receiptPath); assert.equal(fs.statSync(receiptPath).mode & 0o777, 0o600); assert.equal(JSON.parse(fs.readFileSync(receiptPath, "utf8")).decision, "cleanup-verified"); + + const receiptNames = fs.readdirSync(path.dirname(receiptPath)); + const repeated = await runRegisteredBrokerReaper({ + mode: "apply-registered", + env, + getLiveProcessPidsImpl: () => [], + terminateProcessGroupImpl: async () => { + throw new Error("a terminal registry must not retry child cleanup"); + }, + terminateProcessTreeImpl: async () => { + throw new Error("a terminal registry must not retry broker cleanup"); + }, + attemptIdFactory: () => "attempt-should-not-run" + }); + assert.equal(repeated.scanned, 0); + assert.deepEqual(fs.readdirSync(path.dirname(receiptPath)), receiptNames); }); test("each verified child is released before a later target can block convergence", async (t) => { diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 787a31476..1386357fe 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -513,11 +513,11 @@ test("fake app-server crash reclaims an observed regrouped helper without replac assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 1); }); -test("shared broker reclaims a post-snapshot helper before allowing replacement", async (t) => { +test("shared broker durably observes and reclaims a post-activation regrouped helper", async (t) => { const repo = makeTempDir(); const binDir = makeTempDir(); const fakeStatePath = path.join(binDir, "fake-codex-state.json"); - installFakeCodex(binDir, "crash-with-post-snapshot-helper"); + installFakeCodex(binDir, "crash-with-post-activation-regrouped-helper"); const env = withBrokerOwner({ ...buildEnv(binDir), CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" From ec40de806df423a996da3fd12e30666984835dfb Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Thu, 30 Jul 2026 10:16:16 -0700 Subject: [PATCH 25/29] [CC-5589] fail closed on observation contention Attributed-To: Codex --- plugins/codex/scripts/app-server-broker.mjs | 39 +++++++- tests/fake-codex-fixture.mjs | 11 ++- tests/runtime.test.mjs | 99 +++++++++++++++++++++ 3 files changed, 145 insertions(+), 4 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 4b15edbf1..64619e92e 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -23,6 +23,7 @@ const BROKER_CLEANUP_UNVERIFIED_RPC_CODE = -32002; const BROKER_SHUTDOWN_RPC_CODE = -32003; const BROKER_NOT_ACTIVATED_RPC_CODE = -32004; const BROKER_ACTIVATION_ACK = "activated"; +const CHILD_OBSERVATION_RETRY_DELAYS_MS = [10, 25, 50, 100]; export function isBrokerRequestAllowedDuringShutdown(shuttingDown, message) { return !shuttingDown || message?.method === "broker/shutdown"; @@ -72,6 +73,22 @@ function isInterruptRequest(message) { return message?.method === "turn/interrupt"; } +function isRegistryContention(reason) { + return reason === "registry-busy" || reason === "registry-lock-contention"; +} + +async function publishChildObservationWithRetry(registration, options) { + let observation = publishBrokerChildObservation(registration, options); + for (const delayMs of CHILD_OBSERVATION_RETRY_DELAYS_MS) { + if (observation.observed === true || !isRegistryContention(observation.reason)) { + return observation; + } + await new Promise((resolve) => setTimeout(resolve, delayMs)); + observation = publishBrokerChildObservation(registration, options); + } + return observation; +} + function writePidFile(pidFile) { if (!pidFile) { return; @@ -233,6 +250,19 @@ async function main() { await appClientClosePromise; } + async function prepareRequestErrorForReply(error) { + if (error?.requiresChildTeardown !== true) { + return error; + } + await closeAppClient(); + if (!blockedCleanup) { + return error; + } + const cleanupError = new Error("Shared Codex app-server cleanup is unverified after ownership publication failed."); + cleanupError.rpcCode = BROKER_CLEANUP_UNVERIFIED_RPC_CODE; + return cleanupError; + } + function scheduleChildIdleClose() { cancelChildIdleClose(); if (!appClient || hasActiveWork()) { @@ -348,13 +378,14 @@ async function main() { error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; throw error; } - const observation = publishBrokerChildObservation(registration, { + const observation = await publishChildObservationWithRetry(registration, { child: childRegistration.child, ownershipSnapshot }); if (observation.observed !== true) { const error = new Error(`Unable to publish shared Codex app-server helper ownership (${observation.reason ?? "unknown"}).`); error.rpcCode = BROKER_OWNERSHIP_RPC_CODE; + error.requiresChildTeardown = true; throw error; } childRegistration = { @@ -607,9 +638,10 @@ async function main() { const result = await client.request(message.method, message.params ?? {}); send(socket, { id: message.id, result }); } catch (error) { + const replyError = await prepareRequestErrorForReply(error); send(socket, { id: message.id, - error: buildJsonRpcError(error.rpcCode ?? -32000, error.message) + error: buildJsonRpcError(replyError.rpcCode ?? -32000, replyError.message) }); } finally { inFlightRequests -= 1; @@ -645,9 +677,10 @@ async function main() { activeRequestSocket = null; } } catch (error) { + const replyError = await prepareRequestErrorForReply(error); send(socket, { id: message.id, - error: buildJsonRpcError(error.rpcCode ?? -32000, error.message) + error: buildJsonRpcError(replyError.rpcCode ?? -32000, replyError.message) }); if (activeRequestSocket === socket) { activeRequestSocket = null; diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index 2cc0ad795..061221ebb 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -288,7 +288,7 @@ if (BEHAVIOR === "with-helper-child" || BEHAVIOR === "slow-task-with-helper-chil bootState.helperPids = [...(bootState.helperPids || []), helper.pid]; } } -if (BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "crash-with-post-snapshot-helper" || BEHAVIOR === "crash-with-post-activation-regrouped-helper") { +if (BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "crash-with-post-snapshot-helper" || BEHAVIOR === "crash-with-post-activation-regrouped-helper" || BEHAVIOR === "post-activation-helper-on-thread-list") { bootState.appServerPids = [...(bootState.appServerPids || []), process.pid]; } saveState(bootState); @@ -376,6 +376,15 @@ rl.on("line", (line) => { } case "thread/list": { + if (BEHAVIOR === "post-activation-helper-on-thread-list" && !state.helperPids?.length) { + const helper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { + detached: process.platform !== "win32", + stdio: "ignore" + }); + helper.unref(); + state.helperPids = [helper.pid]; + saveState(state); + } let threads = state.threads.slice(); if (message.params.cwd) { threads = threads.filter((thread) => thread.cwd === message.params.cwd); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 1386357fe..f0715b26c 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -590,6 +590,105 @@ test("shared broker durably observes and reclaims a post-activation regrouped he assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 2); }); +test("shared broker tears down a post-activation helper before reporting observation lock contention", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for helper ownership cleanup."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "post-activation-helper-on-thread-list"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" + }, "observation-contention"); + const brokerSession = await ensureBrokerSession(repo, { env }); + if (!brokerSession) { + t.skip("broker socket unavailable in this sandbox"); + return; + } + + let helperPid = null; + let appServerPid = null; + let registryLock = null; + const client = await CodexAppServerClient.connect(repo, { env }); + t.after(async () => { + await client.close().catch(() => {}); + if (registryLock?.acquired === true) { + const currentRegistration = loadBrokerRegistration({ + endpoint: brokerSession.endpoint, + brokerIdentity: brokerSession.pidIdentity, + env + }); + releaseBrokerRegistryLock(currentRegistration, registryLock); + registryLock = null; + } + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo }) + }); + for (const pid of [helperPid, appServerPid]) { + if (!pid) { + continue; + } + try { + process.kill(pid, "SIGKILL"); + } catch { + // Ignore fixture processes already reclaimed by the broker. + } + } + }); + + function isLive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code !== "ESRCH"; + } + } + + await client.request("account/read", {}); + const registration = loadBrokerRegistration({ + endpoint: brokerSession.endpoint, + brokerIdentity: brokerSession.pidIdentity, + env + }); + assert.equal(registration.registered, true); + registryLock = acquireBrokerRegistryLock(registration); + assert.equal(registryLock.acquired, true); + + try { + await assert.rejects( + client.request("thread/list", { cwd: repo }), + (error) => { + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + helperPid = state.helperPids?.[0] ?? null; + appServerPid = state.appServerPids?.[0] ?? null; + return ( + error?.rpcCode === -32005 && + /registry-busy/.test(error.message) && + helperPid !== null && + appServerPid !== null && + !isLive(helperPid) && + !isLive(appServerPid) + ); + } + ); + } finally { + assert.equal(releaseBrokerRegistryLock(registration, registryLock).released, true); + registryLock = null; + } + + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + assert.equal(state.appServerStarts, 1); + assert.equal(isLive(helperPid), false); + assert.equal(isLive(appServerPid), false); +}); + test("automatic broker registration preserves a shared broker until its final owner releases", async (t) => { if (process.platform === "win32") { t.skip("Unix process identities are required for the registered broker contract."); From 4f6a87f1810b84a851b50dac85a5dc9c05674215 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Thu, 30 Jul 2026 10:22:07 -0700 Subject: [PATCH 26/29] [CC-5589] constrain observed cleanup authority Attributed-To: Codex --- .../codex/scripts/lib/broker-ownership.mjs | 57 ++++++++----- tests/broker-ownership.test.mjs | 85 ++++++++++++++++++- 2 files changed, 122 insertions(+), 20 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-ownership.mjs b/plugins/codex/scripts/lib/broker-ownership.mjs index c1f78f26e..e15cad3b9 100644 --- a/plugins/codex/scripts/lib/broker-ownership.mjs +++ b/plugins/codex/scripts/lib/broker-ownership.mjs @@ -671,6 +671,39 @@ function validSnapshotMember(record) { ); } +function validOwnershipTree(snapshot, rootPid, rootIdentity) { + if (!Array.isArray(snapshot?.members) || snapshot.members.length === 0) { + return false; + } + const membersByPid = new Map(); + const memberIdentities = new Set(); + for (const member of snapshot.members) { + if ( + !validSnapshotMember(member) || + membersByPid.has(member.pid) || + memberIdentities.has(member.identity) + ) { + return false; + } + membersByPid.set(member.pid, member); + memberIdentities.add(member.identity); + } + const root = membersByPid.get(rootPid); + if (root?.identity !== rootIdentity || root.depth !== 0) { + return false; + } + for (const member of snapshot.members) { + if (member.pid === rootPid) { + continue; + } + const parent = membersByPid.get(member.parentPid); + if (!parent || member.depth !== parent.depth + 1) { + return false; + } + } + return true; +} + function validChildRecord(record, filePath, registration) { const snapshot = record?.ownershipSnapshot; return Boolean( @@ -688,15 +721,8 @@ function validChildRecord(record, filePath, registration) { (snapshot.sessionId == null || (isSafePid(snapshot.sessionId) && snapshot.sessionId === snapshot.rootPid)) && - Array.isArray(snapshot.members) && - snapshot.members.length > 0 && - snapshot.members.every(validSnapshotMember) && - snapshot.members.some( - (member) => - member.pid === record.pid && - member.identity === record.pidIdentity && - (snapshot.sessionId == null || member.sessionId === snapshot.sessionId) - ) + validOwnershipTree(snapshot, record.pid, record.pidIdentity) && + (snapshot.sessionId == null || snapshot.members.find((member) => member.pid === record.pid)?.sessionId === snapshot.sessionId) ); } @@ -716,15 +742,8 @@ function validChildObservation(record, filePath, registration, child) { snapshot.processGroupId === child.processGroupId && (snapshot.sessionId == null || (isSafePid(snapshot.sessionId) && snapshot.sessionId === snapshot.rootPid)) && - Array.isArray(snapshot.members) && - snapshot.members.length > 0 && - snapshot.members.every(validSnapshotMember) && - snapshot.members.some( - (member) => - member.pid === child.pid && - member.identity === child.pidIdentity && - (snapshot.sessionId == null || member.sessionId === snapshot.sessionId) - ) && + validOwnershipTree(snapshot, child.pid, child.pidIdentity) && + (snapshot.sessionId == null || snapshot.members.find((member) => member.pid === child.pid)?.sessionId === snapshot.sessionId) && typeof record.observedAt === "string" && record.observedAt.length > 0 ); @@ -1086,7 +1105,7 @@ export function publishBrokerChild(registration, options = {}) { !isSafePid(ownershipSnapshot?.processGroupId) || (ownershipSnapshot?.sessionId != null && (!isSafePid(ownershipSnapshot.sessionId) || ownershipSnapshot.sessionId !== pid)) || - !Array.isArray(ownershipSnapshot?.members) + !validOwnershipTree(ownershipSnapshot, pid, identity) ) { return { registered: false, reason: "child-identity-unavailable" }; } diff --git a/tests/broker-ownership.test.mjs b/tests/broker-ownership.test.mjs index fbbcd77ab..5f9687c3d 100644 --- a/tests/broker-ownership.test.mjs +++ b/tests/broker-ownership.test.mjs @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { createHash } from "node:crypto"; import test from "node:test"; import assert from "node:assert/strict"; @@ -478,6 +479,7 @@ test("post-activation child observations extend durable cleanup ownership", (t) } }); const helperIdentity = "6202@Mon Jul 27 00:08:02 2026"; + const helperParentIdentity = "6201@Mon Jul 27 00:08:01 2026"; const observed = publishBrokerChildObservation(registration, { child: child.child, ownershipSnapshot: { @@ -494,6 +496,15 @@ test("post-activation child observations extend durable cleanup ownership", (t) identity: rootIdentity, depth: 0 }, + { + pid: 6201, + parentPid: 6200, + processGroupId: 6200, + state: "S", + startedAt: "Mon Jul 27 00:08:01 2026", + identity: helperParentIdentity, + depth: 1 + }, { pid: 6202, parentPid: 6201, @@ -522,6 +533,7 @@ test("post-activation child observations extend durable cleanup ownership", (t) assert.equal(loaded.valid, true); assert.deepEqual(loaded.children[0].ownershipSnapshot.members.map((member) => member.identity), [ rootIdentity, + helperParentIdentity, helperIdentity ]); @@ -530,5 +542,76 @@ test("post-activation child observations extend durable cleanup ownership", (t) cleanupOutcome: { verified: true, survivors: [], survivorIdentities: [] } }); assert.equal(released.released, true); - assert.equal(loadBrokerChildren(registration).releasedChildren[0].ownershipSnapshot.members.length, 2); + assert.equal(loadBrokerChildren(registration).releasedChildren[0].ownershipSnapshot.members.length, 3); +}); + +test("an observation member outside the immutable child tree makes the registry report-only", (t) => { + const { registration } = makeFixture(t); + const rootIdentity = "6300@Mon Jul 27 00:09:00 2026"; + const child = publishBrokerChild(registration, { + ownershipSnapshot: { + rootPid: 6300, + rootIdentity, + processGroupId: 6300, + members: [ + { + pid: 6300, + parentPid: 4100, + processGroupId: 6300, + state: "S", + startedAt: "Mon Jul 27 00:09:00 2026", + identity: rootIdentity, + depth: 0 + } + ] + } + }); + const ownershipSnapshot = { + rootPid: 6300, + rootIdentity, + processGroupId: 6300, + members: [ + child.child.ownershipSnapshot.members[0], + { + pid: 9900, + parentPid: 1, + processGroupId: 9900, + state: "S", + startedAt: "Mon Jul 27 00:09:01 2026", + identity: "9900@Mon Jul 27 00:09:01 2026", + depth: 1 + } + ] + }; + + const refused = publishBrokerChildObservation(registration, { + child: child.child, + ownershipSnapshot + }); + assert.equal(refused.observed, false); + assert.equal(refused.reason, "child-observation-invalid"); + + const observationKey = createHash("sha256").update(JSON.stringify(ownershipSnapshot)).digest("hex"); + const observationDir = path.join(registration.registryDir, "child-observations", child.child.childKey); + const observationPath = path.join(observationDir, `${observationKey}.json`); + fs.mkdirSync(observationDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync( + observationPath, + `${JSON.stringify({ + version: 1, + kind: "child-observation", + brokerKey: registration.brokerKey, + childKey: child.child.childKey, + observationKey, + ownershipSnapshot, + observedAt: "2026-07-27T00:09:02.000Z" + })}\n`, + { mode: 0o600 } + ); + + const loaded = loadBrokerChildren(registration); + assert.equal(loaded.valid, false); + assert.equal(loaded.reason, "malformed-child-registry"); + assert.deepEqual(loaded.children, []); + assert.deepEqual(loaded.malformed, [observationPath]); }); From d17945529a8bd488f4078b083735a45a0ac84a55 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Fri, 31 Jul 2026 10:26:35 -0700 Subject: [PATCH 27/29] [CC-5589] Bind broker locks and owner publication to process identity Attributed-To: Codex --- .../codex/scripts/lib/broker-ownership.mjs | 97 +++++++++++--- tests/broker-ownership.test.mjs | 124 ++++++++++++++++-- tests/registered-broker-reaper.test.mjs | 46 ++++++- tests/runtime.test.mjs | 55 ++++++++ 4 files changed, 288 insertions(+), 34 deletions(-) diff --git a/plugins/codex/scripts/lib/broker-ownership.mjs b/plugins/codex/scripts/lib/broker-ownership.mjs index e15cad3b9..d76686eb5 100644 --- a/plugins/codex/scripts/lib/broker-ownership.mjs +++ b/plugins/codex/scripts/lib/broker-ownership.mjs @@ -3,7 +3,7 @@ import fs from "node:fs"; import path from "node:path"; import process from "node:process"; -import { getLiveProcessPids, hasLiveProcessIdentity } from "./process.mjs"; +import { getLiveProcessPids, getProcessIdentity, hasLiveProcessIdentity } from "./process.mjs"; export const BROKER_OWNERSHIP_VERSION = 1; export const SESSION_OWNER_PID_ENV = "CODEX_COMPANION_SESSION_OWNER_PID"; @@ -143,6 +143,8 @@ function validRegistryLock(registration, lock) { lock?.acquired === true && lock.brokerKey === registration.brokerKey && isRegistryLockToken(lock.token) && + isSafePid(lock.pid) && + isPidIdentity(lock.pid, lock.pidIdentity) && lock.path === path.join(registration.registryDir, REGISTRY_LOCK_DIR_NAME) ); } @@ -152,6 +154,18 @@ function createPreparedRegistryLock(registration, preparedRegistryDir, options = if (!isSafePid(pid)) { return { acquired: false, reason: "registry-lock-owner-invalid" }; } + const getProcessIdentityImpl = options.getProcessIdentityImpl ?? getProcessIdentity; + let pidIdentity = options.pidIdentity ?? null; + if (!pidIdentity) { + try { + pidIdentity = getProcessIdentityImpl(pid); + } catch { + return { acquired: false, reason: "registry-lock-owner-identity-unavailable" }; + } + } + if (!isPidIdentity(pid, pidIdentity)) { + return { acquired: false, reason: "registry-lock-owner-identity-unavailable" }; + } const token = randomUUID(); const preparedLockPath = path.join(preparedRegistryDir, REGISTRY_LOCK_DIR_NAME); ensurePrivateDir(preparedLockPath); @@ -161,12 +175,15 @@ function createPreparedRegistryLock(registration, preparedRegistryDir, options = brokerKey: registration.brokerKey, token, pid, + pidIdentity, acquiredAt: (options.now ?? (() => new Date().toISOString()))() }); return { acquired: true, brokerKey: registration.brokerKey, token, + pid, + pidIdentity, path: path.join(registration.registryDir, REGISTRY_LOCK_DIR_NAME) }; } @@ -176,7 +193,10 @@ export function acquireBrokerRegistryLock( { now = () => new Date().toISOString(), pid = process.pid, - getLiveProcessPidsImpl = getLiveProcessPids + pidIdentity: suppliedPidIdentity = null, + getProcessIdentityImpl = getProcessIdentity, + getLiveProcessPidsImpl = getLiveProcessPids, + hasLiveProcessIdentityImpl = hasLiveProcessIdentity } = {} ) { if (!validRegistrationReference(registration)) { @@ -184,6 +204,17 @@ export function acquireBrokerRegistryLock( } requirePrivateDirectory(registration.registryRoot); requirePrivateDirectory(registration.registryDir); + let pidIdentity = suppliedPidIdentity; + if (!pidIdentity) { + try { + pidIdentity = getProcessIdentityImpl(pid); + } catch { + return { acquired: false, reason: "registry-lock-owner-identity-unavailable" }; + } + } + if (!isPidIdentity(pid, pidIdentity)) { + return { acquired: false, reason: "registry-lock-owner-identity-unavailable" }; + } const lockPath = path.join(registration.registryDir, REGISTRY_LOCK_DIR_NAME); const token = randomUUID(); for (let attempt = 0; attempt < 3; attempt += 1) { @@ -196,11 +227,12 @@ export function acquireBrokerRegistryLock( brokerKey: registration.brokerKey, token, pid, + pidIdentity, acquiredAt: now() }); try { fs.renameSync(preparedPath, lockPath); - return { acquired: true, brokerKey: registration.brokerKey, token, path: lockPath }; + return { acquired: true, brokerKey: registration.brokerKey, token, pid, pidIdentity, path: lockPath }; } catch (error) { if (error?.code !== "EEXIST" && error?.code !== "ENOTEMPTY") { throw error; @@ -227,17 +259,29 @@ export function acquireBrokerRegistryLock( existingOwner.kind !== "registry-lock" || existingOwner.brokerKey !== registration.brokerKey || !isRegistryLockToken(existingOwner.token) || - !isSafePid(existingOwner.pid) + !isSafePid(existingOwner.pid) || + (existingOwner.pidIdentity != null && !isPidIdentity(existingOwner.pid, existingOwner.pidIdentity)) ) { return { acquired: false, reason: "registry-lock-malformed", path: lockPath }; } - let live; + let live = false; try { - live = getLiveProcessPidsImpl([existingOwner.pid]); + if (existingOwner.pidIdentity == null) { + const livePids = getLiveProcessPidsImpl([existingOwner.pid]); + if (!Array.isArray(livePids)) { + return { acquired: false, reason: "registry-lock-liveness-unavailable", path: lockPath }; + } + live = livePids.includes(existingOwner.pid); + } else { + live = hasLiveProcessIdentityImpl(existingOwner.pid, existingOwner.pidIdentity); + if (typeof live !== "boolean") { + return { acquired: false, reason: "registry-lock-liveness-unavailable", path: lockPath }; + } + } } catch { return { acquired: false, reason: "registry-lock-liveness-unavailable", path: lockPath }; } - if (!Array.isArray(live) || live.includes(existingOwner.pid)) { + if (live) { return { acquired: false, reason: "registry-busy", path: lockPath }; } @@ -267,7 +311,9 @@ export function releaseBrokerRegistryLock(registration, lock) { owner?.version !== BROKER_OWNERSHIP_VERSION || owner.kind !== "registry-lock" || owner.brokerKey !== registration.brokerKey || - owner.token !== lock.token + owner.token !== lock.token || + owner.pid !== lock.pid || + owner.pidIdentity !== lock.pidIdentity ) { return { released: false, reason: "registry-lock-mismatch" }; } @@ -451,6 +497,7 @@ export function publishRegisteredBroker({ env = process.env, now = () => new Date().toISOString(), hasLiveProcessIdentityImpl = hasLiveProcessIdentity, + getProcessIdentityImpl = getProcessIdentity, retainRegistryLock = false }) { const registryRoot = resolveBrokerOwnershipRoot(env); @@ -507,7 +554,7 @@ export function publishRegisteredBroker({ createImmutableJson(path.join(preparedDir, "broker.json"), broker); createImmutableJson(path.join(preparedDir, "owners", `${owner.ownerKey}.json`), ownerRecord); const registryLock = retainRegistryLock - ? createPreparedRegistryLock(registration, preparedDir, { now }) + ? createPreparedRegistryLock(registration, preparedDir, { now, getProcessIdentityImpl }) : null; if (retainRegistryLock && registryLock?.acquired !== true) { return { registered: false, reason: registryLock?.reason ?? "registry-lock-unavailable" }; @@ -547,15 +594,13 @@ export function registerBrokerOwner(registration, options = {}) { return { registered: false, reason: "session-owner-unavailable" }; } const locked = withBrokerRegistryLock(registration, options, () => { - const payload = { - version: BROKER_OWNERSHIP_VERSION, - kind: "owner", - brokerKey: registration.brokerKey, - ownerKey: owner.ownerKey, - sessionId: owner.sessionId, - pid: owner.pid, - pidIdentity: owner.identity, - registeredAt: now() + const hasLiveIdentity = options.hasLiveProcessIdentityImpl ?? hasLiveProcessIdentity; + const ownerIsLive = () => { + try { + return hasLiveIdentity(owner.pid, owner.identity) === true; + } catch { + return false; + } }; const ownerPath = path.join(registration.registryDir, "owners", `${owner.ownerKey}.json`); if (fs.existsSync(ownerPath)) { @@ -565,9 +610,25 @@ export function registerBrokerOwner(registration, options = {}) { error.code = "BROKER_OWNERSHIP_COLLISION"; throw error; } + if (!ownerIsLive()) { + return { registered: false, reason: "session-owner-not-live" }; + } fs.chmodSync(ownerPath, 0o600); return { registered: true, ownerKey: owner.ownerKey, path: ownerPath, owner: existing }; } + const payload = { + version: BROKER_OWNERSHIP_VERSION, + kind: "owner", + brokerKey: registration.brokerKey, + ownerKey: owner.ownerKey, + sessionId: owner.sessionId, + pid: owner.pid, + pidIdentity: owner.identity, + registeredAt: now() + }; + if (!ownerIsLive()) { + return { registered: false, reason: "session-owner-not-live" }; + } createImmutableJson(ownerPath, payload); return { registered: true, ownerKey: owner.ownerKey, path: ownerPath, owner: payload }; }); diff --git a/tests/broker-ownership.test.mjs b/tests/broker-ownership.test.mjs index 5f9687c3d..451088cb5 100644 --- a/tests/broker-ownership.test.mjs +++ b/tests/broker-ownership.test.mjs @@ -6,19 +6,62 @@ import test from "node:test"; import assert from "node:assert/strict"; import { - acquireBrokerRegistryLock, + acquireBrokerRegistryLock as acquireBrokerRegistryLockImpl, assessBrokerOwners, loadBrokerChildren, - publishBrokerChild, - publishBrokerChildObservation, + publishBrokerChild as publishBrokerChildImpl, + publishBrokerChildObservation as publishBrokerChildObservationImpl, publishBrokerRegistration, publishRegisteredBroker, - registerBrokerOwner, - releaseBrokerChild, - releaseBrokerOwner, + registerBrokerOwner as registerBrokerOwnerImpl, + releaseBrokerChild as releaseBrokerChildImpl, + releaseBrokerOwner as releaseBrokerOwnerImpl, releaseBrokerRegistryLock } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; +const TEST_LOCK_PID = 4900; +const TEST_LOCK_IDENTITY = "4900@Mon Jul 27 00:00:40 2026"; + +function withTestRegistryLock(options = {}) { + const pid = options.pid ?? TEST_LOCK_PID; + const pidIdentity = options.pidIdentity ?? (pid === TEST_LOCK_PID + ? TEST_LOCK_IDENTITY + : `${pid}@Mon Jul 27 00:00:40 2026`); + return { + ...options, + pid, + pidIdentity, + hasLiveProcessIdentityImpl: () => true, + ...(options.hasLiveProcessIdentityImpl + ? { hasLiveProcessIdentityImpl: options.hasLiveProcessIdentityImpl } + : {}) + }; +} + +function acquireBrokerRegistryLock(registration, options = {}) { + return acquireBrokerRegistryLockImpl(registration, withTestRegistryLock(options)); +} + +function registerBrokerOwner(registration, options = {}) { + return registerBrokerOwnerImpl(registration, withTestRegistryLock(options)); +} + +function releaseBrokerOwner(registration, options = {}) { + return releaseBrokerOwnerImpl(registration, withTestRegistryLock(options)); +} + +function publishBrokerChild(registration, options = {}) { + return publishBrokerChildImpl(registration, withTestRegistryLock(options)); +} + +function publishBrokerChildObservation(registration, options = {}) { + return publishBrokerChildObservationImpl(registration, withTestRegistryLock(options)); +} + +function releaseBrokerChild(registration, options = {}) { + return releaseBrokerChildImpl(registration, withTestRegistryLock(options)); +} + function makeFixture(t) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-broker-ownership-")); t.after(() => fs.rmSync(root, { recursive: true, force: true })); @@ -105,7 +148,8 @@ test("initial publication can make its transaction lock visible atomically", (t) }, env: ownerEnv({ CLAUDE_PLUGIN_DATA: root }, "session-atomic-lock", 5050, "Mon Jul 27 00:00:45 2026"), retainRegistryLock: true, - hasLiveProcessIdentityImpl: () => true + hasLiveProcessIdentityImpl: () => true, + getProcessIdentityImpl: (pid) => `${pid}@Mon Jul 27 00:00:41 2026` }); assert.equal(registration.registered, true); @@ -210,6 +254,36 @@ test("initial publication revalidates the broker identity immediately before vis assert.deepEqual(fs.readdirSync(path.join(root, "state", "broker-ownership-v1")), []); }); +test("owner publication revalidates the exact owner while holding the registry lock", (t) => { + const { registration, env } = makeFixture(t); + const sessionEnv = ownerEnv(env, "session-revalidated", 5060, "Mon Jul 27 00:00:46 2026"); + const lockOptions = { + pid: 5061, + pidIdentity: "5061@Mon Jul 27 00:00:47 2026" + }; + + const rejectedCreate = registerBrokerOwner(registration, { + env: sessionEnv, + ...lockOptions, + hasLiveProcessIdentityImpl: () => false + }); + assert.deepEqual(rejectedCreate, { registered: false, reason: "session-owner-not-live" }); + + const registered = registerBrokerOwner(registration, { + env: sessionEnv, + ...lockOptions, + hasLiveProcessIdentityImpl: () => true + }); + assert.equal(registered.registered, true); + + const rejectedExisting = registerBrokerOwner(registration, { + env: sessionEnv, + ...lockOptions, + hasLiveProcessIdentityImpl: () => false + }); + assert.deepEqual(rejectedExisting, { registered: false, reason: "session-owner-not-live" }); +}); + test("owner publication fails closed while cleanup holds the registry lock", (t) => { const { registration, env } = makeFixture(t); const lock = acquireBrokerRegistryLock(registration, { @@ -246,7 +320,7 @@ test("a well-formed lock whose creator is absent is quarantined before retry", ( const registered = registerBrokerOwner(registration, { env: ownerEnv(env, "session-after-stale-lock", 5051, "Mon Jul 27 00:00:46 2026"), - getLiveProcessPidsImpl: () => [] + hasLiveProcessIdentityImpl: (pid) => pid !== 5001 }); assert.equal(registered.registered, true); const staleRoot = path.join(registration.registryDir, "stale-locks"); @@ -254,6 +328,34 @@ test("a well-formed lock whose creator is absent is quarantined before retry", ( assert.equal(fs.existsSync(stale.path), false); }); +test("a reused lock-owner PID does not keep an abandoned registry lock busy", (t) => { + const { registration } = makeFixture(t); + const staleIdentity = "5006@Mon Jul 27 00:00:36 2026"; + const stale = acquireBrokerRegistryLock(registration, { + pid: 5006, + pidIdentity: staleIdentity, + now: () => "2026-07-27T00:00:36.000Z" + }); + assert.equal(stale.acquired, true); + + const contender = acquireBrokerRegistryLock(registration, { + pid: 5007, + pidIdentity: "5007@Mon Jul 27 00:00:37 2026", + now: () => "2026-07-27T00:00:37.000Z", + getLiveProcessPidsImpl: () => [5006], + hasLiveProcessIdentityImpl(pid, identity) { + assert.equal(pid, 5006); + assert.equal(identity, staleIdentity); + return false; + } + }); + + assert.equal(contender.acquired, true); + const owner = JSON.parse(fs.readFileSync(path.join(contender.path, "owner.json"), "utf8")); + assert.equal(owner.pidIdentity, "5007@Mon Jul 27 00:00:37 2026"); + assert.deepEqual(releaseBrokerRegistryLock(registration, contender), { released: true }); +}); + test("a stale-lock contender cannot quarantine a replacement live lock", (t) => { const { registration } = makeFixture(t); const stale = acquireBrokerRegistryLock(registration, { @@ -267,7 +369,7 @@ test("a stale-lock contender cannot quarantine a replacement live lock", (t) => const contender = acquireBrokerRegistryLock(registration, { pid: 5004, now: () => "2026-07-27T00:00:34.000Z", - getLiveProcessPidsImpl(pids) { + hasLiveProcessIdentityImpl(pid) { livenessChecks += 1; if (livenessChecks === 1) { const staleRoot = path.join(registration.registryDir, "stale-locks"); @@ -279,9 +381,9 @@ test("a stale-lock contender cannot quarantine a replacement live lock", (t) => now: () => "2026-07-27T00:00:33.000Z" }); assert.equal(replacement.acquired, true); - return []; + return false; } - return pids.includes(5003) ? [5003] : []; + return pid === 5003; } }); diff --git a/tests/registered-broker-reaper.test.mjs b/tests/registered-broker-reaper.test.mjs index bf946a568..1d119cdcb 100644 --- a/tests/registered-broker-reaper.test.mjs +++ b/tests/registered-broker-reaper.test.mjs @@ -5,15 +5,51 @@ import path from "node:path"; import test from "node:test"; import { - acquireBrokerRegistryLock, + acquireBrokerRegistryLock as acquireBrokerRegistryLockImpl, loadBrokerChildren, - publishBrokerChild, + publishBrokerChild as publishBrokerChildImpl, publishBrokerRegistration, - registerBrokerOwner, - releaseBrokerOwner, + registerBrokerOwner as registerBrokerOwnerImpl, + releaseBrokerOwner as releaseBrokerOwnerImpl, releaseBrokerRegistryLock } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; -import { runRegisteredBrokerReaper } from "../plugins/codex/scripts/registered-broker-reaper.mjs"; +import { runRegisteredBrokerReaper as runRegisteredBrokerReaperImpl } from "../plugins/codex/scripts/registered-broker-reaper.mjs"; + +const TEST_LOCK_PID = 4950; +const TEST_LOCK_IDENTITY = "4950@Mon Jul 27 00:00:50 2026"; + +function withTestRegistryLock(options = {}) { + return { + pid: options.pid ?? TEST_LOCK_PID, + pidIdentity: options.pidIdentity ?? TEST_LOCK_IDENTITY, + hasLiveProcessIdentityImpl: () => true, + ...options + }; +} + +function acquireBrokerRegistryLock(registration, options = {}) { + return acquireBrokerRegistryLockImpl(registration, withTestRegistryLock(options)); +} + +function publishBrokerChild(registration, options = {}) { + return publishBrokerChildImpl(registration, withTestRegistryLock(options)); +} + +function registerBrokerOwner(registration, options = {}) { + return registerBrokerOwnerImpl(registration, withTestRegistryLock(options)); +} + +function releaseBrokerOwner(registration, options = {}) { + return releaseBrokerOwnerImpl(registration, withTestRegistryLock(options)); +} + +function runRegisteredBrokerReaper(options = {}) { + return runRegisteredBrokerReaperImpl({ + ...options, + acquireBrokerRegistryLockImpl: + options.acquireBrokerRegistryLockImpl ?? ((registration) => acquireBrokerRegistryLock(registration)) + }); +} function ownerEnv(env, sessionId, pid) { const startedAt = `Mon Jul 27 00:${String(pid % 60).padStart(2, "0")}:00 2026`; diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index f0715b26c..6c6e860f1 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -829,6 +829,61 @@ test("existing broker reuse holds the registry lock through owner publication", assert.equal(heldLock, null); }); +test("existing broker reuse refuses an owner that dies at the publication boundary", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const baseEnv = { + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir, + [SESSION_OWNER_PID_ENV]: String(process.pid), + [SESSION_OWNER_IDENTITY_ENV]: getProcessIdentity(process.pid) + }; + const envA = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "publication-owner-a" }; + const envB = { ...baseEnv, CODEX_COMPANION_SESSION_ID: "publication-owner-b" }; + const first = await ensureBrokerSession(repo, { env: envA }); + t.after(() => { + run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: envA, + input: JSON.stringify({ hook_event_name: "SessionEnd", cwd: repo, session_id: "publication-owner-a" }) + }); + }); + + let publicationChecks = 0; + const reused = await ensureBrokerSession(repo, { + env: envB, + registerBrokerOwnerImpl(registration, options) { + return registerBrokerOwner(registration, { + ...options, + hasLiveProcessIdentityImpl(pid, identity) { + publicationChecks += 1; + assert.equal(pid, process.pid); + assert.equal(identity, envB[SESSION_OWNER_IDENTITY_ENV]); + return false; + } + }); + } + }); + + assert.equal(reused, null); + assert.equal(publicationChecks, 1); + assert.equal(loadBrokerSession(repo)?.pid, first.pid); + const registration = loadBrokerRegistration({ + endpoint: first.endpoint, + brokerIdentity: first.pidIdentity, + env: envB + }); + const owners = fs.readdirSync(path.join(registration.registryDir, "owners")) + .map((name) => JSON.parse(fs.readFileSync(path.join(registration.registryDir, "owners", name), "utf8"))); + assert.equal(owners.some((owner) => owner.sessionId === "publication-owner-b"), false); +}); + test("deterministic broker launch lock never splits across fallback resources", async (t) => { const repo = makeTempDir(); const port = brokerLaunchLockPort(repo); From d735a66a62e5c7f1a7372b4a9fb7b5be40d1f280 Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Fri, 31 Jul 2026 10:47:25 -0700 Subject: [PATCH 28/29] [CC-5589] Close late orphan lifecycle races Attributed-To: Codex --- plugins/codex/scripts/lib/app-server.mjs | 47 +++++- .../codex/scripts/lib/broker-lifecycle.mjs | 37 +++++ plugins/codex/scripts/lib/state.mjs | 8 +- plugins/codex/scripts/lib/tracked-jobs.mjs | 8 +- tests/fake-codex-fixture.mjs | 18 ++- tests/runtime.test.mjs | 139 +++++++++++++++++- 6 files changed, 243 insertions(+), 14 deletions(-) diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index cb88c0ef4..330e3ad02 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -159,10 +159,14 @@ class AppServerClientBase { } if (message.method && this.notificationHandler) { - this.notificationHandler(/** @type {AppServerNotification} */ (message)); + this.handleNotification(/** @type {AppServerNotification} */ (message)); } } + handleNotification(message) { + this.notificationHandler?.(message); + } + handleServerRequest(message) { this.sendMessage({ id: message.id, @@ -195,6 +199,7 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { constructor(cwd, options = {}) { super(cwd, options); this.transport = "direct"; + this.notificationRefreshPromise = Promise.resolve(); } async initialize() { @@ -305,6 +310,29 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { return ownershipSnapshot; } + handleNotification(message) { + this.notificationRefreshPromise = this.notificationRefreshPromise.then(async () => { + if (this.exitResolved) { + return; + } + try { + // Streaming requests return before their turn is complete. Publish a + // fresh helper observation before forwarding each later notification + // so independently grouped helpers are durable before a crash can + // strand them outside both local and registered cleanup. + await this.refreshOwnership(); + } catch (error) { + this.startUnexpectedExitCleanup(this.proc?.pid); + this.handleExit(error); + return; + } + if (!this.closed) { + this.notificationHandler?.(message); + } + }); + return this.notificationRefreshPromise; + } + async request(method, params) { let result; let requestError; @@ -333,12 +361,15 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { return this.unexpectedExitCleanupPromise ?? null; } - this.unexpectedExitCleanupPromise = terminateProcessGroup(pid, { - ownershipSnapshot: this.ownershipSnapshot, - cwd: this.cwd, - env: this.options.env ?? process.env, - warnImpl: () => {} - }) + const observationBarrier = this.notificationRefreshPromise ?? Promise.resolve(); + this.unexpectedExitCleanupPromise = observationBarrier + .catch(() => {}) + .then(() => terminateProcessGroup(pid, { + ownershipSnapshot: this.ownershipSnapshot, + cwd: this.cwd, + env: this.options.env ?? process.env, + warnImpl: () => {} + })) .then((outcome) => { this.cleanupOutcome = normalizeProcessCleanupOutcome(outcome); if (!outcome.verified) { @@ -376,6 +407,8 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { this.closed = true; + await this.notificationRefreshPromise?.catch(() => {}); + if (this.readline) { this.readline.close(); } diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index 73c331146..8e5b168d5 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -10,9 +10,11 @@ import { acquireBrokerRegistryLock, assessBrokerOwners, hasLiveBrokerOwnerIdentity, + loadBrokerChildren, loadBrokerRegistration, publishRegisteredBroker, registerBrokerOwner, + releaseBrokerChild, releaseBrokerOwner, releaseBrokerRegistryLock, resolveBrokerOwnershipRoot @@ -471,6 +473,41 @@ async function cleanupExistingBrokerSession(cwd, existing, options) { } } + if (!currentIdentity) { + const loadChildren = options.loadBrokerChildrenImpl ?? loadBrokerChildren; + const children = loadChildren(registration); + if (children?.valid !== true) { + const error = new Error(`Registered broker children are invalid (${children?.reason ?? "unknown"}); cleanup remains report-only.`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + const terminateChild = options.terminateBrokerChildImpl ?? terminateProcessTree; + const releaseChild = options.releaseBrokerChildImpl ?? releaseBrokerChild; + for (const child of children.children) { + const outcome = await terminateChild(child.pid, { + expectedRootIdentity: child.pidIdentity, + ownershipSnapshot: child.ownershipSnapshot, + cwd, + env + }); + if (outcome?.verified !== true) { + const error = new Error("Registered broker child cleanup is unverified; refusing to start a replacement broker."); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + const released = releaseChild(registration, { + child, + cleanupOutcome: outcome, + registryLock + }); + if (released?.released !== true) { + const error = new Error(`Registered broker child release failed (${released?.reason ?? "unknown"}); refusing to start a replacement broker.`); + error.code = "BROKER_CLEANUP_UNVERIFIED"; + throw error; + } + } + } + const teardownBroker = options.teardownBrokerSessionImpl ?? teardownBrokerSession; const cleanup = await teardownBroker({ endpoint: existing.endpoint ?? null, diff --git a/plugins/codex/scripts/lib/state.mjs b/plugins/codex/scripts/lib/state.mjs index 6878ae4f8..63329e4e7 100644 --- a/plugins/codex/scripts/lib/state.mjs +++ b/plugins/codex/scripts/lib/state.mjs @@ -107,7 +107,7 @@ export function saveState(cwd, state) { if (retainedIds.has(job.id)) { continue; } - removeJobFile(cwd, job.id); + removeJobFile(cwd, job.id, { preserveCancelFlag: hasCancelFlag(cwd, job.id) }); removeFileIfExists(job.logFile); } @@ -194,9 +194,11 @@ export function removeCancelFlag(cwd, jobId) { removeFileIfExists(resolveCancelFlag(cwd, jobId)); } -function removeJobFile(cwd, jobId) { +function removeJobFile(cwd, jobId, options = {}) { removeFileIfExists(resolveJobFile(cwd, jobId)); - removeCancelFlag(cwd, jobId); + if (options.preserveCancelFlag !== true) { + removeCancelFlag(cwd, jobId); + } } export function resolveJobLogFile(cwd, jobId) { diff --git a/plugins/codex/scripts/lib/tracked-jobs.mjs b/plugins/codex/scripts/lib/tracked-jobs.mjs index 3414ed3c5..995ede1bc 100644 --- a/plugins/codex/scripts/lib/tracked-jobs.mjs +++ b/plugins/codex/scripts/lib/tracked-jobs.mjs @@ -1,7 +1,7 @@ import fs from "node:fs"; import process from "node:process"; -import { hasCancelFlag, readJobFile, resolveJobFile, resolveJobLogFile, upsertJob, writeJobFile } from "./state.mjs"; +import { hasCancelFlag, readJobFile, removeCancelFlag, resolveJobFile, resolveJobLogFile, upsertJob, writeJobFile } from "./state.mjs"; export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; const JOB_CANCELLED_CODE = "JOB_CANCELLED"; @@ -211,6 +211,12 @@ export async function runTrackedJob(job, runner, options = {}) { errorMessage, completedAt }); + if (terminalStatus === "cancelled") { + // The worker has now durably acknowledged the cancellation. Until this + // point the tombstone must survive state pruning so a worker that read + // the queued request cannot cross the read-to-start race. + removeCancelFlag(job.workspaceRoot, job.id); + } throw error; } } diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index 061221ebb..7978407c6 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -288,7 +288,7 @@ if (BEHAVIOR === "with-helper-child" || BEHAVIOR === "slow-task-with-helper-chil bootState.helperPids = [...(bootState.helperPids || []), helper.pid]; } } -if (BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "crash-with-post-snapshot-helper" || BEHAVIOR === "crash-with-post-activation-regrouped-helper" || BEHAVIOR === "post-activation-helper-on-thread-list") { +if (BEHAVIOR === "crash-with-regrouped-helper" || BEHAVIOR === "crash-with-post-snapshot-helper" || BEHAVIOR === "crash-with-post-activation-regrouped-helper" || BEHAVIOR === "post-activation-helper-on-thread-list" || BEHAVIOR === "streaming-helper-after-response") { bootState.appServerPids = [...(bootState.appServerPids || []), process.pid]; } saveState(bootState); @@ -511,6 +511,22 @@ rl.on("line", (line) => { saveState(state); send({ id: message.id, result: { turn: buildTurn(turnId) } }); + if (BEHAVIOR === "streaming-helper-after-response") { + setTimeout(() => { + const helper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { + detached: process.platform !== "win32", + stdio: "ignore" + }); + helper.unref(); + const current = loadState(); + current.helperPids = [...(current.helperPids || []), helper.pid]; + saveState(current); + send({ method: "item/started", params: { threadId: thread.id, turnId, item: { type: "commandExecution", id: "late-helper" } } }); + setTimeout(() => process.exit(1), 750); + }, 100); + break; + } + const payload = message.params.outputSchema && message.params.outputSchema.properties && message.params.outputSchema.properties.verdict ? structuredReviewPayload(prompt) : taskPayload(prompt, thread.name && thread.name.startsWith("Codex Companion Task") && prompt.includes("Continue from the current thread state")); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 6c6e860f1..4c42acea1 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -12,11 +12,11 @@ import { enqueueBackgroundTask, handleCancel, handleTaskWorker } from "../plugin import { cleanupSessionJobs, handleSessionEnd, handleSessionStart } from "../plugins/codex/scripts/session-lifecycle-hook.mjs"; import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; import { isBrokerRequestAllowedDuringShutdown } from "../plugins/codex/scripts/app-server-broker.mjs"; -import { acquireBrokerRegistryLock, loadBrokerChildren, loadBrokerRegistration, publishRegisteredBroker, registerBrokerOwner, releaseBrokerOwner, releaseBrokerRegistryLock, SESSION_OWNER_IDENTITY_ENV, SESSION_OWNER_PID_ENV } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; +import { acquireBrokerRegistryLock, loadBrokerChildren, loadBrokerRegistration, publishBrokerChild, publishRegisteredBroker, registerBrokerOwner, releaseBrokerOwner, releaseBrokerRegistryLock, SESSION_OWNER_IDENTITY_ENV, SESSION_OWNER_PID_ENV } from "../plugins/codex/scripts/lib/broker-ownership.mjs"; import { acquireBrokerLaunchLock, activateBrokerProcess, brokerLaunchLockPort, clearBrokerSession, ensureBrokerSession, loadBrokerSession, loadReusableBrokerSession, saveBrokerSession, sendBrokerShutdown, teardownBrokerSession, waitForBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; import { readStoredJob } from "../plugins/codex/scripts/lib/job-control.mjs"; import { captureProcessOwnership, getProcessIdentity, hasLiveProcessIdentity, terminateProcessGroup, terminateProcessTree } from "../plugins/codex/scripts/lib/process.mjs"; -import { hasCancelFlag, listJobs, loadState, resolveStateDir, saveState, upsertJob, writeCancelFlag, writeJobFile } from "../plugins/codex/scripts/lib/state.mjs"; +import { hasCancelFlag, listJobs, loadState, resolveStateDir, resolveStateFile, saveState, upsertJob, writeCancelFlag, writeJobFile } from "../plugins/codex/scripts/lib/state.mjs"; import { runTrackedJob } from "../plugins/codex/scripts/lib/tracked-jobs.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -590,6 +590,76 @@ test("shared broker durably observes and reclaims a post-activation regrouped he assert.equal(JSON.parse(fs.readFileSync(fakeStatePath, "utf8")).appServerStarts, 2); }); +test("shared broker observes a helper spawned after a streaming response before forwarding notifications", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for helper ownership cleanup."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + const fakeStatePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir, "streaming-helper-after-response"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CODEX_COMPANION_BROKER_CHILD_IDLE_MS: "1000" + }, "streaming-helper-observation"); + const brokerSession = await ensureBrokerSession(repo, { env }); + if (!brokerSession) { + t.skip("broker socket unavailable in this sandbox"); + return; + } + + let helperPid = null; + let appServerPid = null; + let registeredChild = null; + const client = await CodexAppServerClient.connect(repo, { env }); + t.after(async () => { + await client.close().catch(() => {}); + await sendBrokerShutdown(brokerSession.endpoint).catch(() => {}); + for (const pid of [helperPid, appServerPid, brokerSession.pid]) { + if (!Number.isFinite(pid)) { + continue; + } + try { + process.kill(pid, "SIGKILL"); + } catch { + // Ignore processes already reclaimed by the ownership cleanup path. + } + } + clearBrokerSession(repo); + }); + + const started = await client.request("thread/start", { cwd: repo, ephemeral: true }); + await waitFor(() => { + registeredChild = loadBrokerChildren(brokerSession.registry).children[0] ?? null; + appServerPid = registeredChild?.pid ?? null; + return Number.isFinite(appServerPid) && Boolean(registeredChild?.pidIdentity); + }); + + await client.request("turn/start", { + threadId: started.thread.id, + input: [{ type: "text", text: "spawn a helper after returning this response" }] + }); + await waitFor(() => { + if (!fs.existsSync(fakeStatePath)) { + return false; + } + const state = JSON.parse(fs.readFileSync(fakeStatePath, "utf8")); + helperPid = state.helperPids?.[0] ?? null; + return Number.isFinite(helperPid) && Number.isFinite(appServerPid); + }); + await waitFor(() => !hasLiveProcessIdentity(appServerPid, registeredChild.pidIdentity)); + await waitFor(() => { + try { + process.kill(helperPid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }, { timeoutMs: 3000 }); +}); + test("shared broker tears down a post-activation helper before reporting observation lock contention", async (t) => { if (process.platform === "win32") { t.skip("Unix process identities are required for helper ownership cleanup."); @@ -1289,6 +1359,56 @@ test("a registered broker with a missing socket is not reusable", async (t) => { assert.equal(loadReusableBrokerSession(repo, env), null); }); +test("a crashed broker reclaims its registered child before starting a replacement", async (t) => { + if (process.platform === "win32") { + t.skip("Unix process identities are required for the registered broker contract."); + return; + } + + const repo = makeTempDir(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "review-ok"); + const env = withBrokerOwner({ + ...buildEnv(binDir), + CLAUDE_PLUGIN_DATA: runtimePluginDataDir + }, "crashed-broker-child-reclaim"); + const session = await ensureBrokerSession(repo, { env }); + const helper = spawn(process.execPath, ["-e", SELF_EXPIRING_KEEPALIVE], { + cwd: repo, + env, + detached: true, + stdio: "ignore" + }); + helper.unref(); + const helperOwnership = captureProcessOwnership(helper.pid, { cwd: repo, env }); + const childRegistration = publishBrokerChild(session.registry, { ownershipSnapshot: helperOwnership }); + assert.equal(childRegistration.registered, true); + + let replacement = null; + t.after(async () => { + await sendBrokerShutdown(replacement?.endpoint).catch(() => {}); + for (const target of [helper, { pid: replacement?.pid }]) { + if (!Number.isFinite(target?.pid)) { + continue; + } + try { + process.kill(target.pid, "SIGKILL"); + } catch { + // Ignore processes already reclaimed by the recovery path. + } + } + clearBrokerSession(repo); + }); + + process.kill(session.pid, "SIGKILL"); + await waitFor(() => !hasLiveProcessIdentity(session.pid, session.pidIdentity)); + replacement = await ensureBrokerSession(repo, { env }); + + assert.notEqual(replacement?.pid, session.pid); + await waitFor(() => !hasLiveProcessIdentity(helper.pid, helperOwnership.rootIdentity)); + assert.equal(loadBrokerChildren(session.registry).children.length, 0); +}); + test("automatic broker falls back without leaving a process when session ownership is unavailable", async (t) => { if (process.platform === "win32") { t.skip("Unix process identities are required for the registered broker contract."); @@ -3529,6 +3649,21 @@ test("session cleanup preserves a queued cancel between worker read and pid publ enqueueBackgroundTask(workspace, job, request, { spawnDetachedTaskWorkerImpl() {} }); + const queuedState = loadState(workspace); + const queuedJob = queuedState.jobs.find((candidate) => candidate.id === job.id); + const newerJobs = Array.from({ length: 50 }, (_, index) => ({ + id: `newer-job-${index}`, + status: "completed", + phase: "done", + pid: null, + createdAt: new Date(Date.UTC(2026, 6, 29, 9, index, 0)).toISOString(), + updatedAt: new Date(Date.UTC(2026, 6, 29, 9, index, 0)).toISOString() + })); + fs.writeFileSync( + resolveStateFile(workspace), + `${JSON.stringify({ ...queuedState, jobs: [...newerJobs, { ...queuedJob, updatedAt: "2026-07-28T08:03:00.000Z" }] }, null, 2)}\n`, + "utf8" + ); await assert.rejects( handleTaskWorker(["--cwd", workspace, "--job-id", job.id], { From 00015e3ba209bfb6d81ba81f93fd5b9806d5adfa Mon Sep 17 00:00:00 2001 From: Chris Cheng Date: Fri, 31 Jul 2026 10:59:52 -0700 Subject: [PATCH 29/29] [CC-5589] Bound detached test broker lifetime Attributed-To: Codex --- plugins/codex/scripts/app-server-broker.mjs | 31 +++++++++++++ tests/fake-codex-fixture.mjs | 1 + tests/runtime.test.mjs | 48 +++++++++++++++++++-- 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index 64619e92e..94a44084c 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -19,6 +19,7 @@ import { getLiveProcessPids, getProcessIdentity } from "./lib/process.mjs"; const DEFAULT_CHILD_IDLE_MS = 5 * 60 * 1000; const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]); +const TEST_BROKER_TTL_ENV = "CODEX_COMPANION_TEST_BROKER_TTL_MS"; const BROKER_CLEANUP_UNVERIFIED_RPC_CODE = -32002; const BROKER_SHUTDOWN_RPC_CODE = -32003; const BROKER_NOT_ACTIVATED_RPC_CODE = -32004; @@ -97,6 +98,18 @@ function writePidFile(pidFile) { fs.writeFileSync(pidFile, `${process.pid}\n`, "utf8"); } +function resolveTestBrokerTtlMs() { + const raw = process.env[TEST_BROKER_TTL_ENV]; + if (raw == null || raw === "") { + return null; + } + const value = Number(raw); + if (!Number.isInteger(value) || value < 100 || value > 10 * 60 * 1000) { + throw new Error(`${TEST_BROKER_TTL_ENV} must be an integer from 100 to 600000 milliseconds.`); + } + return value; +} + async function main() { const [subcommand, ...argv] = process.argv.slice(2); if (subcommand !== "serve") { @@ -117,6 +130,7 @@ async function main() { const listenTarget = parseBrokerEndpoint(endpoint); const pidFile = options["pid-file"] ? path.resolve(options["pid-file"]) : null; const activationRequired = options["require-activation-stdin"] === true; + const testBrokerTtlMs = resolveTestBrokerTtlMs(); writePidFile(pidFile); const childIdleMs = resolveChildIdleMs(); @@ -141,6 +155,7 @@ async function main() { let streamTurnCounter = 0; let blockedCleanup = null; let brokerRegistration = null; + let testBrokerTtlTimer = null; const sockets = new Set(); function getBrokerRegistration() { @@ -458,6 +473,10 @@ async function main() { server.close(resolve); }); shutdownPromise = (async () => { + if (testBrokerTtlTimer) { + clearTimeout(testBrokerTtlTimer); + testBrokerTtlTimer = null; + } cancelChildIdleClose(); for (const socket of sockets) { socket.destroy(); @@ -722,6 +741,18 @@ async function main() { if (activationRequired) { setupActivationGate(server); } + if (testBrokerTtlMs != null) { + // Test suites intentionally exercise real detached broker behavior. A + // bounded test-only lifetime prevents an interrupted runner from leaving + // its fixture broker behind after normal teardown becomes impossible. + testBrokerTtlTimer = setTimeout(() => { + testBrokerTtlTimer = null; + void shutdown(server) + .catch(() => {}) + .finally(() => process.exit(0)); + }, testBrokerTtlMs); + testBrokerTtlTimer.unref?.(); + } }); } diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index 7978407c6..3dd3752bd 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -726,6 +726,7 @@ export function buildEnv(binDir) { const sep = process.platform === "win32" ? ";" : ":"; return { ...process.env, + CODEX_COMPANION_TEST_BROKER_TTL_MS: "30000", PATH: `${binDir}${sep}${process.env.PATH}` }; } diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 4c42acea1..5928706e2 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -1527,6 +1527,38 @@ test("an unreachable registered broker is never terminated while an owner is liv assert.equal(loadBrokerSession(repo)?.pid, first.pid); }); +test("a detached test broker self-expires when its test runner cannot clean it", async (t) => { + if (process.platform === "win32") { + t.skip("Unix broker sockets are required for this contract."); + return; + } + + const repo = makeTempDir(); + const socketPath = path.join("/private/tmp", `cxc-test-ttl-${process.pid}-${Date.now()}.sock`); + const endpoint = `unix:${socketPath}`; + const broker = spawn(process.execPath, [BROKER_SCRIPT, "serve", "--endpoint", endpoint, "--cwd", repo], { + cwd: repo, + env: { + ...process.env, + CODEX_COMPANION_TEST_BROKER_TTL_MS: "250" + }, + detached: true, + stdio: "ignore" + }); + broker.unref(); + const ownership = captureProcessOwnership(broker.pid, { cwd: repo }); + t.after(async () => { + await terminateProcessTree(broker.pid, { + expectedRootIdentity: ownership.rootIdentity, + ownershipSnapshot: ownership + }).catch(() => {}); + }); + + assert.equal(await waitForBrokerEndpoint(endpoint, 2000), true); + await waitFor(() => !hasLiveProcessIdentity(broker.pid, ownership.rootIdentity), { timeoutMs: 2000 }); + assert.equal(fs.existsSync(socketPath), false); +}); + test("pre-activation broker exits when its launcher pipe closes", async (t) => { if (process.platform === "win32") { t.skip("Unix process groups are required for this contract."); @@ -1541,7 +1573,14 @@ test("pre-activation broker exits when its launcher pipe closes", async (t) => { const child = spawn( process.execPath, [BROKER_SCRIPT, "serve", "--endpoint", endpoint, "--cwd", repo, "--pid-file", pidFile, "--require-activation-stdin"], - { cwd: repo, detached: true, stdio: ["pipe", "ignore", "ignore"] } + { + cwd: repo, + env: { ...process.env, CODEX_COMPANION_TEST_BROKER_TTL_MS: "30000" }, + detached: true, + stdio: ["pipe", "ignore", "ignore"], + timeout: 30000, + killSignal: "SIGKILL" + } ); const ownershipSnapshot = captureProcessOwnership(child.pid, { cwd: repo }); t.after(async () => { @@ -1595,8 +1634,10 @@ test("an unregistered broker refuses to activate a detached app-server child", a const broker = spawn(process.execPath, [BROKER_SCRIPT, "serve", "--endpoint", endpoint, "--cwd", repo], { cwd: repo, env, - detached: true, - stdio: ["ignore", "ignore", "ignore"] + detached: false, + stdio: ["ignore", "ignore", "ignore"], + timeout: 30000, + killSignal: "SIGKILL" }); const brokerOwnership = captureProcessOwnership(broker.pid, { cwd: repo, env }); t.after(async () => { @@ -1605,6 +1646,7 @@ test("an unregistered broker refuses to activate a detached app-server child", a expectedRootIdentity: brokerOwnership.rootIdentity, ownershipSnapshot: brokerOwnership }).catch(() => {}); + await waitFor(() => !hasLiveProcessIdentity(broker.pid, brokerOwnership.rootIdentity)); }); assert.equal(await waitForBrokerEndpoint(endpoint, 2000), true);