fix(server): self-heal a replaced package tree via drain-and-restart

src/lib/package-tree-integrity.ts

@@ -14,10 +14,37 @@ export type PackageTreeIntegrityStatus =
 
 export interface PackageTreeIntegrityGuard {
   status(): PackageTreeIntegrityStatus;
+  /**
+   * Permanently disarms the guard: cancels any pending restart timer and
+   * invalidates queued callbacks. Called from `server.stop()` so a still-queued
+   * replacement callback cannot schedule a drain-and-restart after shutdown
+   * has already begun.
+   */
+  dispose(): void;
 }
 
-type ObservePackageTree = () => PackageTreeObservation | null;
-type PackageTreeRuntimeInstall = "bun" | "npm" | "pnpm" | "source";
+export interface PackageTreeIntegrityOptions {
+  /**
+   * Called once when a replaced package tree persists past `replacedRestartDelayMs`
+   * of sustained failure. The intended handler is the graceful drain-and-restart
+   * acceptor: an out-of-band install (npm/bun/pnpm global upgrade under a live
+   * proxy) then self-heals instead of serving 503s until someone restarts by hand.
+   * Only `package_tree_replaced` counts — an unreadable manifest resets the timer,
+   * so an install still mid-write does not trigger a restart on partial state.
+   */
+  onReplaced?: () => void;
+  /** Sustained-replacement delay before `onReplaced` fires. 0 fires on first detection. */
+  replacedRestartDelayMs?: number;
+  /**
+   * Test seam; production uses an unref'd timer. May return a cancellation
+   * function; when it does, `resetRestartTimer` cancels the pending callback
+   * instead of leaving it queued behind a generation check.
+   */
+  schedule?: (callback: () => void, delayMs: number) => (() => void) | void;
+}
+
+export type ObservePackageTree = () => PackageTreeObservation | null;
+export type PackageTreeRuntimeInstall = "bun" | "npm" | "pnpm" | "source";
 
 const packageManifestUrl = new URL("../../package.json", import.meta.url);
 
@@ -70,17 +97,123 @@ const PACKAGE_TREE_RECHECK_MS = 1_000;
 export function createPackageTreeIntegrityGuard(
   observe: ObservePackageTree = observePackageManifest,
   now: () => number = Date.now,
+  options: PackageTreeIntegrityOptions = {},
 ): PackageTreeIntegrityGuard {
   const boot = observe();
   let lastOkAt: number | null = null;
+  let notified = false;
+  let timerGeneration = 0;
+  let timerScheduled = false;
+  let cancelScheduled: (() => void) | null = null;
+  let waitingForReadableTree = false;
+  let replacementCandidate: PackageTreeObservation | null = null;
+  const restartDelayMs = options.replacedRestartDelayMs ?? 5_000;
+  const schedule = options.schedule ?? ((callback, delayMs) => {
+    const timer = setTimeout(callback, delayMs);
+    timer.unref?.();
+    return () => clearTimeout(timer);
+  });
+
+  const resetRestartTimer = (): void => {
+    timerGeneration += 1;
+    timerScheduled = false;
+    const cancel = cancelScheduled;
+    cancelScheduled = null;
+    cancel?.();
+  };
+
+  const armRestartTimer = (delayMs = restartDelayMs): void => {
+    if (!options.onReplaced || notified || timerScheduled) return;
+    timerScheduled = true;
+    const generation = timerGeneration;
+    const verifyAndNotify = () => {
+      if (generation !== timerGeneration || notified) return;
+      timerScheduled = false;
+      const current = observe();
+      if (boot === null || current === null) {
+        // A package manager may replace package.json before the rest of the tree.
+        // Wait for a readable tree, then require a fresh full debounce interval.
+        resetRestartTimer();
+        waitingForReadableTree = true;
+        armRestartTimer(PACKAGE_TREE_RECHECK_MS);
+        return;
+      }
+      if (sameObservation(boot, current)) {
+        resetRestartTimer();
+        waitingForReadableTree = false;
+        replacementCandidate = null;
+        return;
+      }
+      if (waitingForReadableTree) {
+        waitingForReadableTree = false;
+        replacementCandidate = current;
+        resetRestartTimer();
+        armRestartTimer();
+        return;
+      }
+      if (replacementCandidate === null || !sameObservation(replacementCandidate, current)) {
+        replacementCandidate = current;
+        resetRestartTimer();
+        armRestartTimer();
+        return;
+      }
+      try {
+        options.onReplaced?.();
+        notified = true;
+      } catch {
+        // A failed restart admission must not leave the proxy fenced forever.
+        // Re-observe after the normal debounce and try again if replacement persists.
+        armRestartTimer(Math.max(PACKAGE_TREE_RECHECK_MS, restartDelayMs));
+      }
+    };
+    if (delayMs === 0) {
+      // Defer like the scheduled path: verifyAndNotify can arm the next timer, and a
+      // synchronous verify inside this frame would re-enter armRestartTimer while this
+      // arm is still running.
+      queueMicrotask(verifyAndNotify);
+    } else {
+      // The seam may run the callback synchronously; defer the work so
+      // cancelScheduled ownership is settled before verifyAndNotify can
+      // re-enter armRestartTimer.
+      const cancel = schedule(() => {
+        queueMicrotask(verifyAndNotify);
+      }, delayMs);
+      if (generation === timerGeneration && timerScheduled && typeof cancel === "function") {
+        cancelScheduled = cancel;
+      }
+    }
+  };
+
   return {
+    dispose(): void {
+      resetRestartTimer();
+      notified = true;
+    },
     status(): PackageTreeIntegrityStatus {
       const at = now();
       if (lastOkAt !== null && at - lastOkAt < PACKAGE_TREE_RECHECK_MS) return { ok: true };
       const current = observe();
-      if (boot === null || current === null) return { ok: false, reason: "package_tree_unreadable" };
-      if (!sameObservation(boot, current)) return { ok: false, reason: "package_tree_replaced" };
+      if (boot === null || current === null) {
+        const wasWatchingReplacement = timerScheduled;
+        resetRestartTimer();
+        if (wasWatchingReplacement && boot !== null) {
+          waitingForReadableTree = true;
+          armRestartTimer(PACKAGE_TREE_RECHECK_MS);
+        }
+        return { ok: false, reason: "package_tree_unreadable" };
+      }
+      if (!sameObservation(boot, current)) {
+        if (replacementCandidate === null || !sameObservation(replacementCandidate, current)) {
+          replacementCandidate = current;
+          resetRestartTimer();
+        }
+        armRestartTimer();
+        return { ok: false, reason: "package_tree_replaced" };
+      }
       lastOkAt = at;
+      resetRestartTimer();
+      waitingForReadableTree = false;
+      replacementCandidate = null;
       return { ok: true };
     },
   };
@@ -96,7 +229,10 @@ export function createRuntimePackageTreeIntegrityGuard(
   installer: PackageTreeRuntimeInstall,
   observe: ObservePackageTree = observePackageManifest,
   now: () => number = Date.now,
-): PackageTreeIntegrityGuard {
-  if (installer === "source" || isStandaloneBinary()) return { status: () => ({ ok: true }) };
-  return createPackageTreeIntegrityGuard(observe, now);
-}
+  options: PackageTreeIntegrityOptions = {},
+  ): PackageTreeIntegrityGuard {
+    if (installer === "source" || isStandaloneBinary()) {
+      return { status: () => ({ ok: true }), dispose: () => {} };
+    }
+    return createPackageTreeIntegrityGuard(observe, now, options);
+  }

src/server/index.ts

@@ -193,13 +193,9 @@ import {
   createLocalAttestationSecret,
 } from "../lib/local-management-attestation";
 import { createReadinessGate, type ReadinessGate } from "./readiness";
-import {
-  createRuntimePackageTreeIntegrityGuard,
-  type PackageTreeIntegrityGuard,
-} from "../lib/package-tree-integrity";
-import { detectInstall } from "../update/index";
 import { createServeOptions, type ServerIngress } from "./index/serve-options";
 import { createClaudeInterceptLifecycle } from "./index/claude-intercept-lifecycle";
+import { createPackageTreeIntegrityGuardForServer } from "./index/package-tree-guard";
 import { inspectStartupOwnership, resolveInboundBodyLimitWithWarning, setStartupCacheInvalidationWrite, warnAgentTaskRecoveryStartup, warnPlaintextV2AgentMessagesStartup, type StartServerDeps } from "./index/startup-warnings";
 import { acquireSpendLedgerServerLifecycle, recordFailedStartRollback, type SpendLedgerServerLifecycle } from "./index/spend-ledger-lifecycle";
 export { waitForFailedStartRollback } from "./index/spend-ledger-lifecycle";
@@ -536,8 +532,7 @@ function startServerWithSpendLedgerOwner(port: number | undefined, deps: StartSe
   // passes it in, and transitions it after the post-startup sync settles. When
   // no gate is supplied (tests, ad-hoc starts) a fresh pending gate is created.
   const readinessGate = deps.readinessGate ?? createReadinessGate();
-  const packageTreeIntegrity = deps.packageTreeIntegrity
-    ?? createRuntimePackageTreeIntegrityGuard(detectInstall());
+  const packageTreeIntegrity = createPackageTreeIntegrityGuardForServer(deps);
   // Actual bound port, filled in after Bun.serve binds so /readyz reports the
   // real ephemeral port for startServer(0). /healthz keeps its existing port
   // field (the requested listenPort) byte-for-byte.
@@ -755,6 +750,10 @@ function startServerWithSpendLedgerOwner(port: number | undefined, deps: StartSe
     value: async (closeActiveConnections?: boolean): Promise<void> => {
       remoteWorkspaceStopping = true;
       liveCallBindings.clear();
+      // Disarm the package-tree restart timer before listener teardown: a queued
+      // replacement callback must not call acceptSystemRestart() after stop() has
+      // begun, or it would schedule a drain-and-restart on a stopped server.
+      packageTreeIntegrity.dispose();
       // The orchestration lives in `runListenerShutdown` so its two competing properties —
       // cleanup completes, failure propagates — are testable without a live socket.
       await runListenerShutdown(

src/server/index/package-tree-guard.ts

@@ -0,0 +1,34 @@
+import {
+  createRuntimePackageTreeIntegrityGuard,
+  type PackageTreeIntegrityGuard,
+} from "../../lib/package-tree-integrity";
+import { acceptSystemRestart } from "../management/system-restart";
+import { detectInstall } from "../../update/index";
+import type { StartServerDeps } from "./startup-warnings";
+
+// Production guard wiring for the package-tree integrity fence. Tests inject
+// deps.packageTreeIntegrity and never reach this path; everything else is the
+// same default the inline construction used to build.
+export function createPackageTreeIntegrityGuardForServer(
+  deps: StartServerDeps,
+): PackageTreeIntegrityGuard {
+  if (deps.packageTreeIntegrity) {
+    return deps.packageTreeIntegrity;
+  }
+  const acceptPackageTreeRestart = deps.acceptSystemRestart ?? acceptSystemRestart;
+  return createRuntimePackageTreeIntegrityGuard(
+    deps.packageTreeInstaller ?? detectInstall(),
+    deps.observePackageTree,
+    undefined,
+    {
+      ...deps.packageTreeIntegrityOptions,
+      onReplaced: () => {
+        // An out-of-band install replaced the package under this live process. Serve
+        // the 503 for the triggering request, then let the standard drain-and-restart
+        // path bring the new tree up instead of refusing traffic until a manual
+        // restart. acceptSystemRestart is idempotent and supervisor-aware.
+        acceptPackageTreeRestart();
+      },
+    },
+  );
+}

src/server/index/startup-warnings.ts

@@ -6,6 +6,11 @@ import {
   type OwnershipInspection,
 } from "../../integrations/native/ownership-preflight";
 import { registerCodexQuotaAutoRefreshWorker } from "../../codex/quota-auto-refresh";
+import type {
+  ObservePackageTree,
+  PackageTreeIntegrityOptions,
+  PackageTreeRuntimeInstall,
+} from "../../lib/package-tree-integrity";
 import {
   consumeForInspection,
   relaySseWithHeartbeat,
@@ -141,6 +146,14 @@ export interface StartServerDeps {
   readinessGate?: ReadinessGate;
   /** Test-only package-tree observation; production captures package.json identity at boot. */
   packageTreeIntegrity?: PackageTreeIntegrityGuard;
+  /** Test-only default-guard options; production observes the installed package manifest. */
+  packageTreeIntegrityOptions?: PackageTreeIntegrityOptions;
+  /** Test-only installed-package identity; production detects the current install. */
+  packageTreeInstaller?: PackageTreeRuntimeInstall;
+  /** Test-only manifest observer; production stats the installed package.json. */
+  observePackageTree?: ObservePackageTree;
+  /** Test-only restart acceptor; production uses the normal drain-and-restart path. */
+  acceptSystemRestart?: typeof import("../management/system-restart").acceptSystemRestart;
   /** Test-only seam for observing quota-worker registration ownership. */
   registerCodexQuotaAutoRefreshWorker?: typeof registerCodexQuotaAutoRefreshWorker;
 }

structure/ops/docs-and-release.md

@@ -273,6 +273,17 @@ Invariants:
 - Public docs (root READMEs + `docs-site` installation pages, all locales) state Node 18+ as the only
   prerequisite. Do not reintroduce "install Bun first" / "bun must be on PATH" guidance for npm users.
 
+### Package-tree integrity fence
+
+Installed npm, bun, and pnpm packages bind each server process to the package manifest identity
+observed at startup. Replacing that manifest under a live process fences `/healthz`, `/readyz`, and
+`/v1/*` with `package_tree_changed`. The first observed replacement starts an unref'd five-second
+stability timer; if the same new manifest identity remains readable and distinct, the timer enters the existing
+drain-and-restart handoff without waiting for another request. A temporarily unreadable manifest
+is polled until readable and then receives a fresh full stability interval, while a return to the
+startup identity cancels the pending restart. Failed restart admission retries after the same
+bounded delay. Source checkouts and standalone binaries remain outside this integrity fence.
+
 ## Release workflow
 
 Package release is npm-focused. `package.json` exposes `opencodex` and `ocx`, `prepublishOnly` runs

structure/runtime.md

@@ -188,6 +188,8 @@ described in [OpenAI quota ownership](providers/openai-tiers.md#public-provider-
 until shutdown. Normal shutdown restores native Codex. Service mode sets
 `OCX_SERVICE=1`, so managed restarts do not repeatedly restore/reinject; explicit service stop and
 uninstall still restore.
+The package-tree integrity fence for live package replacement follows the
+[update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence).
 
 A busy preferred port is never resolved by starting somewhere else. Both questions a start asks
 about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts`