🤖 feat: initializeCore() becomes a runtime-run startup effect with per-step timeouts (Wave 4 PR 3)

src/cli/server.ts

@@ -12,6 +12,8 @@ import { initializeXumHomeTransition } from "@/node/compat/xumTransition";
 import { ServerLockfile } from "@/node/services/serverLockfile";
 import { log } from "@/node/services/log";
 import { shutdownStep } from "@/node/services/shutdownStep";
+import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout";
+import { SERVICE_TEARDOWN_BUDGET_MS } from "@/constants/terminationTimeouts";
 import type { BrowserWindow } from "electron";
 import { Command } from "commander";
 import { validateProjectPath } from "@/node/utils/pathUtils";
@@ -49,6 +51,10 @@ process.on("beforeExit", (code) => {
 // Track the launch project path for initial navigation
 let launchProjectPath: string | null = null;
 
+// Set as soon as the container exists so a startup that fails afterwards (main() rejecting) still
+// runs the bounded teardown before the process exits.
+let constructedServices: ServiceContainer | undefined;
+
 // Minimal BrowserWindow stub for services that expect one
 // eslint-disable-next-line local/no-chained-type-assertions -- grandfathered when the rule was introduced; fix the underlying type instead of copying this pattern
 const mockWindow: BrowserWindow = {
@@ -131,6 +137,7 @@ async function main(): Promise<void> {
   const stores = createConfigStores();
   const config = stores.config;
   const serviceContainer = new ServiceContainer(stores);
+  constructedServices = serviceContainer;
   // Headless server has no interactive host-key dialog
   setOpenSSHHostKeyPolicyMode("headless-fallback");
   // Core init (including agent-task recovery, which must finish before any client can act on
@@ -247,11 +254,11 @@ async function main(): Promise<void> {
     const forceExitTimer = setTimeout(() => {
       appendServerCrashLogSync({
         event: "Server cleanup timed out",
-        context: { timeoutMs: 5000 },
+        context: { timeoutMs: SERVICE_TEARDOWN_BUDGET_MS },
       });
       console.log("Cleanup timed out, forcing exit...");
       process.exit(1);
-    }, 5000);
+    }, SERVICE_TEARDOWN_BUDGET_MS);
 
     try {
       // Close all PTY sessions first
@@ -290,12 +297,28 @@ async function main(): Promise<void> {
   process.on("SIGTERM", () => void cleanup());
 }
 
-void main().catch((error) => {
+void main().catch(async (error: unknown) => {
   appendServerCrashLogSync({
     event: "Failed to initialize server",
     detail: error,
   });
   console.error("Failed to initialize server:", error);
+  if (constructedServices) {
+    // Parity with the desktop before-quit race and the ACP root: a startup step that failed — or
+    // timed out and is still running as a plain promise (StartupStepTimeoutError) — must not leave
+    // half-started services behind. Bounded like the SIGTERM cleanup; the process exits either way.
+    const teardown = await raceWithAbortAndTimeout(
+      constructedServices.dispose().catch((disposeError: unknown) => {
+        log.error("[shutdown] dispose after failed startup failed", { error: disposeError });
+      }),
+      { timeoutMs: SERVICE_TEARDOWN_BUDGET_MS }
+    );
+    if (teardown.kind === "timeout") {
+      log.warn("[shutdown] dispose after failed startup timed out; exiting", {
+        timeoutMs: SERVICE_TEARDOWN_BUDGET_MS,
+      });
+    }
+  }
   process.exit(1);
 });
 

src/constants/terminationTimeouts.ts

@@ -30,3 +30,25 @@ export const APP_FIBER_SCOPE_CLOSE_TIMEOUT_MS = 2 * 1000;
  * inside the same 5 s quit budgets.
  */
 export const STARTUP_HOUSEKEEPING_JOIN_TIMEOUT_MS = 500;
+
+/**
+ * Outer budget the `xum server` and ACP roots give the whole
+ * `ServiceContainer.dispose()` — the SIGTERM cleanup and the dispose after a
+ * failed startup; `desktop/main.ts` races its before-quit dispose against the
+ * same 5 s. The bounded steps above are sized to fit inside it.
+ */
+export const SERVICE_TEARDOWN_BUDGET_MS = 5 * 1000;
+
+/**
+ * Bounds each hard startup step of `ServiceContainer.initializeCore()` on the app
+ * runtime's clock. A step that has not settled by then fails startup with a
+ * `StartupStepTimeoutError` through the same exit path as a throwing step
+ * (desktop "Startup Failed" dialog, `xum server`/ACP log-and-exit after the
+ * bounded `dispose()`), instead of pinning the splash screen or the listener
+ * bind forever. Deliberately generous — a false timeout turns a slow-but-fine
+ * start into a crash: sandbox cold starts measured ≤ 60 ms for the slowest core
+ * step (`taskService.recoverInterruptedTasks`, which scales with the number of
+ * active agent tasks, not with deployment size), so this is ≥ 1000× the observed
+ * maximum and still above the policy service's own 10 s fetch timeout.
+ */
+export const STARTUP_STEP_TIMEOUT_MS = 60 * 1000;

src/node/acp/serverConnection.ts

@@ -5,11 +5,13 @@ import { RPCLink as WebSocketRPCLink } from "@orpc/client/websocket";
 import type { RouterClient } from "@orpc/server";
 import WebSocket from "ws";
 import { getXumHome } from "@/common/constants/paths";
+import { SERVICE_TEARDOWN_BUDGET_MS } from "@/constants/terminationTimeouts";
 import { createConfigStores } from "@/node/config";
 import type { AppRouter } from "@/node/orpc/router";
 import { createOrpcServer } from "@/node/orpc/server";
 import { ServiceContainer } from "@/node/services/serviceContainer";
 import { ServerLockfile } from "@/node/services/serverLockfile";
+import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout";
 
 interface ConnectViaWebSocketResult {
   client: ORPCClient;
@@ -154,12 +156,10 @@ async function connectToInProcessServer(requestedAuthToken?: string): Promise<Se
   const stores = createConfigStores();
   const serviceContainer = new ServiceContainer(stores);
 
-  let initialized = false;
   let inProcessServer: InProcessOrpcServer | undefined;
 
   try {
     await serviceContainer.initialize();
-    initialized = true;
 
     const context = serviceContainer.toORPCContext();
     inProcessServer = await createOrpcServer({
@@ -207,9 +207,16 @@ async function connectToInProcessServer(requestedAuthToken?: string): Promise<Se
       await inProcessServer.close().catch(() => undefined);
     }
 
-    if (initialized) {
-      await serviceContainer.dispose().catch(() => undefined);
-    }
+    // Also after a rejected initialize(): a startup step that failed — or timed out and is still
+    // running as a plain promise (StartupStepTimeoutError) — must not leave half-started services
+    // behind, and the stdio adapter must still exit if a teardown step hangs. dispose() is safe
+    // on a container that never finished initializing.
+    await raceWithAbortAndTimeout(
+      serviceContainer.dispose().catch(() => undefined),
+      {
+        timeoutMs: SERVICE_TEARDOWN_BUDGET_MS,
+      }
+    );
 
     throw error;
   }

src/node/services/di/appRuntime.ts

@@ -96,6 +96,32 @@
  *   torn down before step 2 below, and never fork long-lived I/O work through
  *   `EffectRunner` expecting shutdown to await it.
  *
+ * ## Startup (`ServiceContainer.initializeCore()`, Wave 4 PR 3)
+ *
+ * The hard startup steps (`startupCoreSteps`: the initializations request
+ * handling depends on, then agent-task recovery) run as one startup effect on
+ * the runtime — `runtime.managed.runPromise(startupCoreEffect())`, a
+ * root fiber on the built context, not a layer (I1 keeps layer bodies
+ * synchronous, so asynchronous acquisition lives here). Each step is a Promise
+ * thunk in `Effect.tryPromise` with an identity catch, bounded by
+ * `Effect.timeoutOrElse(STARTUP_STEP_TIMEOUT_MS)` on the runtime's `Clock`
+ * (tests inject a `TestClock` beneath the graph and drive the bound
+ * deterministically). Contract: step names/order are the `stepDurationsMs`
+ * keys of the completion log; the first failure or timeout rejects the facade
+ * with the step's own error (identity — v4 `runPromise` rejects with the raw
+ * failure) or a `StartupStepTimeoutError`, later steps do not run, and a
+ * timed-out step keeps running as a plain promise (the timeout interrupts only
+ * the wait; nothing observes the step's result afterwards). Every root
+ * therefore runs the bounded `dispose()` before exiting on a rejected startup
+ * (desktop before-quit race, `cli/server.ts` `main().catch`, ACP
+ * `connectToInProcessServer` catch) so an abandoned step's work is cut off by
+ * the same latches as a quit. Disposing the runtime does not interrupt an
+ * in-flight startup fiber (root fibers are not scope children), matching the
+ * promise chain it replaced. `runStartupHousekeeping()` stays a Promise
+ * pipeline on purpose: its steps are already cancellable through the dispose
+ * abort signal and non-fatal by policy, so a per-step timeout there would be a
+ * policy change, not a lifecycle fix.
+ *
  * ## Shutdown order (`ServiceContainer.dispose()`, one shared teardown behind
  * a latch so concurrent/repeated calls — the desktop's two `before-quit`
  * listeners, tests' dispose-then-shutdown — await the same sequence)
@@ -142,12 +168,13 @@
  *
  * ## Deliberately not done in Phase 11
  *
- * `initialize()` as a Layer/startup effect (would break I1's failure
- * semantics), layer finalizers for the existing `dispose()` steps (I5),
- * `streamBridge` on the runtime, per-service optional tags (optional
- * cross-cutting services stay optional via `CoreOptionsTag`). The streamManager
- * engine core became the `AppFiberScope` occupant in Wave 4 PR 1; the
- * pre-registration stream-start window (`pendingStreamStarts`) stays
+ * Startup as a Layer (would break I1's failure semantics; it became a
+ * runtime-run effect instead, see "Startup"), layer finalizers for the existing
+ * `dispose()` steps (I5), `streamBridge` on the runtime, per-service optional
+ * tags (optional cross-cutting services stay optional via `CoreOptionsTag`),
+ * per-step timeouts for `runStartupHousekeeping()` (policy, see "Startup"). The
+ * streamManager engine core became the `AppFiberScope` occupant in Wave 4 PR 1;
+ * the pre-registration stream-start window (`pendingStreamStarts`) stays
  * unsupervised (nothing durable exists for it yet).
  */
 import assert from "@/common/utils/assert";

src/node/services/serviceContainer.test.ts

@@ -13,6 +13,7 @@ import { AppFiberScopeTag } from "@/node/services/di/appFiberScope";
 import { EffectRunnerTag } from "@/node/services/di/effectRunner";
 import * as appLayers from "@/node/services/di/layers/app";
 import { CoreOptionsTag } from "@/node/services/di/layers/core";
+import { STARTUP_STEP_TIMEOUT_MS } from "@/constants/terminationTimeouts";
 import {
   AgentBrowserSessionDiscovery,
   AgentPluginInstall,
@@ -78,7 +79,7 @@ import {
   WorktreeArchiveSnapshot,
   type AppTags,
 } from "@/node/services/di/tags";
-import { ServiceContainer } from "./serviceContainer";
+import { ServiceContainer, StartupStepTimeoutError } from "./serviceContainer";
 
 /**
  * Independent field → tag listing for every ORPC context field (the production
@@ -345,6 +346,160 @@ describe("ServiceContainer", () => {
     expect(agentStatusStart).toHaveBeenCalledTimes(1);
   });
 
+  const CORE_STEP_NAMES = [
+    "extensionMetadata.initialize",
+    "telemetryService.initialize",
+    "policyService.initialize",
+    "experimentsService.initialize",
+    "taskService.recoverInterruptedTasks",
+  ];
+
+  /** The container's private startup bookkeeping, read for assertions only. */
+  function startupInternals(container: ServiceContainer) {
+    return container as unknown as {
+      extensionMetadata: { initialize: () => Promise<void> };
+      startupStepDurationsMs: Record<string, number>;
+    };
+  }
+
+  /** The rejection reason of `promise` as-is (identity assertions), or a marker if it resolved. */
+  function rejectionOf(promise: Promise<unknown>): Promise<unknown> {
+    return promise.then(
+      () => "<resolved>",
+      (reason: unknown) => reason
+    );
+  }
+
+  it("initializeCore times out a hung step on the runtime clock and skips the later steps", async () => {
+    // TestClock beneath the real graph: the per-step bound must sleep on the runtime's clock
+    // (the effect runs through the ManagedRuntime, not a global Effect.runPromise).
+    const realAppLive = appLayers.AppLive;
+    const appLiveSpy = spyOn(appLayers, "AppLive").mockImplementation((appStores) =>
+      realAppLive(appStores).pipe(Layer.provideMerge(TestClock.layer()))
+    );
+    try {
+      services = new ServiceContainer(stores);
+    } finally {
+      appLiveSpy.mockRestore();
+    }
+    const runtime = services.runtime.managed;
+    spyOn(startupInternals(services).extensionMetadata, "initialize").mockResolvedValue(undefined);
+    spyOn(services.telemetryService, "initialize").mockResolvedValue(undefined);
+    let policyCalled: (() => void) | undefined;
+    const policyCalledPromise = new Promise<void>((resolve) => {
+      policyCalled = resolve;
+    });
+    let rejectAbandonedStep: ((error: unknown) => void) | undefined;
+    spyOn(services.policyService, "initialize").mockImplementation(() => {
+      policyCalled?.();
+      return new Promise<void>((_resolve, reject) => {
+        rejectAbandonedStep = reject;
+      });
+    });
+    const experimentsInitialize = spyOn(services.experimentsService, "initialize");
+    const recoverTasks = spyOn(services.taskService, "recoverInterruptedTasks");
+
+    let outcome: { settled: boolean; error?: unknown } = { settled: false };
+    const core = services.initializeCore().then(
+      () => {
+        outcome = { settled: true };
+      },
+      (error: unknown) => {
+        outcome = { settled: true, error };
+      }
+    );
+    await policyCalledPromise;
+
+    // One millisecond short of the budget the wait is still pending...
+    await runtime.runPromise(TestClock.adjust(Duration.millis(STARTUP_STEP_TIMEOUT_MS - 1)));
+    await new Promise<void>((resolve) => setTimeout(resolve, 0));
+    expect(outcome.settled).toBe(false);
+    // ...and exactly at the budget the step is abandoned.
+    await runtime.runPromise(TestClock.adjust(Duration.millis(1)));
+    await core;
+    expect(outcome.error).toBeInstanceOf(StartupStepTimeoutError);
+    const timeoutError = outcome.error as StartupStepTimeoutError;
+    expect(timeoutError.step).toBe("policyService.initialize");
+    expect(timeoutError.timeoutMs).toBe(STARTUP_STEP_TIMEOUT_MS);
+    // The roots' default Error formatting (dialog / log line) names the class and the step.
+    expect(String(timeoutError)).toMatch(/^StartupStepTimeoutError: policyService\.initialize /);
+    expect(experimentsInitialize).not.toHaveBeenCalled();
+    expect(recoverTasks).not.toHaveBeenCalled();
+    const durations = startupInternals(services).startupStepDurationsMs;
+    expect(Object.keys(durations)).toEqual(CORE_STEP_NAMES.slice(0, 3));
+    const durationsAtTimeout = { ...durations };
+
+    // The abandoned step keeps running as a plain promise: its late rejection is neither
+    // unhandled nor a late side effect on the container.
+    const unhandled: unknown[] = [];
+    const onUnhandledRejection = (reason: unknown) => {
+      unhandled.push(reason);
+    };
+    process.on("unhandledRejection", onUnhandledRejection);
+    try {
+      rejectAbandonedStep?.(new Error("late policy failure"));
+      await new Promise<void>((resolve) => setTimeout(resolve, 0));
+      await new Promise<void>((resolve) => setTimeout(resolve, 0));
+    } finally {
+      process.off("unhandledRejection", onUnhandledRejection);
+    }
+    expect(unhandled).toEqual([]);
+    expect(durations).toEqual(durationsAtTimeout);
+    expect(experimentsInitialize).not.toHaveBeenCalled();
+    expect(recoverTasks).not.toHaveBeenCalled();
+  });
+
+  it("initializeCore rejects with the failing step's own error and skips the later steps", async () => {
+    services = new ServiceContainer(stores);
+    const boom = new Error("policy endpoint unreachable");
+    spyOn(services.policyService, "initialize").mockImplementation(() => Promise.reject(boom));
+    const experimentsInitialize = spyOn(services.experimentsService, "initialize");
+    const recoverTasks = spyOn(services.taskService, "recoverInterruptedTasks");
+
+    // Identity, not a wrapped copy: roots log/print the object they receive.
+    expect(await rejectionOf(services.initializeCore())).toBe(boom);
+    expect(experimentsInitialize).not.toHaveBeenCalled();
+    expect(recoverTasks).not.toHaveBeenCalled();
+  });
+
+  it("initializeCore rejects with a synchronously thrown step error", async () => {
+    services = new ServiceContainer(stores);
+    const boom = new Error("policy store corrupt");
+    spyOn(services.policyService, "initialize").mockImplementation(() => {
+      throw boom;
+    });
+    const recoverTasks = spyOn(services.taskService, "recoverInterruptedTasks");
+
+    expect(await rejectionOf(services.initializeCore())).toBe(boom);
+    expect(recoverTasks).not.toHaveBeenCalled();
+  });
+
+  it("initializeCore records the five core steps and re-runs them when called again", async () => {
+    services = new ServiceContainer(stores);
+    const recoverTasks = spyOn(services.taskService, "recoverInterruptedTasks").mockResolvedValue(
+      undefined
+    );
+
+    await services.initializeCore();
+    expect(Object.keys(startupInternals(services).startupStepDurationsMs)).toEqual(CORE_STEP_NAMES);
+    // Not re-entrancy guarded (parity with the plain promise chain it replaced).
+    await services.initializeCore();
+    expect(recoverTasks).toHaveBeenCalledTimes(2);
+  });
+
+  it("initializeCore after dispose() fails fast without running a step", async () => {
+    services = new ServiceContainer(stores);
+    const recoverTasks = spyOn(services.taskService, "recoverInterruptedTasks");
+    await services.dispose();
+
+    // A disposed ManagedRuntime would otherwise reject with a bare "ManagedRuntime disposed"
+    // defect string from inside the first step.
+    const rejection = await rejectionOf(services.initializeCore());
+    expect(rejection).toBeInstanceOf(Error);
+    expect((rejection as Error).message).toContain("after dispose()");
+    expect(recoverTasks).not.toHaveBeenCalled();
+  });
+
   it("exposes desktopSessionManager in the ORPC context", () => {
     services = new ServiceContainer(stores);