diff --git a/src/common/orpc/schemas.ts b/src/common/orpc/schemas.ts index 17e054d6d6..6640e17f9a 100644 --- a/src/common/orpc/schemas.ts +++ b/src/common/orpc/schemas.ts @@ -253,6 +253,7 @@ export { ErrorEventSchema, GoalBudgetLimitedEventSchema, LanguageModelV2UsageSchema, + ModelFallbackProgressSchema, OnChatDowngradeReasonSchema, QueuedMessageChangedEventSchema, ReasoningDeltaEventSchema, diff --git a/src/common/orpc/schemas/message.ts b/src/common/orpc/schemas/message.ts index 56b9da49c4..657e8f5856 100644 --- a/src/common/orpc/schemas/message.ts +++ b/src/common/orpc/schemas/message.ts @@ -190,6 +190,10 @@ export const MuxMessageSchema = z.object({ retrySendOptions: z.any().optional(), agentId: AgentIdSchema.optional().catch(undefined), partial: z.boolean().optional(), + // Steps the cut turn had left under its ceiling when a queued message interrupted it + // (stamped on the committed partial): a startup retry of that turn runs under this budget + // instead of a fresh ceiling. Self-healing read path: a malformed value reads as absent. + stepsRemaining: z.number().int().nonnegative().optional().catch(undefined), synthetic: z.boolean().optional(), uiVisible: z.boolean().optional(), // RLM keep-recent floor: sanitized post-boundary copy of a pre-compaction row. diff --git a/src/common/orpc/schemas/stream.ts b/src/common/orpc/schemas/stream.ts index 6108159fcb..0b24ddfe5f 100644 --- a/src/common/orpc/schemas/stream.ts +++ b/src/common/orpc/schemas/stream.ts @@ -292,7 +292,10 @@ export const StreamEndEventSchema = z.object({ }), }); -export const StreamAbortReasonSchema = z.enum(["user", "startup", "system"]); +// "queued-message": the backend's own soft stop at a provider-executed tool boundary so a queued +// tool-end message can dispatch; distinct from "system" so a concurrent hard stop cannot be +// mistaken for it. +export const StreamAbortReasonSchema = z.enum(["user", "startup", "system", "queued-message"]); export const StreamLifecyclePhaseSchema = z.enum([ "idle", @@ -320,6 +323,12 @@ export const StreamLifecycleEventSchema = StreamLifecycleSnapshotSchema.extend({ workspaceId: z.string(), }); +// Refusal-fallback chain a turn runs under and how far along it is. A stream that resumes a cut +// turn continues this chain instead of resolving one from the model it resumes on. +export const ModelFallbackProgressSchema = ModelFallbackRecordSchema.extend({ + chain: z.array(z.string()), +}); + export const StreamAbortEventSchema = z.object({ type: z.literal("stream-abort"), workspaceId: z.string(), @@ -336,6 +345,16 @@ export const StreamAbortEventSchema = z.object({ // Last step's provider metadata (for context window cache display) contextProviderMetadata: z.record(z.string(), z.unknown()).optional(), duration: z.number().optional(), + // Model active at the abort (a configured fallback may differ from the requested model) + model: z.string().optional(), + // Steps left under the stream's ceiling at the abort; a turn cut for a queued message + // resumes under this budget rather than a fresh one. + stepsRemaining: z.number().int().nonnegative().optional(), + // A required completion tool succeeded in the interrupted step: the turn was complete, so a + // queued-message soft stop owes it no continuation. + requiredToolSatisfied: z.boolean().optional(), + // Fallback chain state at the abort, carried into the resumed stream for the same reason. + modelFallbackProgress: ModelFallbackProgressSchema.optional(), }) .optional() .meta({ diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 128d63f0b0..2d45680687 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -9,6 +9,7 @@ import type { } from "@/common/constants/contextBoundary"; import type { GoalSyntheticMessageKind } from "@/constants/goals"; import type { SendMessageOptions } from "@/common/orpc/types"; +import type { ModelFallbackProgress } from "./stream"; import { withLegacyPtcExclusiveMirror } from "@/common/constants/experiments"; import type { z } from "zod"; import type { AgentMode } from "./mode"; @@ -220,6 +221,14 @@ export interface CompactionFollowUpRequest extends CompactionFollowUpInput, Pres goalId?: string; /** Internal dispatch guardrails for crash-safe follow-up recovery. */ dispatchOptions?: CompactionFollowUpDispatchOptions; + /** + * What the turn interrupted for mid-stream compaction had left of its step ceiling, the + * fallback chain state it reached, and whether it ran under admission revalidation: the + * follow-up continues that turn, not a fresh one. + */ + stepBudget?: number; + modelFallbackProgress?: ModelFallbackProgress; + revalidateAdmission?: boolean; /** * Open delegated workspace-turn correlation captured before on-send * compaction consumed this follow-up (e.g. a bash-monitor wake continuing a @@ -928,6 +937,8 @@ export interface MuxMetadata { contextProviderMetadata?: Record; systemMessageTokens?: number; // Token count for system message sent with this request (calculated by AIService) partial?: boolean; // Whether this message was interrupted and is incomplete + /** Steps a queued-message cut left under the turn's ceiling; a startup retry runs under it. */ + stepsRemaining?: number; synthetic?: boolean; // Whether this message was synthetically generated (e.g., [CONTINUE] sentinel) /** * For queue-dispatched user turns: when the user last added to the queued diff --git a/src/common/types/stream.ts b/src/common/types/stream.ts index c1979b0bd4..3b16f114f8 100644 --- a/src/common/types/stream.ts +++ b/src/common/types/stream.ts @@ -11,6 +11,7 @@ import type { AutoRetryScheduledEventSchema, AutoRetryStartingEventSchema, ErrorEventSchema, + ModelFallbackProgressSchema, ReasoningDeltaEventSchema, ReasoningEndEventSchema, StreamAbortReasonSchema, @@ -45,6 +46,7 @@ export type StreamStartEvent = z.infer; export type StreamDeltaEvent = z.infer; export type StreamEndEvent = z.infer; export type StreamAbortReason = z.infer; +export type ModelFallbackProgress = z.infer; export type StreamLifecyclePhase = z.infer; export type StreamLifecycleSnapshot = z.infer; export type StreamLifecycleEvent = z.infer; diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 0161cb9d1e..243a4baf64 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -1160,8 +1160,14 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { session.dispose(); }); - test("hides default follow-up sentinel in mid-stream auto-compaction prompts", async () => { + test("mid-stream auto-compaction hides the default follow-up sentinel and hands over the interrupted turn's remainder", async () => { const workspaceId = "ws-auto-compaction-mid-stream-sentinel"; + // The interrupted stream had already moved down its fallback chain and spent steps. + const interruptedProgress = { + requestedModel: "openai:gpt-4o", + refusedModels: ["openai:gpt-4o"], + chain: ["openai:gpt-4o-fallback"], + }; const { historyService, cleanup } = await createTestHistoryService(); historyCleanup = cleanup; @@ -1211,6 +1217,11 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { workspaceId, messageId: "assistant-mid-stream", abortReason: "system", + metadata: { + model: "openai:gpt-4o-fallback", + stepsRemaining: 7, + modelFallbackProgress: interruptedProgress, + }, }); return Promise.resolve(Ok(undefined)); @@ -1275,14 +1286,18 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { ownerWorkspaceId: "parent-mid-stream-compaction", turnId: "turn-mid-stream-compaction", } as const; - const result = await session.sendMessage( - "hello", + // The interrupted turn is a revalidated resume (a stranded delegated turn's continuation). + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-hello", "user", "hello", { timestamp: Date.now() }) + ); + const result = await session.resumeStream( { model: "openai:gpt-4o", agentId: "exec", muxMetadata: workspaceTurnMetadata, }, - { agentInitiated: true } + { agentInitiated: true, revalidateAdmission: true } ); expect(result.success).toBe(true); @@ -1309,6 +1324,14 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { workspaceTurnMetadata ); expect(compactionRequestMetadata.parsed.followUpContent?.agentInitiated).toBe(true); + // The follow-up continues the interrupted turn: on the model it reached, under what it had + // left of the ceiling, with the refusals so far. + expect(compactionRequestMetadata.parsed.followUpContent).toMatchObject({ + model: "openai:gpt-4o-fallback", + stepBudget: 7, + modelFallbackProgress: interruptedProgress, + revalidateAdmission: true, + }); const compactionRequestText = compactionRequestMessage?.parts.find((part) => part.type === "text")?.text ?? ""; diff --git a/src/node/services/agentSession.continueMessageAgentId.test.ts b/src/node/services/agentSession.continueMessageAgentId.test.ts index a2786e8920..612d8bef8a 100644 --- a/src/node/services/agentSession.continueMessageAgentId.test.ts +++ b/src/node/services/agentSession.continueMessageAgentId.test.ts @@ -3,7 +3,9 @@ import { createMuxMessage } from "@/common/types/message"; import type { CompactionFollowUpRequest, MuxMessage } from "@/common/types/message"; import type { FilePart, SendMessageOptions } from "@/common/orpc/types"; import type { Config } from "@/node/config"; +import { GOAL_CONTINUATION_KIND } from "@/constants/goals"; import { AgentSession } from "./agentSession"; +import type { WorkspaceGoalService } from "./workspaceGoalService"; import { createStreamLifecycleMocks } from "./agentSession.testHarness"; import type { AIService } from "./aiService"; import type { BackgroundProcessManager } from "./backgroundProcessManager"; @@ -22,14 +24,29 @@ type SendMessageResult = interface AutoRetryResumeRequest { options: SendMessageOptions; agentInitiated?: boolean; + stepBudget?: number; + modelFallbackProgress?: unknown; + revalidateAdmission?: boolean; + workspaceTurnMetadata?: unknown; +} + +interface SendInternal { + synthetic?: boolean; + agentInitiated?: boolean; + stepBudget?: number; + modelFallbackProgress?: unknown; + revalidateAdmission?: boolean; + workspaceTurnMetadata?: unknown; + refuseStreamStart?: () => boolean; } interface SessionInternals { dispatchPendingFollowUp: () => Promise; + retryActiveStream: () => Promise; sendMessage: ( message: string, options?: SendOptions, - internal?: { synthetic?: boolean; agentInitiated?: boolean } + internal?: SendInternal ) => Promise; scheduleStartupRecovery: () => void; startupRecoveryPromise: Promise | null; @@ -150,7 +167,18 @@ describe("AgentSession continue-message agentId fallback", () => { historyCleanup = undefined; }); - const createSession = async (messages: MuxMessage[] = [], config = createConfig()) => { + const createSession = async ( + messages: MuxMessage[] = [], + { + config = createConfig(), + ...turnOptions + }: Pick< + ConstructorParameters[0], + | "admitStrandedTurnResume" + | "settleForfeitedWorkspaceTurnContinuation" + | "workspaceGoalService" + > & { config?: Config } = {} + ) => { const { historyService, cleanup } = await createTestHistoryService(); historyCleanup = cleanup; for (const message of messages) { @@ -164,6 +192,7 @@ describe("AgentSession continue-message agentId fallback", () => { aiService: createAiService(), initStateManager: createInitStateManager(), backgroundProcessManager: createBackgroundProcessManager(), + ...turnOptions, }); sessions.push(session); @@ -266,6 +295,440 @@ describe("AgentSession continue-message agentId fallback", () => { expect(internals.lastAutoRetryResumeRequest?.agentInitiated).toBe(true); }); + test("dispatchPendingFollowUp continues the interrupted turn's step budget, fallback chain, and admission revalidation", async () => { + const progress = { + requestedModel: "anthropic:claude-sonnet-4-5", + refusedModels: ["anthropic:claude-sonnet-4-5"], + chain: ["openai:gpt-4o", "google:gemini-fallback"], + }; + const dispatched: SendInternal[] = []; + const { internals } = await createSession([ + compactionSummaryMessage("summary-remainder", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + stepBudget: 7, + modelFallbackProgress: progress, + revalidateAdmission: true, + }), + ]); + internals.sendMessage = mock( + (_message: string, _options?: SendOptions, internal?: SendInternal) => { + dispatched.push(internal ?? {}); + return Promise.resolve({ success: true as const }); + } + ); + + await internals.dispatchPendingFollowUp(); + + expect(dispatched[0]).toMatchObject({ + stepBudget: 7, + modelFallbackProgress: progress, + revalidateAdmission: true, + }); + expect(internals.lastAutoRetryResumeRequest).toMatchObject({ + stepBudget: 7, + modelFallbackProgress: progress, + revalidateAdmission: true, + }); + }); + + const DELEGATED_TURN = { + type: "workspace-turn-task", + taskHandleId: "wst_follow_up", + ownerWorkspaceId: "owner-ws", + turnId: "turn-follow-up", + } as const; + + test("dispatchPendingFollowUp discards a follow-up whose interrupted turn spent its step budget", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-spent", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + stepBudget: 0, + muxMetadata: DELEGATED_TURN, + }), + ], + { settleForfeitedWorkspaceTurnContinuation: settle } + ); + const sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = sendMessage; + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + // The ceiling ended the turn; the follow-up is dropped rather than left to redispatch later, + // and the delegated turn it continued is settled since no successor stream will end it. + expect(sendMessage).not.toHaveBeenCalled(); + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + const tail = await historyService.getLastMessages("ws", 1); + expect(tail.success).toBe(true); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.id).toBe("summary-spent"); + expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + }); + + test("dispatchPendingFollowUp admits a delegated turn's follow-up like a stranded resume", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + let stale = false; + const admit = mock((_correlation: unknown) => + Promise.resolve({ admissible: true, admissionStale: () => stale }) + ); + const dispatched: SendInternal[] = []; + const { internals } = await createSession( + [ + compactionSummaryMessage("summary-delegated", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }), + ], + { admitStrandedTurnResume: admit, settleForfeitedWorkspaceTurnContinuation: settle } + ); + internals.sendMessage = mock( + (_message: string, _options?: SendOptions, internal?: SendInternal) => { + dispatched.push(internal ?? {}); + return Promise.resolve({ success: true as const }); + } + ); + + expect(await internals.dispatchPendingFollowUp()).toBe(true); + + // Admitted against the delegated turn, with the handle probe carried to the launch boundary, + // and retried under revalidation even though the interrupted turn was not a stranded resume. + expect(admit.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + expect(dispatched[0]?.refuseStreamStart?.()).toBe(false); + stale = true; + expect(dispatched[0]?.refuseStreamStart?.()).toBe(true); + expect(dispatched[0]?.revalidateAdmission).toBe(true); + expect(internals.lastAutoRetryResumeRequest?.revalidateAdmission).toBe(true); + expect(settle).not.toHaveBeenCalled(); + }); + + test("dispatchPendingFollowUp settles a delegated turn's follow-up refused at the launch boundary", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + let stale = false; + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-launch-refused", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }), + ], + { + admitStrandedTurnResume: mock(() => + Promise.resolve({ admissible: true, admissionStale: () => stale }) + ), + settleForfeitedWorkspaceTurnContinuation: settle, + } + ); + // The handle is interrupted while the send prepares: StreamManager refuses the launch and the + // send still resolves Ok (a startup-aborted handle), so the dispatch must read the probe. + internals.sendMessage = mock(() => { + stale = true; + return Promise.resolve({ success: true as const }); + }); + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + }); + + test("dispatchPendingFollowUp settles a delegated turn's goal follow-up its goal refuses at the launch boundary", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + let goalStale = false; + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-goal-launch-refused", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + goalKind: GOAL_CONTINUATION_KIND, + goalId: "2f5a4c9e-3b7d-4e1f-9a6b-8c0d1e2f3a4b", + muxMetadata: DELEGATED_TURN, + }), + ], + { + workspaceGoalService: { + buildGoalRedispatchAdmission: mock(() => + Promise.resolve({ admissible: true, admissionStale: () => goalStale }) + ), + } as unknown as WorkspaceGoalService, + admitStrandedTurnResume: mock(() => + Promise.resolve({ admissible: true, admissionStale: () => false }) + ), + settleForfeitedWorkspaceTurnContinuation: settle, + } + ); + // The goal is paused while the send prepares: only the goal probe trips, and the send still + // resolves Ok (a startup-aborted handle). + internals.sendMessage = mock(() => { + goalStale = true; + return Promise.resolve({ success: true as const }); + }); + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + }); + + test("dispatchPendingFollowUp clears a dropped follow-up only once its delegated turn is settled", async () => { + let settled!: () => void; + const settle = mock( + (_correlation: unknown, _reason: string) => + new Promise((resolve) => { + settled = resolve; + }) + ); + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-settle-first", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }), + ], + { + admitStrandedTurnResume: mock(() => Promise.resolve({ admissible: false })), + settleForfeitedWorkspaceTurnContinuation: settle, + } + ); + const pendingFollowUp = async () => { + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + const muxMeta = summary?.metadata?.muxMetadata; + return muxMeta?.type === "compaction-summary" ? muxMeta.pendingFollowUp : undefined; + }; + + const dispatch = internals.dispatchPendingFollowUp(); + await new Promise((resolve) => setTimeout(resolve, 20)); + // Settlement outstanding: the follow-up stays durable so a crash here can still rediscover + // the correlation (the owed settlement itself lives only in memory). + expect(settle).toHaveBeenCalledTimes(1); + expect(await pendingFollowUp()).toBeDefined(); + + settled(); + expect(await dispatch).toBe(false); + expect(await pendingFollowUp()).toBeUndefined(); + }); + + test("dispatchPendingFollowUp keeps a dropped follow-up pending when its delegated turn fails to settle", async () => { + const settle = mock((_correlation: unknown, _reason: string) => + Promise.reject(new Error("task store unavailable")) + ); + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-settle-failed", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }), + ], + { + admitStrandedTurnResume: mock(() => Promise.resolve({ admissible: false })), + settleForfeitedWorkspaceTurnContinuation: settle, + } + ); + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + // Left for the next startup to re-drop and re-settle. + expect(settle).toHaveBeenCalledTimes(1); + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.metadata?.muxMetadata).toMatchObject({ + type: "compaction-summary", + pendingFollowUp: { text: "Continue" }, + }); + }); + + test("dispatchPendingFollowUp revalidates an on-send-compacted wake's retries against its delegated turn", async () => { + let admissible = true; + const admit = mock((_correlation: unknown) => + Promise.resolve( + admissible ? { admissible: true, admissionStale: () => false } : { admissible: false } + ) + ); + const dispatched: SendInternal[] = []; + const { internals } = await createSession( + [ + compactionSummaryMessage("summary-wake", { + text: "Background monitor wake", + model: "openai:gpt-4o", + agentId: "exec", + // The wake's own metadata carries no correlation; on-send compaction stamped the + // delegated turn it continued beside it. + muxMetadata: { type: "bash-monitor-wake", records: [] }, + workspaceTurnMetadata: DELEGATED_TURN, + }), + ], + { admitStrandedTurnResume: admit } + ); + internals.sendMessage = mock( + (_message: string, _options?: SendOptions, internal?: SendInternal) => { + dispatched.push(internal ?? {}); + return Promise.resolve({ success: true as const }); + } + ); + + expect(await internals.dispatchPendingFollowUp()).toBe(true); + expect(admit.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + expect(dispatched[0]?.workspaceTurnMetadata).toEqual(DELEGATED_TURN); + + // The handle is interrupted during the backoff: the retry's admission sees the delegated + // turn, not just the wake, and refuses. + admissible = false; + await internals.retryActiveStream(); + expect(admit).toHaveBeenCalledTimes(2); + expect(admit.mock.calls[1]?.[0]).toEqual(DELEGATED_TURN); + }); + + test("dispatchPendingFollowUp ignores a malformed persisted correlation", async () => { + const admit = mock(() => Promise.resolve({ admissible: false })); + const sendMessage = mock(() => Promise.resolve({ success: true as const })); + const { internals } = await createSession( + [ + compactionSummaryMessage("summary-malformed-correlation", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: { + type: "workspace-turn-task", + } as unknown as CompactionFollowUpRequest["muxMetadata"], + }), + ], + { admitStrandedTurnResume: admit } + ); + internals.sendMessage = sendMessage; + + // Not a delegated turn to admit or settle: the follow-up dispatches as an ordinary one. + expect(await internals.dispatchPendingFollowUp()).toBe(true); + expect(admit).not.toHaveBeenCalled(); + expect(sendMessage).toHaveBeenCalledTimes(1); + }); + + test("dispatchPendingFollowUp settles a delegated turn's follow-up rejected for malformed goal attribution", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + const sendMessage = mock(() => Promise.resolve({ success: true as const })); + const { internals } = await createSession( + [ + compactionSummaryMessage("summary-malformed-goal", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + goalKind: "not-a-goal-kind" as unknown as CompactionFollowUpRequest["goalKind"], + muxMetadata: DELEGATED_TURN, + }), + ], + { settleForfeitedWorkspaceTurnContinuation: settle } + ); + internals.sendMessage = sendMessage; + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + expect(sendMessage).not.toHaveBeenCalled(); + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + }); + + test("dispatchPendingFollowUp settles and drops a delegated turn's follow-up its owner no longer admits", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-refused", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + muxMetadata: DELEGATED_TURN, + }), + ], + { + admitStrandedTurnResume: mock(() => Promise.resolve({ admissible: false })), + settleForfeitedWorkspaceTurnContinuation: settle, + } + ); + const sendMessage = mock(() => Promise.resolve({ success: true as const })); + internals.sendMessage = sendMessage; + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + expect(sendMessage).not.toHaveBeenCalled(); + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + }); + + test("dispatchPendingFollowUp drops a malformed persisted chain state", async () => { + const dispatched: SendInternal[] = []; + const { internals } = await createSession([ + compactionSummaryMessage("summary-malformed-chain", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + modelFallbackProgress: { + requestedModel: 1, + } as unknown as CompactionFollowUpRequest["modelFallbackProgress"], + }), + ]); + internals.sendMessage = mock( + (_message: string, _options?: SendOptions, internal?: SendInternal) => { + dispatched.push(internal ?? {}); + return Promise.resolve({ success: true as const }); + } + ); + + await internals.dispatchPendingFollowUp(); + + // The chain state is only a preference order: the follow-up runs on its model's own chain. + expect(dispatched).toHaveLength(1); + expect(dispatched[0]?.modelFallbackProgress).toBeUndefined(); + }); + + test("dispatchPendingFollowUp settles and drops a follow-up whose persisted step budget is malformed", async () => { + const settle = mock((_correlation: unknown, _reason: string) => Promise.resolve()); + const sendMessage = mock(() => Promise.resolve({ success: true as const })); + const { internals, historyService } = await createSession( + [ + compactionSummaryMessage("summary-malformed-budget", { + text: "Continue", + model: "openai:gpt-4o", + agentId: "exec", + stepBudget: "seven" as unknown as number, + muxMetadata: DELEGATED_TURN, + }), + ], + { settleForfeitedWorkspaceTurnContinuation: settle } + ); + internals.sendMessage = sendMessage; + + expect(await internals.dispatchPendingFollowUp()).toBe(false); + + // The interrupted turn's ceiling is unknowable from this row; it must not get the default one. + expect(sendMessage).not.toHaveBeenCalled(); + expect(settle).toHaveBeenCalledTimes(1); + expect(settle.mock.calls[0]?.[0]).toEqual(DELEGATED_TURN); + const tail = await historyService.getLastMessages("ws", 1); + const summary = tail.success ? tail.data[0] : undefined; + expect(summary?.metadata?.muxMetadata).toEqual({ type: "compaction-summary" }); + }); + test("dispatchPendingFollowUp forwards strictAgentResolution to the resumed turn", async () => { let dispatchedOptions: SendOptions | undefined; const { internals } = await createSession([ @@ -311,7 +774,7 @@ describe("AgentSession continue-message agentId fallback", () => { agentId: "exec", }), ], - archivedConfig + { config: archivedConfig } ); internals.sendMessage = mock(() => Promise.resolve({ success: true as const })); diff --git a/src/node/services/agentSession.postCompactionRetry.test.ts b/src/node/services/agentSession.postCompactionRetry.test.ts index 5307678945..d53732c1eb 100644 --- a/src/node/services/agentSession.postCompactionRetry.test.ts +++ b/src/node/services/agentSession.postCompactionRetry.test.ts @@ -504,4 +504,154 @@ describe("AgentSession post-compaction context retry", () => { session.dispose(); }); + + // A revalidated turn's retry is admitted, then the delegated handle is interrupted while the + // retry's launch prepares: StreamManager refuses the launch as a startup-aborted Ok. That is + // not a started retry; the episode must settle terminal so the owner's waiter is released. + test("a revalidated retry refused at the launch boundary settles terminal, not retry-started", async () => { + const workspaceId = "ws-launch-refused"; + const sessionsDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), "mux-agentSession-")); + const sessionDir = path.join(sessionsDir, workspaceId); + await fsPromises.mkdir(sessionDir); + await createPersistedPostCompactionState({ + filePath: path.join(sessionDir, "post-compaction.json"), + diffs: [{ path: "/tmp/foo.ts", diff: "@@ -1 +1 @@\n-foo\n+bar\n", truncated: false }], + }); + const delegatedTurn = { + type: "workspace-turn-task", + taskHandleId: "wst_wake", + ownerWorkspaceId: "owner-ws", + turnId: "turn-wake", + } as const; + + const { historyService, cleanup } = await createTestHistoryService(); + historyCleanup = cleanup; + // An on-send-compacted wake continuation: the stream inherits its delegated turn from the + // summary, not from the wake row or the resume options. + await historyService.appendToHistory(workspaceId, { + id: "compaction-summary", + role: "assistant", + parts: [{ type: "text", text: "Summary" }], + metadata: { + timestamp: 1000, + compacted: "user", + muxMetadata: { + type: "compaction-summary", + pendingFollowUp: { + text: "Background monitor wake", + model: "openai:gpt-4o", + agentId: "exec", + workspaceTurnMetadata: delegatedTurn, + }, + }, + }, + }); + await historyService.appendToHistory(workspaceId, { + id: "user-1", + role: "user", + parts: [{ type: "text", text: "Background monitor wake" }], + metadata: { timestamp: 1100, muxMetadata: { type: "bash-monitor-wake", records: [] } }, + }); + + const aiEmitter = new EventEmitter(); + let retryLaunched!: () => void; + const retryLaunch = new Promise((resolve) => { + retryLaunched = resolve; + }); + let stale = false; + let callCount = 0; + const streamMessage = mock((..._args: unknown[]) => { + callCount += 1; + if (callCount === 1) { + aiEmitter.emit("error", { + workspaceId, + messageId: "assistant-ctx-exceeded", + error: "Context length exceeded", + errorType: "context_exceeded", + }); + return Promise.resolve(contextExceededResult("assistant-ctx-exceeded")); + } + // The handle is interrupted right before registration: no stream-start, a startup-aborted + // handle. + stale = true; + retryLaunched(); + return Promise.resolve({ + success: true as const, + data: { + messageId: "assistant-retry", + completion: Promise.resolve({ status: "aborted" as const, abortReason: "startup" }), + }, + }); + }); + const admit = mock((_correlation: unknown) => + Promise.resolve({ admissible: true, admissionStale: () => stale }) + ); + + const session = new AgentSession({ + workspaceId, + config: { + rootDir: sessionsDir, + sessionsDir, + srcDir: "/tmp", + loadConfigOrDefault: mock(() => ({})), + } as unknown as Config, + historyService, + aiService: { + ...createStreamLifecycleMocks(), + on(eventName: string | symbol, listener: (...args: unknown[]) => void) { + aiEmitter.on(String(eventName), listener); + return this; + }, + off(eventName: string | symbol, listener: (...args: unknown[]) => void) { + aiEmitter.off(String(eventName), listener); + return this; + }, + streamMessage, + getWorkspaceMetadata: mock(() => + Promise.resolve({ success: false as const, error: "nope" }) + ), + } as unknown as AIService, + initStateManager: { + on() { + return this; + }, + off() { + return this; + }, + } as unknown as InitStateManager, + backgroundProcessManager: { + setMessageQueued: mock(() => undefined), + cleanup: mock(() => Promise.resolve()), + } as unknown as BackgroundProcessManager, + admitStrandedTurnResume: admit, + }); + + const resumed = await session.resumeStream( + { model: "openai:gpt-4o", agentId: "exec" }, + { revalidateAdmission: true } + ); + expect(resumed.success).toBe(true); + + const withTimeout = (promise: Promise, label: string): Promise => + Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`${label} timeout`)), 1000) + ), + ]); + await withTimeout(retryLaunch, "retry launch"); + expect( + await withTimeout( + session.waitForPendingStreamErrorRecoveryDecision("assistant-ctx-exceeded"), + "decision" + ) + ).toBe("terminal"); + expect(session.isPreparingTurn()).toBe(false); + expect(callCount).toBe(2); + // The retry revalidated against the inherited delegated turn, not just the wake. + expect(admit).toHaveBeenCalledTimes(2); + expect(admit.mock.calls[1]?.[0]).toEqual(delegatedTurn); + + session.dispose(); + }); }); diff --git a/src/node/services/agentSession.queueDispatch.test.ts b/src/node/services/agentSession.queueDispatch.test.ts index 08c6774a27..82e198b6b6 100644 --- a/src/node/services/agentSession.queueDispatch.test.ts +++ b/src/node/services/agentSession.queueDispatch.test.ts @@ -1,12 +1,31 @@ import { describe, expect, mock, spyOn, test } from "bun:test"; -import type { MuxMessageMetadata } from "@/common/types/message"; +import { EventEmitter } from "node:events"; + +import type { SendMessageOptions } from "@/common/orpc/types"; +import { createMuxMessage, type MuxMessageMetadata } from "@/common/types/message"; import { Err, Ok } from "@/common/types/result"; +import { createDisplayUsage } from "@/common/utils/tokens/displayUsage"; +import { getTotalCost } from "@/common/utils/tokens/usageAggregator"; +import { GOAL_CONTINUATION_KIND } from "@/constants/goals"; import type { WorkspaceGoalService } from "./workspaceGoalService"; -import { createAgentSessionHarness, createStartedTurnHandle } from "./agentSession.testHarness"; -import type { AIService } from "./aiService"; +import { + createAgentSessionHarness, + createFailedTurnHandle, + createStartedTurnHandle, + type AgentSessionHarnessOptions, +} from "./agentSession.testHarness"; +import type { AIService, StreamMessageOptions } from "./aiService"; +import type { HistoryService } from "./historyService"; +import type { TurnCompletion, TurnStreamHandle } from "./streamManager"; const TEST_MODEL = "anthropic:claude-sonnet-4-5"; +/** Steps a cut stream reports as left; tests about the budget itself pass their own value. */ +const CUT_STEPS_REMAINING = 1_000; +const queuedStop = (modelString = TEST_MODEL) => ({ + modelString, + stepsRemaining: CUT_STEPS_REMAINING, +}); const WORKSPACE_TURN_CORRELATION = { type: "workspace-turn-task", taskHandleId: "wst_preparing", @@ -38,14 +57,107 @@ function streamStartEvent(workspaceId: string): Record { function streamAbortEvent( workspaceId: string, - abortReason: "system" | "user" + abortReason: "system" | "user" | "queued-message", + stepsRemaining = CUT_STEPS_REMAINING ): Record { return { type: "stream-abort", workspaceId, messageId: "assistant-1", abortReason, - metadata: { duration: 1 }, + metadata: { duration: 1, stepsRemaining }, + }; +} + +function streamEndEvent(workspaceId: string): Record { + return { + type: "stream-end", + workspaceId, + messageId: "assistant-1", + parts: [], + metadata: { + model: TEST_MODEL, + contextUsage: { inputTokens: 5, outputTokens: 5, totalTokens: 10 }, + providerMetadata: {}, + finishReason: "tool-calls", + }, + }; +} + +/** + * Session whose engine double behaves like the real one for turn phases: every + * streamMessage call emits stream-start before resolving, so the session is STREAMING + * (not back to IDLE) once a send or resume returns. + */ +async function createStreamingTurnHarness( + workspaceId: string, + setup?: { + harness?: Partial>; + seedHistory?: (historyService: HistoryService) => Promise; + sendOptions?: Partial; + sendInternal?: { synthetic?: boolean; agentInitiated?: boolean }; + } +) { + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const harness = await createAgentSessionHarness({ + ...setup?.harness, + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + await setup?.seedHistory?.(harness.historyService); + const sent = await harness.session.sendMessage( + "run the checks", + { model: TEST_MODEL, agentId: "exec", ...setup?.sendOptions }, + setup?.sendInternal + ); + expect(sent.success).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(harness.session.isBusy()).toBe(true); + // The tool step's committed assistant row: the row the model never answered when stranded. + await harness.historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + /** Queue a synthetic tool-end entry whose cancel signal the caller controls. */ + const queueCancelable = (message: string, muxMetadata?: MuxMessageMetadata): AbortController => { + const controller = new AbortController(); + harness.session.queueMessage( + message, + { model: TEST_MODEL, agentId: "exec", muxMetadata }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + return controller; + }; + // A real wake is typed so a delegated turn's owner recognizes it as the turn's continuation. + const queueCancelableWake = (): AbortController => + queueCancelable("Background monitor wake", { type: "bash-monitor-wake", records: [] }); + const queueCancelableUnrelatedEntry = (): AbortController => queueCancelable("peer follow-up"); + const latestRequest = (): StreamMessageOptions => { + const call = streamMessage.mock.calls[streamMessage.mock.calls.length - 1]; + if (call == null) { + throw new Error("no streamMessage call recorded"); + } + return call[0]; + }; + + return { + ...harness, + streamMessage, + queueCancelableWake, + queueCancelableUnrelatedEntry, + latestRequest, }; } @@ -67,8 +179,9 @@ describe("AgentSession queued message tool-call dispatch", () => { hasQueuedOrDispatchingEntry( continuationMetadata?: Extract ): boolean; - hasPendingWorkspaceTurnContinuation( - continuationMetadata: Extract + claimWorkspaceTurnContinuation( + continuationMetadata: Extract, + streamEndMessageId: string ): boolean; }; } = {}; @@ -92,12 +205,13 @@ describe("AgentSession queued message tool-call dispatch", () => { }) === true, uncorrelated: session?.hasQueuedOrDispatchingEntry() === true, pendingSameTurn: - session?.hasPendingWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION) === true, + session?.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") === + true, pendingDifferentTurn: - session?.hasPendingWorkspaceTurnContinuation({ - ...WORKSPACE_TURN_CORRELATION, - turnId: "turn-different", - }) === true, + session?.claimWorkspaceTurnContinuation( + { ...WORKSPACE_TURN_CORRELATION, turnId: "turn-different" }, + "assistant-1" + ) === true, }; return Promise.resolve(Ok(createStartedTurnHandle())); }); @@ -357,10 +471,10 @@ describe("AgentSession queued message tool-call dispatch", () => { expect(stopStream).toHaveBeenCalledWith(workspaceId, { soft: true, - abortReason: "system", + abortReason: "queued-message", }); - aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message")); const didDispatch = await waitForCondition(() => sendQueuedMessages.mock.calls.length > 0); expect(didDispatch).toBe(true); expect(sendQueuedMessages).toHaveBeenCalledTimes(1); @@ -785,62 +899,34 @@ describe("AgentSession queued message tool-call dispatch", () => { } }); - test("rollback failure preserves the wake and continues acceptance", async () => { - const workspaceId = "queue-dispatch-cancel-rollback-failure"; - const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); - const originalAppend = historyService.appendToHistory.bind(historyService); - let markAppendStarted: () => void = () => undefined; - const appendStarted = new Promise((resolve) => { - markAppendStarted = resolve; - }); - let releaseAppend: () => void = () => undefined; - const appendRelease = new Promise((resolve) => { - releaseAppend = resolve; - }); - const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( - async (...args) => { - markAppendStarted(); - await appendRelease; - return originalAppend(...args); - } - ); - const deleteMessagesSpy = spyOn(historyService, "deleteMessages").mockResolvedValue( - Err("injected rollback failure") - ); + test("resumes a turn whose queued tool-end stop message is withdrawn before acceptance", async () => { + const workspaceId = "queue-dispatch-stranded-resume"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, historyService, streamMessage } = harness; try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; - let accepted = false; - const sendPromise = session.sendMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, - onAccepted: () => { - accepted = true; - }, - } - ); - - await appendStarted; - controller.abort("monitor canceled"); - releaseAppend(); - const result = await sendPromise; + const wake = harness.queueCancelableWake(); + const request = harness.latestRequest(); + expect(request.hasQueuedMessages?.("tool-end")).toBe(true); + // StreamManager stopped the loop for the queued wake; the wake is then withdrawn + // (its output was consumed another way) before the stream-end drain dispatches it. + request.onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); - expect(result.success).toBe(true); - expect(deleteMessagesSpy).toHaveBeenCalledTimes(1); - expect(canceledReasons).toEqual([]); - expect(cancelState.canceledBeforeAcceptance).toBe(false); - expect(accepted).toBe(true); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const resumed = harness.latestRequest(); + // The continuation keeps the interrupted (user-started) turn's attribution. + expect(resumed.agentInitiated).toBe(streamMessage.mock.calls[0]?.[0].agentInitiated); + expect(resumed.modelString).toBe(TEST_MODEL); + expect(resumed.agentId).toBe("exec"); + // The resumed request ends with a user turn so the model has something to answer. + expect(resumed.messages[resumed.messages.length - 1]?.role).toBe("user"); + expect(session.isBusy()).toBe(true); + expect(session.hasQueuedMessages()).toBe(false); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(2); const history = await historyService.getHistoryFromLatestBoundary(workspaceId); expect(history.success).toBe(true); if (history.success) { @@ -850,190 +936,3019 @@ describe("AgentSession queued message tool-call dispatch", () => { (part) => part.type === "text" && part.text === "Background monitor wake" ) ) - ).toBe(true); + ).toBe(false); } } finally { - releaseAppend(); - deleteMessagesSpy.mockRestore(); - appendSpy.mockRestore(); session.dispose(); await cleanup(); } }); - test("verifies a committed rollback when batch deletion reports a post-write failure", async () => { - const workspaceId = "queue-dispatch-cancel-post-write-failure"; - const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); - const originalAppend = historyService.appendToHistory.bind(historyService); - const originalDeleteMessages = historyService.deleteMessages.bind(historyService); - let markAppendStarted: () => void = () => undefined; - const appendStarted = new Promise((resolve) => { - markAppendStarted = resolve; - }); - let releaseAppend: () => void = () => undefined; - const appendRelease = new Promise((resolve) => { - releaseAppend = resolve; - }); - const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( - async (...args) => { - markAppendStarted(); - await appendRelease; - return originalAppend(...args); - } - ); - const deleteMessagesSpy = spyOn(historyService, "deleteMessages").mockImplementation( - async (...args) => { - const result = await originalDeleteMessages(...args); - expect(result.success).toBe(true); - return Err("injected post-write failure"); - } - ); + test("a queued tool-end message that dispatches normally is the continuation", async () => { + const workspaceId = "queue-dispatch-stranded-dispatched"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; - let accepted = false; - const sendPromise = session.sendMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, - onAccepted: () => { - accepted = true; - }, - } - ); - - await appendStarted; - controller.abort("monitor canceled"); - releaseAppend(); - const result = await sendPromise; + harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); - expect(result.success).toBe(true); - expect(deleteMessagesSpy).toHaveBeenCalledTimes(1); - expect(canceledReasons).toEqual(["monitor canceled"]); - expect(cancelState.canceledBeforeAcceptance).toBe(true); - expect(accepted).toBe(false); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const dispatched = harness.latestRequest(); + const lastMessage = dispatched.messages[dispatched.messages.length - 1]; + expect(lastMessage?.role).toBe("user"); + expect( + lastMessage?.parts.some( + (part) => part.type === "text" && part.text === "Background monitor wake" + ) + ).toBe(true); + expect(session.isBusy()).toBe(true); - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (history.success) { - expect( - history.data.some((message) => - message.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(false); - } + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(2); } finally { - releaseAppend(); - deleteMessagesSpy.mockRestore(); - appendSpy.mockRestore(); session.dispose(); await cleanup(); } }); - test("cancellation during goal sync crosses the acceptance point of no return", async () => { - const workspaceId = "queue-dispatch-cancel-goal-reconcile"; - let markInitialSyncStarted: () => void = () => undefined; - const initialSyncStarted = new Promise((resolve) => { - markInitialSyncStarted = resolve; - }); - let releaseInitialSync: () => void = () => undefined; - const initialSyncRelease = new Promise((resolve) => { - releaseInitialSync = resolve; - }); - let syncCalls = 0; - const syncGoalModeWithChatTail = mock(async () => { - syncCalls += 1; - if (syncCalls === 1) { - markInitialSyncStarted(); - await initialSyncRelease; - } - return null; - }); - const workspaceGoalService = { - assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), - syncGoalModeWithChatTail, - } as unknown as WorkspaceGoalService; - const { session, cleanup, historyService } = await createAgentSessionHarness({ - workspaceId, - workspaceGoalService, - }); + test("does not resume when the loop ended without stopping for a queued message", async () => { + const workspaceId = "queue-dispatch-stranded-not-stopped"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; try { - const controller = new AbortController(); - const cancelState = { canceledBeforeAcceptance: false }; - const canceledReasons: string[] = []; - let accepted = false; - const sendPromise = session.sendMessage( - "Background monitor wake", - { model: TEST_MODEL, agentId: "exec" }, - { - synthetic: true, - agentInitiated: true, - cancelState, - cancelSignal: controller.signal, - onCanceled: (reason) => { - canceledReasons.push(reason); - }, - onAccepted: () => { - accepted = true; - }, - } - ); + // A required tool (agent_report) ended the turn while a wake happened to be queued. + const wake = harness.queueCancelableWake(); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); - await initialSyncStarted; - controller.abort("monitor canceled"); - releaseInitialSync(); - const result = await sendPromise; + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); - expect(result.success).toBe(true); - expect(syncGoalModeWithChatTail).toHaveBeenCalledTimes(1); - expect(canceledReasons).toEqual([]); - expect(cancelState.canceledBeforeAcceptance).toBe(false); - expect(accepted).toBe(true); + test("a turn stranded after each of several awaited monitors resumes every time", async () => { + const workspaceId = "queue-dispatch-stranded-chain"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (history.success) { - expect( - history.data.some((message) => - message.parts.some( - (part) => part.type === "text" && part.text === "Background monitor wake" - ) - ) - ).toBe(true); + try { + // Each cut follows a completed step whose task_await consumed the monitor's wake (dogfood + // UAT: four sequential background+await calls in one prompt); the failed-start cap must + // not end it. What bounds the chain is the turn's step budget, run down by every cut. + const strandTurn = (stepsRemaining: number) => { + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL, stepsRemaining }); + harness.queueCancelableWake().abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + }; + + for (let resumes = 1; resumes <= 4; resumes += 1) { + const stepsRemaining = 5 - resumes; + strandTurn(stepsRemaining); + expect(await waitForCondition(() => streamMessage.mock.calls.length === resumes + 1)).toBe( + true + ); + expect(session.isBusy()).toBe(true); + expect(harness.latestRequest().stepBudget).toBe(stepsRemaining); } } finally { - releaseInitialSync(); session.dispose(); await cleanup(); } }); - test("disposed sessions finalize durable wakes after goal sync completes", async () => { - const workspaceId = "queue-dispatch-disposed-after-goal-sync"; - let markSyncStarted: () => void = () => undefined; - const syncStarted = new Promise((resolve) => { - markSyncStarted = resolve; - }); - let releaseSync: () => void = () => undefined; - const syncRelease = new Promise((resolve) => { - releaseSync = resolve; - }); - const syncGoalModeWithChatTail = mock(async () => { - markSyncStarted(); - await syncRelease; + test("a resumed turn runs under the cut stream's remaining step budget", async () => { + const workspaceId = "queue-dispatch-stranded-step-budget"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + expect(harness.latestRequest().stepBudget).toBeUndefined(); + + // Cut by the loop's stop condition: the resume inherits what that stream had left. + harness.latestRequest().onQueuedMessageStop?.({ modelString: TEST_MODEL, stepsRemaining: 7 }); + harness.queueCancelableWake().abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().stepBudget).toBe(7); + + // Cut by a provider-tool soft stop: the abort reports the budget the next stream gets. + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message", 3)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 3)).toBe(true); + expect(harness.latestRequest().stepBudget).toBe(3); + + // A cut stream that spent its whole ceiling ended the turn: nothing is owed. + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:2" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(2); + expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message", 0)); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(3); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("a provider-tool soft stop after a successful required tool owes no continuation", async () => { + const workspaceId = "queue-dispatch-soft-stop-required-tool"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + // The provider-executed tool that ended the batch was the turn's required completion tool: + // the loop would have stopped on it one result later, so the withdrawn wake strands nothing. + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); + aiEmitter.emit("stream-abort", { + ...streamAbortEvent(workspaceId, "queued-message", 3), + metadata: { duration: 1, stepsRemaining: 3, requiredToolSatisfied: true }, + }); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("a resumed turn continues the cut stream's fallback chain", async () => { + const workspaceId = "queue-dispatch-stranded-fallback-chain"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + // The requested model refused and the cut reached its first fallback. + const progress = { + requestedModel: TEST_MODEL, + refusedModels: [TEST_MODEL], + chain: ["openai:gpt-5-fallback", "google:gemini-fallback"], + }; + + try { + expect(harness.latestRequest().modelFallbackProgress).toBeUndefined(); + + // Cut by the loop's stop condition: the resume runs on the fallback under the cut turn's + // chain, not a chain of the fallback's own. + harness.latestRequest().onQueuedMessageStop?.({ + modelString: "openai:gpt-5-fallback", + stepsRemaining: 7, + modelFallbackProgress: progress, + }); + harness.queueCancelableWake().abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().modelString).toBe("openai:gpt-5-fallback"); + expect(harness.latestRequest().modelFallbackProgress).toEqual(progress); + + // Cut by a provider-tool soft stop: the abort reports the chain state the next stream gets. + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); + const abortedProgress = { ...progress, refusedModels: [TEST_MODEL, "openai:gpt-5-fallback"] }; + aiEmitter.emit("stream-abort", { + ...streamAbortEvent(workspaceId, "queued-message", 3), + metadata: { + duration: 1, + stepsRemaining: 3, + model: "google:gemini-fallback", + modelFallbackProgress: abortedProgress, + }, + }); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 3)).toBe(true); + expect(harness.latestRequest().modelString).toBe("google:gemini-fallback"); + expect(harness.latestRequest().modelFallbackProgress).toEqual(abortedProgress); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("an aborted stream is accounted under the model it ran on, not the one requested", async () => { + const workspaceId = "queue-dispatch-abort-accounting-effective-model"; + const recordStreamAccounting = mock((_input: { costUsd: number }) => Promise.resolve()); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + recordStreamAccounting, + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter } = harness; + + try { + // The request named TEST_MODEL; a configured fallback ran the stream on a differently + // priced model and reported the usage for it. + const effectiveModel = "anthropic:claude-opus-4-1"; + const usage = { inputTokens: 1_000_000, outputTokens: 0, totalTokens: 1_000_000 }; + const effectiveCost = getTotalCost(createDisplayUsage(usage, effectiveModel)); + const requestedCost = getTotalCost(createDisplayUsage(usage, TEST_MODEL)); + expect(effectiveCost).toBeGreaterThan(0); + expect(effectiveCost).not.toBe(requestedCost); + + aiEmitter.emit("stream-abort", { + ...streamAbortEvent(workspaceId, "system"), + metadata: { duration: 1, usage, model: effectiveModel }, + }); + expect(await waitForCondition(() => recordStreamAccounting.mock.calls.length === 1)).toBe( + true + ); + expect(recordStreamAccounting.mock.calls[0]?.[0].costUsd).toBeCloseTo(effectiveCost ?? -1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a send's launch-boundary admission probe reaches the stream request", async () => { + const workspaceId = "queue-dispatch-send-launch-probe"; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + + try { + const refuseStreamStart = () => false; + const sent = await session.sendMessage( + "Continue", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, refuseStreamStart } + ); + expect(sent.success).toBe(true); + // StreamManager rechecks this probe right before the stream registers (see resumeStream). + expect(streamMessage.mock.calls[0]?.[0].refuseStreamStart).toBe(refuseStreamStart); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a send refused by its launch probe at the boundary leaves no durable row", async () => { + const workspaceId = "queue-dispatch-send-launch-refused"; + const aiEmitter = new EventEmitter(); + let refused = false; + const streamMessage = mock((_options: StreamMessageOptions) => { + // The goal is paused right before registration: StreamManager returns a startup-aborted + // handle without a stream-start. + refused = true; + return Promise.resolve( + Ok({ + messageId: "assistant-1", + completion: Promise.resolve({ status: "aborted" as const, abortReason: "startup" }), + }) + ); + }); + const { session, historyService, events, cleanup } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + captureEvents: true, + }); + + try { + const sent = await session.sendMessage( + "Continue", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, refuseStreamStart: () => refused } + ); + expect(sent.success).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(1); + + // The row was persisted and shown, then withdrawn with the launch it was appended for. + const persisted = events.find((event) => event.type === "message" && event.role === "user"); + const persistedSequence = + persisted?.type === "message" ? persisted.metadata?.historySequence : undefined; + expect(persistedSequence).toBeDefined(); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success && history.data.some((message) => message.role === "user")).toBe( + false + ); + const deleted = events.find((event) => event.type === "delete"); + expect(deleted?.type === "delete" ? deleted.historySequences : undefined).toEqual([ + persistedSequence ?? -1, + ]); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an auto-retry runs under what the failed resumed attempt left of the step budget", async () => { + const workspaceId = "queue-dispatch-stranded-retry-step-budget"; + const aiEmitter = new EventEmitter(); + let streams = 0; + const streamMessage = mock((_options: StreamMessageOptions) => { + streams += 1; + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + // The resumed stream spends steps, then fails with a retryable error. + return Promise.resolve( + Ok( + streams === 2 + ? { + messageId: "assistant-2", + completion: Promise.resolve({ + status: "failed" as const, + streamError: { + messageId: "assistant-2", + error: "provider closed the connection", + errorType: "api" as const, + }, + stepsRemaining: 2, + }), + } + : createStartedTurnHandle(`assistant-${streams}`) + ) + ); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + + try { + const sent = await session.sendMessage("run the checks", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(sent.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ + modelString: TEST_MODEL, + stepsRemaining: 5, + }); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(streamMessage.mock.calls[1]?.[0].stepBudget).toBe(5); + + // The retry continues the same logical turn under the 2 steps the failed attempt left, not + // the 5 it started with. + expect(await waitForCondition(() => streamMessage.mock.calls.length === 3, 4_000)).toBe(true); + expect(streamMessage.mock.calls[2]?.[0].stepBudget).toBe(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an auto-retry continues from the model the failed resumed attempt reached", async () => { + const workspaceId = "queue-dispatch-stranded-retry-fallback-chain"; + const aiEmitter = new EventEmitter(); + const progressAtCut = { + requestedModel: TEST_MODEL, + refusedModels: [TEST_MODEL], + chain: ["openai:gpt-5-fallback", "google:gemini-fallback"], + }; + // The resumed attempt started on the first fallback, which refused too; the second fallback + // then failed with a retryable error. + const progressAtFailure = { + ...progressAtCut, + refusedModels: [TEST_MODEL, "openai:gpt-5-fallback"], + }; + let streams = 0; + const streamMessage = mock((_options: StreamMessageOptions) => { + streams += 1; + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve( + Ok( + streams === 2 + ? { + messageId: "assistant-2", + completion: Promise.resolve({ + status: "failed" as const, + streamError: { + messageId: "assistant-2", + error: "provider closed the connection", + errorType: "api" as const, + }, + stepsRemaining: 2, + modelString: "google:gemini-fallback", + modelFallbackProgress: progressAtFailure, + }), + } + : createStartedTurnHandle(`assistant-${streams}`) + ) + ); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + + try { + const sent = await session.sendMessage("run the checks", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(sent.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ + modelString: "openai:gpt-5-fallback", + stepsRemaining: 5, + modelFallbackProgress: progressAtCut, + }); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(streamMessage.mock.calls[1]?.[0].modelString).toBe("openai:gpt-5-fallback"); + + // The retry picks the chain up where the failed attempt left it instead of re-running the + // first fallback's refusal. + expect(await waitForCondition(() => streamMessage.mock.calls.length === 3, 4_000)).toBe(true); + expect(streamMessage.mock.calls[2]?.[0].modelString).toBe("google:gemini-fallback"); + expect(streamMessage.mock.calls[2]?.[0].modelFallbackProgress).toEqual(progressAtFailure); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an auto-retry is abandoned when the failed resumed attempt spent the step budget", async () => { + const workspaceId = "queue-dispatch-stranded-retry-step-budget-spent"; + const aiEmitter = new EventEmitter(); + let streams = 0; + const streamMessage = mock((_options: StreamMessageOptions) => { + streams += 1; + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve( + Ok( + streams === 2 + ? { + messageId: "assistant-2", + completion: Promise.resolve({ + status: "failed" as const, + streamError: { + messageId: "assistant-2", + error: "provider closed the connection", + errorType: "api" as const, + }, + stepsRemaining: 0, + }), + } + : createStartedTurnHandle(`assistant-${streams}`) + ) + ); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + const abandoned: string[] = []; + const unsubscribe = session.onChatEvent((event) => { + if (event.message.type === "auto-retry-abandoned") { + abandoned.push(event.message.reason); + } + }); + + try { + const sent = await session.sendMessage("run the checks", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(sent.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ + modelString: TEST_MODEL, + stepsRemaining: 1, + }); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + + // The failed attempt spent the turn's last step: the ceiling ended the turn, no retry runs. + expect(await waitForCondition(() => abandoned.length === 1, 4_000)).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(2); + expect(session.isBusy()).toBe(false); + } finally { + unsubscribe(); + session.dispose(); + await cleanup(); + } + }); + + test("caps resume attempts that never start a stream", async () => { + const workspaceId = "queue-dispatch-stranded-cap"; + let gateOpen = true; + const pricingGate = mock(() => + Promise.resolve(gateOpen ? Ok(undefined) : Err({ type: "unknown", raw: "gate closed" })) + ); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: pricingGate, + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + gateOpen = false; + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + const attemptsBefore = pricingGate.mock.calls.length; + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + + // The failing resume is swept again on its own until the cap, with no poke; then the + // marker is dropped and later pokes do nothing, even once the gate opens. + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(pricingGate.mock.calls.length - attemptsBefore).toBe(3); + expect(streamMessage).toHaveBeenCalledTimes(1); + + gateOpen = true; + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(pricingGate.mock.calls.length - attemptsBefore).toBe(3); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("resumes after a provider-tool soft stop whose wake was canceled but not yet drained", async () => { + const workspaceId = "queue-dispatch-stranded-provider-tool-canceled-wake"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + const wake = harness.queueCancelableWake(); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + + // The reconciler withdraws the wake by abort: it no longer counts as pending work but + // still occupies the queue until the post-abort dispatch drains it. + wake.abort("monitor consumed"); + expect(session.hasQueuedMessages("tool-end")).toBe(false); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message")); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(session.isBusy()).toBe(true); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("resumes after a provider-tool soft stop whose queued message was withdrawn", async () => { + const workspaceId = "queue-dispatch-stranded-provider-tool"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + + // Withdrawn between the soft stop request and the abort it produces. + expect(session.removeQueuedMessagesByDedupeKeyPrefix("wake:", "superseded")).toBe(1); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message")); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().agentInitiated).toBe( + streamMessage.mock.calls[0]?.[0].agentInitiated + ); + expect(session.isBusy()).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(2); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("owes the delegated turn its continuation from the stop decision onward", async () => { + const workspaceId = "queue-dispatch-stranded-delegated"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + // Nothing owed yet: a tool-calls cut with an empty queue is a plain interruption. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + // The entry is gone before the stream ends (dedupe removal / clearQueue shape). + wake.abort("monitor consumed"); + session.clearQueue("monitor consumed"); + + // The owner's settlement runs synchronously with stream-end; it must already see the + // continuation, and only for this correlation. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + expect( + session.claimWorkspaceTurnContinuation( + { ...WORKSPACE_TURN_CORRELATION, turnId: "another-turn" }, + "assistant-1" + ) + ).toBe(false); + + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); + // Consumed by the resumed stream. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a queued unrelated entry supersedes the delegated turn despite the owed continuation", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-superseded"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup } = harness; + + try { + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + session.queueMessage("user follow-up", { model: TEST_MODEL, agentId: "exec" }); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a wake-started continuation resumes with the correlation it inherited from history", async () => { + const workspaceId = "queue-dispatch-stranded-inherited-correlation"; + const wakeMetadata: MuxMessageMetadata = { + type: "bash-monitor-wake", + records: [ + { + processId: "proc-1", + wakeUpdatedAt: "2026-01-01T00:00:00.000Z", + kind: "match", + displayName: "marker", + filter: "MARKER", + filterExclude: false, + }, + ], + }; + const harness = await createStreamingTurnHarness(workspaceId, { + seedHistory: async (historyService) => { + // The delegated turn's stream was cut at a tool boundary by the first wake. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "delegated prompt", { + timestamp: Date.now(), + muxMetadata: WORKSPACE_TURN_CORRELATION, + }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-0", "assistant", "working", { + timestamp: Date.now(), + finishReason: "tool-calls", + muxMetadata: WORKSPACE_TURN_CORRELATION, + }) + ); + }, + sendOptions: { muxMetadata: wakeMetadata }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + // Owed whether the owner asks while the wake is still queued or after it is cleared. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + session.clearQueue("monitor consumed"); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("yields the stranded resume to a manual send in preflight", async () => { + const workspaceId = "queue-dispatch-stranded-preflight"; + let preflightInFlight = true; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { hasExternalSendPreflight: () => preflightInFlight }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + + // The preflight settled without a turn; its idle drain delivers the owed continuation. + preflightInFlight = false; + session.drainQueuedMessagesIfIdle(); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a resume that fails before its stream starts is swept again without a poke", async () => { + const workspaceId = "queue-dispatch-stranded-retry"; + // The initial send passes the gate; the gate then fails this many calls (the resume's). + let failingGateCalls = 0; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => { + if (failingGateCalls > 0) { + failingGateCalls -= 1; + return Promise.resolve(Err({ type: "unknown", raw: "gate closed" })); + } + return Promise.resolve(Ok(undefined)); + }), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + failingGateCalls = 1; + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + // A transient failure before the resume's stream leaves the continuation owed on an idle + // session that nothing else pokes; the sweep tries again on its own and the turn resumes. + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(failingGateCalls).toBe(0); + expect(session.isBusy()).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a stranded goal continuation resumes under the same goal attribution", async () => { + const workspaceId = "queue-dispatch-stranded-goal"; + const recordStreamAccounting = mock((_input: { streamOriginKind: string }) => + Promise.resolve() + ); + const buildGoalRedispatchAdmission = mock(() => + Promise.resolve({ admissible: true as const, admissionStale: () => false }) + ); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + buildGoalRedispatchAdmission, + recordStreamAccounting, + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(1); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + + // The resumed stream's own end is accounted as the same goal continuation. + aiEmitter.emit("stream-end", { ...streamEndEvent(workspaceId), messageId: "assistant-2" }); + expect(await waitForCondition(() => recordStreamAccounting.mock.calls.length === 2)).toBe( + true + ); + expect(recordStreamAccounting.mock.calls.map((call) => call[0].streamOriginKind)).toEqual([ + "goal_continuation", + "goal_continuation", + ]); + // Resumed under the goal's own admission, like any redispatched goal turn. + expect(buildGoalRedispatchAdmission).toHaveBeenCalledWith( + workspaceId, + "goal-1", + GOAL_CONTINUATION_KIND + ); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an auto-retry of a stranded goal resume is admitted by the goal again", async () => { + const workspaceId = "queue-dispatch-stranded-goal-retry-admission"; + const buildGoalRedispatchAdmission = mock( + (): ReturnType => + Promise.resolve({ admissible: true, admissionStale: () => false }) + ); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + buildGoalRedispatchAdmission, + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + let streams = 0; + const streamMessage = mock((_options: StreamMessageOptions) => { + streams += 1; + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + // The resumed stream (the second) fails with a retryable error. + return Promise.resolve( + Ok( + streams === 2 + ? createFailedTurnHandle("assistant-2", { + error: "provider closed the connection", + errorType: "api", + }) + : createStartedTurnHandle(`assistant-${streams}`) + ) + ); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + const retryEvents: string[] = []; + const unsubscribe = session.onChatEvent((event) => { + if (event.message.type === "auto-retry-abandoned") { + retryEvents.push(event.message.type); + } + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(buildGoalRedispatchAdmission).toHaveBeenCalledTimes(1); + + // The goal is paused during the retry backoff: the retry asks the goal again and, refused, + // starts nothing instead of restarting tool-enabled work after the pause. + buildGoalRedispatchAdmission.mockImplementation(() => Promise.resolve({ admissible: false })); + expect( + await waitForCondition(() => buildGoalRedispatchAdmission.mock.calls.length === 2, 4_000) + ).toBe(true); + expect(await waitForCondition(() => retryEvents.length === 1)).toBe(true); + expect(streamMessage).toHaveBeenCalledTimes(2); + expect(session.isBusy()).toBe(false); + } finally { + unsubscribe(); + session.dispose(); + await cleanup(); + } + }); + + /** + * A stranded goal resume (admitted under revalidation, running under a step budget) whose + * stream fails with context_exceeded after post-compaction context was injected, so the + * in-session retry without that context is the recovery path; the caller settles the failure. + */ + async function strandGoalResumeIntoPostCompactionRetry( + workspaceId: string, + buildGoalRedispatchAdmission: ReturnType< + typeof mock + >, + stepsRemainingAfterFailure: number + ) { + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + buildGoalRedispatchAdmission, + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + let streams = 0; + let failResumed: () => void = () => undefined; + const streamMessage = mock((_options: StreamMessageOptions) => { + streams += 1; + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + if (streams !== 2) { + return Promise.resolve(Ok(createStartedTurnHandle(`assistant-${streams}`))); + } + const completion = new Promise((resolve) => { + failResumed = () => + resolve({ + status: "failed", + streamError: { + messageId: "assistant-2", + error: "context window exceeded", + errorType: "context_exceeded", + }, + stepsRemaining: stepsRemainingAfterFailure, + }); + }); + return Promise.resolve(Ok({ messageId: "assistant-2", completion })); + }); + const harness = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + const { session, historyService } = harness; + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.({ + modelString: TEST_MODEL, + stepsRemaining: 5, + }); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(streamMessage.mock.calls[1]?.[0].stepBudget).toBe(5); + expect(buildGoalRedispatchAdmission).toHaveBeenCalledTimes(1); + // The resumed stream carried post-compaction context (the retry's precondition). + Reflect.set(session, "activeStreamHadPostCompactionInjection", true); + return { ...harness, streamMessage, failResumed: () => failResumed() }; + } + + test("the post-compaction retry of a stranded goal resume is admitted by the goal again", async () => { + const workspaceId = "queue-dispatch-stranded-goal-post-compaction-retry-admission"; + const buildGoalRedispatchAdmission = mock( + (): ReturnType => + Promise.resolve({ admissible: true, admissionStale: () => false }) + ); + const harness = await strandGoalResumeIntoPostCompactionRetry( + workspaceId, + buildGoalRedispatchAdmission, + 3 + ); + const { session, cleanup, streamMessage } = harness; + + try { + // The goal is paused before the failure's cleanup finishes: the retry asks the goal again + // and, refused, starts nothing. + buildGoalRedispatchAdmission.mockImplementation(() => Promise.resolve({ admissible: false })); + harness.failResumed(); + expect( + await waitForCondition(() => buildGoalRedispatchAdmission.mock.calls.length === 2) + ).toBe(true); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("the post-compaction retry of a stranded resume runs under what the failed attempt left", async () => { + const workspaceId = "queue-dispatch-stranded-goal-post-compaction-retry-budget"; + const buildGoalRedispatchAdmission = mock( + (): ReturnType => + Promise.resolve({ admissible: true, admissionStale: () => false }) + ); + const harness = await strandGoalResumeIntoPostCompactionRetry( + workspaceId, + buildGoalRedispatchAdmission, + 3 + ); + const { session, cleanup, streamMessage } = harness; + + try { + harness.failResumed(); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 3)).toBe(true); + // Same logical turn: the retry runs under the 3 steps the failed attempt left, not the 5 it + // started with, and the goal admitted it again. + expect(streamMessage.mock.calls[2]?.[0].stepBudget).toBe(3); + expect(buildGoalRedispatchAdmission).toHaveBeenCalledTimes(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("drops a stranded goal continuation the goal no longer admits", async () => { + const workspaceId = "queue-dispatch-stranded-goal-paused"; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + // The user paused the goal while the turn ran; the pause landed at stream end. + buildGoalRedispatchAdmission: mock(() => Promise.resolve({ admissible: false as const })), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + // Forfeited, not deferred: a later idle poke must not revive the paused goal's turn. + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a user Stop withdraws the owed continuation", async () => { + const workspaceId = "queue-dispatch-stranded-user-stop"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + // WorkspaceService.interruptStream: hard stop, then restore the queue to the composer. + expect((await session.interruptStream()).success).toBe(true); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "user")); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + session.restoreQueueToInput(); + + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + expect(session.hasQueuedMessages()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("the resume holds the turn while its admission gates run", async () => { + const workspaceId = "queue-dispatch-stranded-claims-turn"; + let releaseGate: () => void = () => undefined; + let gateArmed = false; + let gateReached = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => { + if (!gateArmed) { + return Promise.resolve(Ok(undefined)); + } + gateReached = true; + return new Promise((resolve) => { + releaseGate = () => resolve(Ok(undefined)); + }); + }), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + gateArmed = true; + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + // The resume is parked on the pricing gate: the session must already read busy so a + // manual send arriving now queues behind it instead of starting a colliding stream. + expect(await waitForCondition(() => gateReached)).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(true); + + releaseGate(); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("the resume drops the interrupted turn's ACP prompt binding", async () => { + const workspaceId = "queue-dispatch-stranded-acp"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { acpPromptId: "prompt-1", delegatedToolNames: ["bash"] }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const original = harness.latestRequest(); + expect(original.acpPromptId).toBe("prompt-1"); + expect(original.delegatedToolNames).toEqual(["bash"]); + + const wake = harness.queueCancelableWake(); + original.onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const resumed = harness.latestRequest(); + expect(resumed.modelString).toBe(TEST_MODEL); + // The ACP turn completed at the first stream-end; a delegated tool call on the resumed + // stream would otherwise wait on a prompt nobody answers. + expect(resumed.acpPromptId).toBeUndefined(); + expect(resumed.delegatedToolNames).toBeUndefined(); + } finally { + session.dispose(); + await cleanup(); + } + }); + + /** + * Strand a delegated turn whose owner deferred the cut stream-end on the advertised + * continuation. With `holdAdmission`, a history mutation holds turn admission across the + * stream end, so the session sits idle with the continuation still owed and no resume run. + * Without it, the resume fails at the pricing gate before its stream starts. + */ + async function strandDelegatedTurn( + workspaceId: string, + extra?: { + harness?: Partial>; + settleForfeited?: ReturnType< + typeof mock<(metadata: unknown, reason: string) => Promise> + >; + holdAdmission?: boolean; + } + ) { + let gateOpen = true; + const pricingGate = mock(() => + Promise.resolve(gateOpen ? Ok(undefined) : Err({ type: "unknown", raw: "gate closed" })) + ); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: pricingGate, + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const settleForfeited = + extra?.settleForfeited ?? mock((_metadata: unknown, _reason: string) => Promise.resolve()); + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { + workspaceGoalService, + settleForfeitedWorkspaceTurnContinuation: settleForfeited, + ...extra?.harness, + }, + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, aiEmitter } = harness; + const hold = extra?.holdAdmission === true ? session.holdTurnAdmission() : undefined; + gateOpen = hold != null; + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + harness.queueCancelableWake().abort("monitor consumed"); + session.clearQueue("monitor consumed"); + expect(session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1")).toBe( + true + ); + const gateCallsBeforeCut = pricingGate.mock.calls.length; + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + return { + ...harness, + settleForfeited, + resumeAttempts: () => pricingGate.mock.calls.length - gateCallsBeforeCut, + openGate: () => { + gateOpen = true; + }, + releaseAdmission: () => hold?.[Symbol.dispose](), + }; + } + + test("stops advertising the delegated continuation once the resume cap is exhausted", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-cap"; + const harness = await strandDelegatedTurn(workspaceId); + const { session, cleanup, streamMessage, settleForfeited } = harness; + + try { + // The resume keeps failing before its stream starts and is swept again on its own until + // the cap; then the continuation is no longer advertised and the owner is told to settle + // the turn it deferred at the cut, with no poke from anyone. + expect(await waitForCondition(() => settleForfeited.mock.calls.length === 1)).toBe(true); + expect(harness.resumeAttempts()).toBe(3); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + harness.openGate(); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(harness.resumeAttempts()).toBe(3); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a context-discarding mutation settles the delegated turn it had advertised", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-context-discard"; + const harness = await strandDelegatedTurn(workspaceId, { holdAdmission: true }); + const { session, cleanup, streamMessage, settleForfeited } = harness; + + try { + // A history clear admitted on the idle session (under the admission hold) discards the + // transcript the continuation would resume from; no stream follows it to settle the turn + // the owner deferred. + const discarded = await session.discardAutoRetryForContextMutation(); + expect(discarded.success).toBe(true); + expect(settleForfeited).toHaveBeenCalledTimes(1); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + harness.releaseAdmission(); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a task hard stop on the stranded delegated turn settles the turn it had advertised", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-hard-stop"; + const harness = await strandDelegatedTurn(workspaceId, { holdAdmission: true }); + const { session, cleanup, streamMessage, settleForfeited } = harness; + + try { + // The stop found the cut stream completed, so the queue clear is the only boundary the + // session sees; the owner deferred on the marker and no stream event will reach it. + session.clearQueue("task stopped", { hardStop: true }); + expect(settleForfeited).toHaveBeenCalledTimes(1); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + harness.releaseAdmission(); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("disposing the session settles the delegated turn it had advertised", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-dispose"; + const harness = await strandDelegatedTurn(workspaceId, { holdAdmission: true }); + const { session, cleanup, settleForfeited } = harness; + + try { + // Workspace removal tears the idle session down with the continuation still owed. + session.dispose(); + expect(settleForfeited).toHaveBeenCalledTimes(1); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a forfeited turn's failed settlement stays owed and retries through disposal", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-settle-retry"; + let settleAttempts = 0; + const settleForfeited = mock((_metadata: unknown, _reason: string) => { + settleAttempts += 1; + return settleAttempts === 1 + ? Promise.reject(new Error("task store unavailable")) + : Promise.resolve(); + }); + const harness = await strandDelegatedTurn(workspaceId, { + settleForfeited, + holdAdmission: true, + }); + const { session, cleanup } = harness; + + try { + // A history clear forfeits the continuation; the owner's store rejects the first settlement. + expect((await session.discardAutoRetryForContextMutation()).success).toBe(true); + expect(await waitForCondition(() => settleAttempts === 1)).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(settleAttempts).toBe(1); + + // The obligation survives the failure and session teardown: its own retry lands even when + // no future idle poke can occur after workspace removal. + session.dispose(); + expect(await waitForCondition(() => settleAttempts === 2, 1_500)).toBe(true); + expect(settleForfeited.mock.calls[1]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(settleAttempts).toBe(2); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a resume whose workspace or delegated turn no longer admits it settles the turn", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-turn-stopped"; + let turnActive = true; + const admitStrandedTurnResume = mock((_correlation: unknown) => + Promise.resolve({ admissible: turnActive, admissionStale: () => !turnActive }) + ); + const settleForfeited = mock((_metadata: unknown, _reason: string) => Promise.resolve()); + const harness = await strandDelegatedTurn(workspaceId, { + harness: { admitStrandedTurnResume }, + settleForfeited, + holdAdmission: true, + }); + const { session, cleanup, streamMessage } = harness; + + try { + // No resume has run under the admission hold; the continuation is still advertised. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + + // task_stop / interrupt_active settled the handle while the cut stream was already + // complete: no abort reached the session, only the owner's record changed. + turnActive = false; + harness.releaseAdmission(); + expect(await waitForCondition(() => settleForfeited.mock.calls.length === 1)).toBe(true); + expect(admitStrandedTurnResume.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a stop landing between admission and launch refuses the resume", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-stop-in-flight"; + // Admitted on read, but the workspace's stop epoch moved before the stream could launch. + const admitStrandedTurnResume = mock((_correlation: unknown) => + Promise.resolve({ admissible: true, admissionStale: () => true }) + ); + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + admitStrandedTurnResume, + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { + timestamp: Date.now(), + muxMetadata: WORKSPACE_TURN_CORRELATION, + }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { agentInitiated: true, revalidateAdmission: true } + ); + expect(resumed).toEqual(Ok({ started: false, refusedBy: "workspace-turn" })); + expect(streamMessage).not.toHaveBeenCalled(); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a launch refused after the [CONTINUE] sentinel landed removes the sentinel", async () => { + const workspaceId = "queue-dispatch-stranded-refused-sentinel"; + // Admitted on read; the stop lands while the resume appends its sentinel. + let stale = false; + const admitStrandedTurnResume = mock((_correlation: unknown) => + Promise.resolve({ admissible: true, admissionStale: () => stale }) + ); + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + admitStrandedTurnResume, + }); + const append = historyService.appendToHistory.bind(historyService); + const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( + async (targetWorkspaceId, message) => { + const result = await append(targetWorkspaceId, message); + if ( + message.role === "user" && + message.parts.some((part) => part.type === "text" && part.text === "[CONTINUE]") + ) { + stale = true; + } + return result; + } + ); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "delegated prompt", { + timestamp: Date.now(), + muxMetadata: WORKSPACE_TURN_CORRELATION, + }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-0", "assistant", "working", { + timestamp: Date.now(), + finishReason: "tool-calls", + muxMetadata: WORKSPACE_TURN_CORRELATION, + }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { agentInitiated: true, revalidateAdmission: true } + ); + expect(resumed).toEqual(Ok({ started: false, refusedBy: "workspace-turn" })); + expect(streamMessage).not.toHaveBeenCalled(); + + // The sentinel was appended for a launch that never happened; a later unrelated turn must + // not send it to the provider. + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data.map((message) => message.id)).toEqual(["user-0", "assistant-0"]); + } + } finally { + appendSpy.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("a user Stop during the resume's admission gate cancels it", async () => { + const workspaceId = "queue-dispatch-stranded-stop-in-admission"; + let releaseGate: () => void = () => undefined; + let gateArmed = false; + let gateReached = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => { + if (!gateArmed) { + return Promise.resolve(Ok(undefined)); + } + gateReached = true; + return new Promise((resolve) => { + releaseGate = () => resolve(Ok(undefined)); + }); + }), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + gateArmed = true; + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => gateReached)).toBe(true); + + // Stop lands while the claimed resume is parked on its gate: StreamManager has no stream + // to abort, so the resume itself must not proceed once the gate opens. + expect((await session.interruptStream()).success).toBe(true); + releaseGate(); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a goal transition between admission and launch drops the resume", async () => { + const workspaceId = "queue-dispatch-stranded-goal-stale"; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + // Admitted on read, but the pause generation moved before the stream could launch. + buildGoalRedispatchAdmission: mock(() => + Promise.resolve({ admissible: true as const, admissionStale: () => true }) + ), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an unrelated queued entry that never starts does not revive the superseded delegated turn", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-orphan"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const entry = harness.queueCancelableUnrelatedEntry(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + // The queued entry is not this turn's continuation: the owner settles the delegated turn. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + // The entry is withdrawn after dequeue, before acceptance; the settled turn must stay cut. + entry.abort("superseded"); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an unrelated entry cleared before dispatch does not revive the superseded delegated turn", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-orphan-cleared"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + harness.queueCancelableUnrelatedEntry(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + // The owner settles the turn on this answer. The entry then leaves the queue without ever + // dispatching (user clears the queue), which must not bring the settled turn back. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + session.clearQueue("queue cleared by user"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + // With the queue empty, only a surviving marker could answer true here. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an unrelated entry removed before the owner's claim leaves the continuation owed", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-removed-before-claim"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + harness.queueCancelableUnrelatedEntry(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + session.clearQueue("queue cleared by user"); + // Nothing supersedes the turn by the time the owner asks: it defers, and the resume runs. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a delegated turn cut by a wake withdrawn after dequeue resumes", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-wake-after-dequeue"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + // A wake inherits the turn's correlation when it sends, so the owner defers on it even + // though the queued entry carries none; the continuation stays owed behind it. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + // Withdrawn after dequeue, before acceptance: the deferred turn must resume, not hang. + wake.abort("monitor consumed"); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a withdrawn wake ahead of an unrelated entry is not the delegated turn's continuation", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-withdrawn-wake-ahead"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + const entry = harness.queueCancelableUnrelatedEntry(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + // The wake's monitor was consumed before the owner asks; the live entry behind it is the + // cutter, and it does not continue this turn. + wake.abort("monitor consumed"); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + // The owner settled the turn on that answer: the entry leaving before acceptance must not + // bring it back. + entry.abort("superseded"); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("clearing the queued continuation of the delegated turn settles it before any sweep", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-continuation-cleared"; + const settleForfeited = mock((_metadata: unknown, _reason: string) => Promise.resolve()); + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { settleForfeitedWorkspaceTurnContinuation: settleForfeited }, + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + // The owner queued a continuation of the same turn; its onCanceled is the owner's settlement + // of that turn and takes its time (task-store I/O). + let finishCancel: () => void = () => undefined; + const canceled = new Promise((resolve) => { + finishCancel = resolve; + }); + session.queueMessage( + "continue the turn", + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { agentInitiated: true, onCanceled: () => canceled } + ); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + // Admission is held across the stream end (a history mutation), so the entry stays queued + // on an idle session instead of dispatching. + const hold = session.holdTurnAdmission(); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + + // The user clears the queue: the continuation is that turn's terminal path, so nothing is + // owed to the cut anymore, whatever the sweep that runs next reads from the handle. + session.clearQueue("queue cleared by user"); + expect(settleForfeited).toHaveBeenCalledTimes(1); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + hold[Symbol.dispose](); + await new Promise((resolve) => setTimeout(resolve, 25)); + finishCancel(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a late owner claim after the continuation already ran and ended still defers", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-late-claim"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + // The owner is behind its event lock and has not asked about assistant-1 when the resume + // starts, runs, and ends. + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().muxMetadata).toEqual(WORKSPACE_TURN_CORRELATION); + aiEmitter.emit("stream-end", { ...streamEndEvent(workspaceId), messageId: "assistant-2" }); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + + // The successor's own events settle the turn; the late claim on the cut must not settle it + // as failed first. The evidence is spent by that one claim. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a queued entry whose durable row outlives its failed startup supersedes the continuation", async () => { + const workspaceId = "queue-dispatch-stranded-durable-cutter"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage, historyService } = harness; + + try { + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + // An unrelated synthetic entry cuts the turn; its acceptance hook fails after its row is + // durable, so no stream of its own starts. + session.queueMessage( + "peer follow-up", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + onAccepted: () => { + throw new Error("acceptance exploded"); + }, + } + ); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 50)); + + // The cut turn must not run the follow-up's row as its own continuation. + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + const last = history.data[history.data.length - 1]; + expect(last?.role).toBe("user"); + expect( + last?.parts.some((part) => part.type === "text" && part.text === "peer follow-up") + ).toBe(true); + } + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("an older stream-end's claim does not void a newer cut's continuation", async () => { + const workspaceId = "queue-dispatch-stranded-delegated-stale-claim"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { muxMetadata: WORKSPACE_TURN_CORRELATION }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + harness.queueCancelableUnrelatedEntry(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + // The owner is still settling an earlier stream of this turn: that a later cut owes a + // continuation proves the turn went on, so it defers without touching the marker. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-0") + ).toBe(true); + session.clearQueue("queue cleared by user"); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a message queued behind a rejected resume drains", async () => { + const workspaceId = "queue-dispatch-stranded-rejected-drain"; + let rejectGate: () => void = () => undefined; + let gateArmed = false; + let gateReached = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => { + if (!gateArmed) { + return Promise.resolve(Ok(undefined)); + } + gateReached = true; + return new Promise((resolve) => { + rejectGate = () => resolve(Err({ type: "unknown", raw: "gate closed" })); + }); + }), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + } as unknown as WorkspaceGoalService; + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + gateArmed = true; + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => gateReached)).toBe(true); + + // WorkspaceService queues a send behind the busy (PREPARING) resume. + session.queueMessage("hello", { model: TEST_MODEL, agentId: "exec" }, { synthetic: true }); + gateArmed = false; + rejectGate(); + + // The rejected resume has no stream end to drain the queue at; the message must not wait + // for an unrelated later poke. + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + const dispatched = harness.latestRequest(); + const lastMessage = dispatched.messages[dispatched.messages.length - 1]; + expect(lastMessage?.parts.some((part) => part.type === "text" && part.text === "hello")).toBe( + true + ); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("the resume continues on the model that reached the cut", async () => { + const workspaceId = "queue-dispatch-stranded-fallback-model"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + // StreamManager reports the request that was running at the stop: a configured fallback + // model, not the refused primary this stream was sent with. + harness.latestRequest().onQueuedMessageStop?.(queuedStop("anthropic:claude-opus-4-8")); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().modelString).toBe("anthropic:claude-opus-4-8"); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a mid-turn thinking change carries into the resume", async () => { + const workspaceId = "queue-dispatch-stranded-thinking"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { thinkingLevel: "low" }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const original = harness.latestRequest(); + expect(original.thinkingLevel).toBe("low"); + // The user raised the level mid-turn and the loop applied it at a step boundary. + const holder = original.activeTurnThinkingOverride; + expect(holder).toBeDefined(); + if (holder != null) { + holder.applied = "high"; + } + + const wake = harness.queueCancelableWake(); + original.onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().thinkingLevel).toBe("high"); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a system hard stop during a pending provider-tool soft stop does not resume", async () => { + const workspaceId = "queue-dispatch-stranded-hard-system-stop"; + const harness = await createStreamingTurnHarness(workspaceId); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + + // task_stop / interrupt cascade: clears the queue and hard-stops through aiService directly, + // bypassing interruptStream, while the soft stop is still pending. + session.clearQueue("task stopped"); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + /** + * Strand a turn whose resume parks on its first pre-stream I/O (commitPartial) until the + * caller releases it, so an event can land while the resume is PREPARING with no stream + * registered for StreamManager to abort. + */ + async function strandWithResumeParkedInPreStreamIo(workspaceId: string) { + const harness = await createStreamingTurnHarness(workspaceId, { + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { historyService } = harness; + const originalCommitPartial = historyService.commitPartial.bind(historyService); + let releaseIo: () => void = () => undefined; + let ioReached = false; + const commitPartial = spyOn(historyService, "commitPartial").mockImplementation( + async (...args) => { + ioReached = true; + await new Promise((resolve) => { + releaseIo = resolve; + }); + return originalCommitPartial(...args); + } + ); + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + harness.aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => ioReached)).toBe(true); + expect(harness.session.isBusy()).toBe(true); + return { + ...harness, + releaseIo: () => releaseIo(), + restore: () => commitPartial.mockRestore(), + }; + } + + test("a goal transition during the resume's pre-stream I/O drops the resume", async () => { + const workspaceId = "queue-dispatch-stranded-goal-stale-in-io"; + let stale = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + buildGoalRedispatchAdmission: mock(() => + Promise.resolve({ admissible: true as const, admissionStale: () => stale }) + ), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + }); + const originalCommitPartial = historyService.commitPartial.bind(historyService); + let armed = false; + const commitPartial = spyOn(historyService, "commitPartial").mockImplementation((...args) => { + // The Pause lands after the resume's admission read, inside the stream's own pre-start I/O. + if (armed) { + stale = true; + } + return originalCommitPartial(...args); + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec" }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + armed = true; + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); + controller.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + commitPartial.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("disposing the session during the resume's pre-stream I/O cancels it", async () => { + const workspaceId = "queue-dispatch-stranded-dispose-in-io"; + const harness = await strandWithResumeParkedInPreStreamIo(workspaceId); + const { session, cleanup, streamMessage } = harness; + + try { + // Workspace removal tears the session down while the resume is past streamWithHistory's + // disposed check and StreamManager has no stream to stop. + session.dispose(); + harness.releaseIo(); + + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + harness.restore(); + await cleanup(); + } + }); + + test("a system hard stop with no registered stream cancels a preparing resume", async () => { + const workspaceId = "queue-dispatch-stranded-system-stop-in-io"; + const harness = await strandWithResumeParkedInPreStreamIo(workspaceId); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + // task_stop / interrupt cascade: clears the queue and hard-stops through aiService while + // the resume is still preparing, so StreamManager emits a synthetic pre-stream abort. + session.clearQueue("task stopped"); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + harness.releaseIo(); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + harness.restore(); + session.dispose(); + await cleanup(); + } + }); + + test("the resume keeps the live scratchpad snapshot the cut stream was sent with", async () => { + const workspaceId = "queue-dispatch-stranded-scratchpad"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { additionalSystemContext: "live scratchpad" }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + expect(harness.latestRequest().additionalSystemContext).toBe("live scratchpad"); + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + // Falling back to the persisted scratchpad mid-turn would change the model's instructions. + expect(harness.latestRequest().additionalSystemContext).toBe("live scratchpad"); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a thinking change still pending at the cut carries into the resume", async () => { + const workspaceId = "queue-dispatch-stranded-pending-thinking"; + const harness = await createStreamingTurnHarness(workspaceId, { + sendOptions: { thinkingLevel: "low" }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const original = harness.latestRequest(); + expect(original.thinkingLevel).toBe("low"); + // The user raised the level while a tool was running; the boundary was cut before any + // prepareStep could apply it. + expect(session.setActiveTurnThinkingLevel("high").accepted).toBe(true); + expect(original.activeTurnThinkingOverride?.applied).toBeUndefined(); + + const wake = harness.queueCancelableWake(); + original.onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + expect(await waitForCondition(() => streamMessage.mock.calls.length === 2)).toBe(true); + expect(harness.latestRequest().thinkingLevel).toBe("high"); + } finally { + session.dispose(); + await cleanup(); + } + }); + + /** Goal service double whose stream-end/abort drain can be parked by the test. */ + function createParkableGoalService() { + let releaseDrain: () => void = () => undefined; + let drainArmed = false; + let drainReached = false; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => { + if (!drainArmed) { + return Promise.resolve(); + } + drainReached = true; + return new Promise((resolve) => { + releaseDrain = resolve; + }); + }), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + return { + workspaceGoalService, + armDrain: () => { + drainArmed = true; + }, + drainReached: () => drainReached, + releaseDrain: () => releaseDrain(), + }; + } + + test("a hard stop landing during the soft-stop abort's cleanup does not resume", async () => { + const workspaceId = "queue-dispatch-stranded-soft-stop-cleanup-race"; + const goal = createParkableGoalService(); + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService: goal.workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, aiService, streamMessage } = harness; + const stopStream = spyOn(aiService, "stopStream").mockResolvedValue(Ok(undefined)); + + try { + session.queueMessage( + "follow up", + { model: TEST_MODEL, agentId: "exec" }, + { synthetic: true, dedupeKey: "wake:1" } + ); + aiEmitter.emit("tool-call-end", { + ...toolCallEndEvent(workspaceId), + toolName: "web_search", + providerExecuted: true, + }); + expect(stopStream).toHaveBeenCalledTimes(1); + + // The soft stop's abort handler samples its claim, then parks in its accounting awaits. + goal.armDrain(); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message")); + expect(await waitForCondition(() => goal.drainReached())).toBe(true); + + // task_stop lands in that window: queue cleared, synthetic system abort with no stream. + session.clearQueue("task stopped"); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + await new Promise((resolve) => setTimeout(resolve, 10)); + goal.releaseDrain(); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + stopStream.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("a task hard stop that finds the stream already completed withdraws the continuation", async () => { + const workspaceId = "queue-dispatch-stranded-hard-stop-completed"; + const goal = createParkableGoalService(); + const harness = await createStreamingTurnHarness(workspaceId, { + harness: { workspaceGoalService: goal.workspaceGoalService }, + sendInternal: { synthetic: true, agentInitiated: true }, + }); + const { session, cleanup, aiEmitter, streamMessage } = harness; + + try { + const wake = harness.queueCancelableWake(); + harness.latestRequest().onQueuedMessageStop?.(queuedStop()); + wake.abort("monitor consumed"); + // The loop ended normally; stream-end cleanup is parked in COMPLETING. + goal.armDrain(); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + expect(await waitForCondition(() => goal.drainReached())).toBe(true); + + // TaskService's hard stop: the queue clear is its only session-visible step, because + // stopStream finds a completed stream and emits no abort at all. + session.clearQueue("task stopped", { hardStop: true }); + goal.releaseDrain(); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a goal-refused resume settles the delegated turn it had advertised", async () => { + const workspaceId = "queue-dispatch-stranded-goal-refused-delegated"; + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + buildGoalRedispatchAdmission: mock(() => Promise.resolve({ admissible: false as const })), + recordStreamAccounting: mock(() => Promise.resolve()), + applyPendingAfterStreamEnd: mock(() => Promise.resolve()), + requestContinuationAfterStreamEnd: mock(() => Promise.resolve()), + recordStreamStarted: mock(() => Promise.resolve()), + syncGoalModeWithChatTail: mock(() => Promise.resolve(null)), + completeGoalFromSilentContinuation: mock(() => Promise.resolve(false)), + } as unknown as WorkspaceGoalService; + const settleForfeited = mock((_metadata: unknown, _reason: string) => Promise.resolve()); + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + return Promise.resolve(Ok(createStartedTurnHandle("assistant-1"))); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + workspaceGoalService, + settleForfeitedWorkspaceTurnContinuation: settleForfeited, + }); + + try { + await historyService.appendToHistory( + workspaceId, + createMuxMessage("user-0", "user", "keep going", { timestamp: Date.now() }) + ); + const resumed = await session.resumeStream( + { model: TEST_MODEL, agentId: "exec", muxMetadata: WORKSPACE_TURN_CORRELATION }, + { agentInitiated: true, goalKind: GOAL_CONTINUATION_KIND, goalId: "goal-1" } + ); + expect(resumed.success).toBe(true); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); + controller.abort("monitor consumed"); + session.clearQueue("monitor consumed"); + // The owner defers this stream-end on the strength of the advertised continuation. + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(true); + aiEmitter.emit("stream-end", streamEndEvent(workspaceId)); + + // The Pause refuses the resume: no successor stream will ever settle that deferral. + expect(await waitForCondition(() => settleForfeited.mock.calls.length === 1)).toBe(true); + expect(settleForfeited.mock.calls[0]?.[0]).toEqual(WORKSPACE_TURN_CORRELATION); + expect( + session.claimWorkspaceTurnContinuation(WORKSPACE_TURN_CORRELATION, "assistant-1") + ).toBe(false); + expect(streamMessage).toHaveBeenCalledTimes(1); + } finally { + session.dispose(); + await cleanup(); + } + }); + + test("a stream error after the stop decision discards the owed continuation", async () => { + const workspaceId = "queue-dispatch-stranded-stream-error"; + // The first stream's completion is settled by the test so its terminal processing can fail + // after the stop decision instead of ending the stream. + let failFirstStream: () => void = () => undefined; + const aiEmitter = new EventEmitter(); + const streamMessage = mock((_options: StreamMessageOptions) => { + aiEmitter.emit("stream-start", streamStartEvent(workspaceId)); + const completion = new Promise< + TurnStreamHandle["completion"] extends Promise ? T : never + >((resolve) => { + failFirstStream = () => + resolve({ + status: "failed", + streamError: { + messageId: "assistant-1", + error: "provider closed the connection", + errorType: "api", + }, + }); + }); + return Promise.resolve(Ok({ messageId: "assistant-1", completion })); + }); + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + aiEmitter, + aiServiceOverrides: { streamMessage: streamMessage as unknown as AIService["streamMessage"] }, + }); + + try { + const sent = await session.sendMessage("run the checks", { + model: TEST_MODEL, + agentId: "exec", + }); + expect(sent.success).toBe(true); + // Disabled after the send (a manual send re-enables it): the error is terminal, not retried. + await session.setAutoRetryEnabled(false, { persist: false }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-1", "assistant", "ran the checks", { timestamp: Date.now() }) + ); + const controller = new AbortController(); + session.queueMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState: { canceledBeforeAcceptance: false }, + cancelSignal: controller.signal, + onCanceled: () => undefined, + } + ); + streamMessage.mock.calls[0]?.[0].onQueuedMessageStop?.(queuedStop()); + controller.abort("monitor consumed"); + failFirstStream(); + + expect(await waitForCondition(() => !session.isBusy())).toBe(true); + // A later idle poke (queue cleared, admission block released) must not restart the failed turn. + session.drainQueuedMessagesIfIdle(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(streamMessage).toHaveBeenCalledTimes(1); + expect(session.isBusy()).toBe(false); + } finally { + session.dispose(); + await cleanup(); + } + }); + test("rollback failure preserves the wake and continues acceptance", async () => { + const workspaceId = "queue-dispatch-cancel-rollback-failure"; + const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); + const originalAppend = historyService.appendToHistory.bind(historyService); + let markAppendStarted: () => void = () => undefined; + const appendStarted = new Promise((resolve) => { + markAppendStarted = resolve; + }); + let releaseAppend: () => void = () => undefined; + const appendRelease = new Promise((resolve) => { + releaseAppend = resolve; + }); + const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( + async (...args) => { + markAppendStarted(); + await appendRelease; + return originalAppend(...args); + } + ); + const deleteMessagesSpy = spyOn(historyService, "deleteMessages").mockResolvedValue( + Err("injected rollback failure") + ); + + try { + const controller = new AbortController(); + const cancelState = { canceledBeforeAcceptance: false }; + const canceledReasons: string[] = []; + let accepted = false; + const sendPromise = session.sendMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState, + cancelSignal: controller.signal, + onCanceled: (reason) => { + canceledReasons.push(reason); + }, + onAccepted: () => { + accepted = true; + }, + } + ); + + await appendStarted; + controller.abort("monitor canceled"); + releaseAppend(); + const result = await sendPromise; + + expect(result.success).toBe(true); + expect(deleteMessagesSpy).toHaveBeenCalledTimes(1); + expect(canceledReasons).toEqual([]); + expect(cancelState.canceledBeforeAcceptance).toBe(false); + expect(accepted).toBe(true); + + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect( + history.data.some((message) => + message.parts.some( + (part) => part.type === "text" && part.text === "Background monitor wake" + ) + ) + ).toBe(true); + } + } finally { + releaseAppend(); + deleteMessagesSpy.mockRestore(); + appendSpy.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("verifies a committed rollback when batch deletion reports a post-write failure", async () => { + const workspaceId = "queue-dispatch-cancel-post-write-failure"; + const { session, cleanup, historyService } = await createAgentSessionHarness({ workspaceId }); + const originalAppend = historyService.appendToHistory.bind(historyService); + const originalDeleteMessages = historyService.deleteMessages.bind(historyService); + let markAppendStarted: () => void = () => undefined; + const appendStarted = new Promise((resolve) => { + markAppendStarted = resolve; + }); + let releaseAppend: () => void = () => undefined; + const appendRelease = new Promise((resolve) => { + releaseAppend = resolve; + }); + const appendSpy = spyOn(historyService, "appendToHistory").mockImplementation( + async (...args) => { + markAppendStarted(); + await appendRelease; + return originalAppend(...args); + } + ); + const deleteMessagesSpy = spyOn(historyService, "deleteMessages").mockImplementation( + async (...args) => { + const result = await originalDeleteMessages(...args); + expect(result.success).toBe(true); + return Err("injected post-write failure"); + } + ); + + try { + const controller = new AbortController(); + const cancelState = { canceledBeforeAcceptance: false }; + const canceledReasons: string[] = []; + let accepted = false; + const sendPromise = session.sendMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState, + cancelSignal: controller.signal, + onCanceled: (reason) => { + canceledReasons.push(reason); + }, + onAccepted: () => { + accepted = true; + }, + } + ); + + await appendStarted; + controller.abort("monitor canceled"); + releaseAppend(); + const result = await sendPromise; + + expect(result.success).toBe(true); + expect(deleteMessagesSpy).toHaveBeenCalledTimes(1); + expect(canceledReasons).toEqual(["monitor canceled"]); + expect(cancelState.canceledBeforeAcceptance).toBe(true); + expect(accepted).toBe(false); + + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect( + history.data.some((message) => + message.parts.some( + (part) => part.type === "text" && part.text === "Background monitor wake" + ) + ) + ).toBe(false); + } + } finally { + releaseAppend(); + deleteMessagesSpy.mockRestore(); + appendSpy.mockRestore(); + session.dispose(); + await cleanup(); + } + }); + + test("cancellation during goal sync crosses the acceptance point of no return", async () => { + const workspaceId = "queue-dispatch-cancel-goal-reconcile"; + let markInitialSyncStarted: () => void = () => undefined; + const initialSyncStarted = new Promise((resolve) => { + markInitialSyncStarted = resolve; + }); + let releaseInitialSync: () => void = () => undefined; + const initialSyncRelease = new Promise((resolve) => { + releaseInitialSync = resolve; + }); + let syncCalls = 0; + const syncGoalModeWithChatTail = mock(async () => { + syncCalls += 1; + if (syncCalls === 1) { + markInitialSyncStarted(); + await initialSyncRelease; + } + return null; + }); + const workspaceGoalService = { + assertPricedModelForBudgetedGoal: mock(() => Promise.resolve(Ok(undefined))), + syncGoalModeWithChatTail, + } as unknown as WorkspaceGoalService; + const { session, cleanup, historyService } = await createAgentSessionHarness({ + workspaceId, + workspaceGoalService, + }); + + try { + const controller = new AbortController(); + const cancelState = { canceledBeforeAcceptance: false }; + const canceledReasons: string[] = []; + let accepted = false; + const sendPromise = session.sendMessage( + "Background monitor wake", + { model: TEST_MODEL, agentId: "exec" }, + { + synthetic: true, + agentInitiated: true, + cancelState, + cancelSignal: controller.signal, + onCanceled: (reason) => { + canceledReasons.push(reason); + }, + onAccepted: () => { + accepted = true; + }, + } + ); + + await initialSyncStarted; + controller.abort("monitor canceled"); + releaseInitialSync(); + const result = await sendPromise; + + expect(result.success).toBe(true); + expect(syncGoalModeWithChatTail).toHaveBeenCalledTimes(1); + expect(canceledReasons).toEqual([]); + expect(cancelState.canceledBeforeAcceptance).toBe(false); + expect(accepted).toBe(true); + + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect( + history.data.some((message) => + message.parts.some( + (part) => part.type === "text" && part.text === "Background monitor wake" + ) + ) + ).toBe(true); + } + } finally { + releaseInitialSync(); + session.dispose(); + await cleanup(); + } + }); + + test("disposed sessions finalize durable wakes after goal sync completes", async () => { + const workspaceId = "queue-dispatch-disposed-after-goal-sync"; + let markSyncStarted: () => void = () => undefined; + const syncStarted = new Promise((resolve) => { + markSyncStarted = resolve; + }); + let releaseSync: () => void = () => undefined; + const syncRelease = new Promise((resolve) => { + releaseSync = resolve; + }); + const syncGoalModeWithChatTail = mock(async () => { + markSyncStarted(); + await syncRelease; return null; }); const workspaceGoalService = { @@ -1181,7 +4096,7 @@ describe("AgentSession queued message tool-call dispatch", () => { const interruptResult = await session.interruptStream(); expect(interruptResult.success).toBe(true); // The native soft-stop can still win the event race after the hard user interrupt. - aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "system")); + aiEmitter.emit("stream-abort", streamAbortEvent(workspaceId, "queued-message")); await new Promise((resolve) => setTimeout(resolve, 25)); expect(sendQueuedMessages).not.toHaveBeenCalled(); diff --git a/src/node/services/agentSession.startupAutoRetry.test.ts b/src/node/services/agentSession.startupAutoRetry.test.ts index 42ee1ea7b5..7c09b128e3 100644 --- a/src/node/services/agentSession.startupAutoRetry.test.ts +++ b/src/node/services/agentSession.startupAutoRetry.test.ts @@ -376,6 +376,75 @@ describe("AgentSession startup auto-retry recovery", () => { session.dispose(); }); + test("startup auto-retry runs a queued-cut row under the remainder it persisted", async () => { + const workspaceId = "startup-retry-cut-step-budget"; + const { session, historyService, cleanup } = await createSessionBundle(workspaceId); + cleanups.push(cleanup); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("original-user", "user", "Continue the original task", { + timestamp: Date.now(), + }) + ); + // The soft-aborted partial was committed with the cut turn's remainder, then the process + // exited before the in-memory resume started. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-cut", "assistant", "Interrupted response", { + timestamp: Date.now(), + partial: true, + stepsRemaining: 3, + }) + ); + + session.ensureStartupAutoRetryCheck(); + await (session as unknown as { startupAutoRetryCheckPromise: Promise | null }) + .startupAutoRetryCheckPromise; + + const retryRequest = ( + session as unknown as { lastAutoRetryResumeRequest?: { stepBudget?: number } } + ).lastAutoRetryResumeRequest; + expect(retryRequest?.stepBudget).toBe(3); + session.dispose(); + }); + + test("startup auto-retry fails closed on a malformed persisted remainder", async () => { + const workspaceId = "startup-retry-cut-step-budget-malformed"; + const { session, historyService, events, cleanup } = await createSessionBundle(workspaceId); + cleanups.push(cleanup); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("original-user", "user", "Continue the original task", { + timestamp: Date.now(), + }) + ); + // Raw chat.jsonl is not schema-checked on this path; a corrupt remainder must not read as + // absent and hand the cut turn the default ceiling. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-cut", "assistant", "Interrupted response", { + timestamp: Date.now(), + partial: true, + stepsRemaining: { steps: 3 } as unknown as number, + }) + ); + + session.ensureStartupAutoRetryCheck(); + await (session as unknown as { startupAutoRetryCheckPromise: Promise | null }) + .startupAutoRetryCheckPromise; + + expect(events.find((event) => event.type === "auto-retry-abandoned")).toMatchObject({ + reason: "malformed_step_budget", + }); + expect(events.some((event) => event.type === "auto-retry-scheduled")).toBe(false); + expect( + (session as unknown as { lastAutoRetryResumeRequest?: unknown }).lastAutoRetryResumeRequest + ).toBeUndefined(); + session.dispose(); + }); + test("hidden completed subagent reports preserve the existing startup retry fallback", async () => { const workspaceId = "startup-retry-hidden-subagent-report"; const { session, historyService, events, cleanup } = await createSessionBundle(workspaceId); diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index fac756ca28..86ad42d1a4 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -5,7 +5,11 @@ import type { WorkspaceChatMessage } from "@/common/orpc/types"; import { Err, Ok } from "@/common/types/result"; import type { Config } from "@/node/config"; import type { TurnStreamHandle } from "@/node/services/streamManager"; -import { AgentSession, type AgentSessionAIService } from "@/node/services/agentSession"; +import { + AgentSession, + type AgentSessionAIService, + type AgentSessionOptions, +} from "@/node/services/agentSession"; import type { CompactionCompletionMetadata } from "@/common/types/compaction"; import type { BackgroundProcessManager } from "@/node/services/backgroundProcessManager"; import type { WorkspaceGoalService } from "@/node/services/workspaceGoalService"; @@ -110,6 +114,9 @@ export interface AgentSessionHarnessOptions { workspaceGoalService?: WorkspaceGoalService; mcpServerManager?: MCPServerManager; onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + hasExternalSendPreflight?: () => boolean; + settleForfeitedWorkspaceTurnContinuation?: AgentSessionOptions["settleForfeitedWorkspaceTurnContinuation"]; + admitStrandedTurnResume?: AgentSessionOptions["admitStrandedTurnResume"]; captureEvents?: boolean; } @@ -154,6 +161,9 @@ export async function createAgentSessionHarness( workspaceGoalService: options.workspaceGoalService, backgroundProcessManager, onCompactionComplete: options.onCompactionComplete, + hasExternalSendPreflight: options.hasExternalSendPreflight, + settleForfeitedWorkspaceTurnContinuation: options.settleForfeitedWorkspaceTurnContinuation, + admitStrandedTurnResume: options.admitStrandedTurnResume, }); const events: WorkspaceChatMessage[] = []; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 4ea8bac14a..26c70ce3ce 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -45,7 +45,7 @@ import { SendMessageOptionsSchema, SkillNameSchema, } from "@/common/orpc/schemas"; -import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; +import { ModelFallbackProgressSchema, ToolPolicySchema } from "@/common/orpc/schemas/stream"; import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds"; import { isWorkspaceArchived } from "@/common/utils/archive"; import { findWorkspaceEntry } from "@/node/services/taskUtils"; @@ -103,6 +103,7 @@ import { type MuxMessageMetadata, type MuxFilePart, type MuxMessage, + parseWorkspaceTurnTaskCorrelation, type ReviewNoteDataForDisplay, type StartupRetrySendOptions, } from "@/common/types/message"; @@ -116,9 +117,10 @@ import { createRuntimeForWorkspace, } from "@/node/runtime/runtimeHelpers"; import { MessageQueue, cancelReasonBeforeAcceptance } from "./messageQueue"; -import type { QueueCutCutter } from "./messageQueue"; +import type { QueueClearCallbacks, QueueCutCutter } from "./messageQueue"; import { copyStreamLifecycleSnapshot, + type ModelFallbackProgress, type RuntimeStatusEvent, type StreamAbortReason, type StreamEndEvent, @@ -237,6 +239,17 @@ interface CompactionRequestMetadata { type GoalInterventionPolicy = NonNullable; +/** Consumed continuation cuts kept for a late owner claim (see consumedContinuationCuts). */ +const MAX_RETAINED_CONTINUATION_CUTS = 8; + +interface OwedForfeitSettlement { + key: string; + correlation: WorkspaceTurnMuxMetadata; + reason: string; + /** The attempt under way; resolves true once the owner has the record, false when it failed. */ + inFlight?: Promise; +} + interface AutoRetryResumeRequest { // Same-session auto-retry must preserve the full normalized request because // ACP correlation/delegation lives in transient send options that are @@ -246,6 +259,16 @@ interface AutoRetryResumeRequest { goalKind?: GoalSyntheticMessageKind; /** Goal identity matching goalKind; keeps retried streams goal-scoped. */ goalId?: string; + stepBudget?: number; + /** The retried stream was admitted under resumeStream's revalidation; the retry repeats it. */ + revalidateAdmission?: boolean; + modelFallbackProgress?: ModelFallbackProgress; + /** + * Delegated turn the retried stream continues when `options.muxMetadata` does not carry it (a + * bash-monitor wake inherits its correlation from history), so revalidation still reaches the + * owner's handle. + */ + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; } function stripGoalInterventionPolicy(options: SendMessageOptions): SendMessageOptions { @@ -367,6 +390,89 @@ function getWorkspaceTurnMuxMetadata(muxMetadata: unknown): WorkspaceTurnMuxMeta return metadata?.type === "workspace-turn-task" ? metadata : undefined; } +interface StrandedTurnResume { + options: SendMessageOptions; + /** Assistant message id of the stream the cut ended (claimWorkspaceTurnContinuation). */ + cutMessageId: string; + /** The owner already deferred on this cut's stream-end; no late claim is coming for it. */ + claimed?: boolean; + /** + * Steps the cut stream had left under its ceiling; the resumed stream runs under this budget + * so a chain of cuts and resumes spends one turn's steps, not a fresh cap per resume. + */ + stepBudget?: number; + /** + * Fallback chain the cut stream ran under, with the refusals so far: the resumed stream continues + * it, rather than the chain its own (possibly fallback) model would resolve. + */ + modelFallbackProgress?: ModelFallbackProgress; + agentInitiated?: boolean; + goalKind?: GoalSyntheticMessageKind; + goalId?: string; +} + +/** + * Continuation for a turn stranded by a withdrawn queued message: the interrupted stream's + * request configuration (the same retry-safe whitelist startup recovery resumes with) and goal + * attribution. Anything that replays the original dispatch is dropped: edit target, queue + * dispatch mode, per-message metadata (compaction, skill, wake), and the ACP prompt binding, + * whose turn completion the first stream-end already resolved, so a delegated tool call on the + * resumed stream would wait on a prompt nobody answers. Only the workspace-turn correlation + * survives, taken from the resolved stream context because a wake-started continuation + * inherits it from history rather than from its own send options. + */ +function buildStrandedTurnResume(context: { + /** Model that reached the cut: a configured fallback may differ from the requested one. */ + modelString: string; + cutMessageId: string; + stepBudget?: number; + modelFallbackProgress?: ModelFallbackProgress; + options?: SendMessageOptions; + agentInitiated?: boolean; + goalKind?: GoalSyntheticMessageKind; + goalId?: string; + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; + /** + * Level a mid-turn thinking change left in effect at the cut: still pending when the boundary + * was cut during a tool call (no prepareStep consumed it), else the last applied one. The + * request's own level is stale then; streamWithHistory re-clamps against the model floor. + */ + thinkingLevelAtCut?: ThinkingLevel; +}): StrandedTurnResume { + const resumeOptions = pickStartupRetrySendOptions( + context.options ?? { model: context.modelString, agentId: WORKSPACE_DEFAULTS.agentId } + ); + return { + options: { + ...resumeOptions, + // Not durable retry state (the picker omits it), but this continuation is the same turn in + // memory: the resumed stream must see the live scratchpad snapshot the cut stream did, not + // whatever the renderer's save had persisted by then. + ...(context.options?.additionalSystemContext != null + ? { additionalSystemContext: context.options.additionalSystemContext } + : {}), + model: context.modelString, + ...(context.thinkingLevelAtCut != null ? { thinkingLevel: context.thinkingLevelAtCut } : {}), + muxMetadata: + context.workspaceTurnMetadata ?? getWorkspaceTurnMuxMetadata(context.options?.muxMetadata), + }, + cutMessageId: context.cutMessageId, + ...(context.stepBudget != null ? { stepBudget: context.stepBudget } : {}), + ...(context.modelFallbackProgress != null + ? { modelFallbackProgress: context.modelFallbackProgress } + : {}), + ...(context.agentInitiated != null ? { agentInitiated: context.agentInitiated } : {}), + ...(context.goalKind != null ? { goalKind: context.goalKind } : {}), + ...(context.goalId != null ? { goalId: context.goalId } : {}), + }; +} + +/** A persisted correlation (unchecked chat.jsonl) is used only when well formed. */ +function parsePersistedWorkspaceTurnMetadata(value: unknown): WorkspaceTurnMuxMetadata | undefined { + const correlation = parseWorkspaceTurnTaskCorrelation(value); + return correlation == null ? undefined : { type: "workspace-turn-task", ...correlation }; +} + function hasSameWorkspaceTurnCorrelation( first: WorkspaceTurnMuxMetadata | undefined, second: WorkspaceTurnMuxMetadata | undefined @@ -521,6 +627,15 @@ const SESSION_SHUTDOWN_SEND_BLOCKED_MESSAGE = "Xum is shutting down; the message const STARTUP_AUTO_RETRY_HISTORY_FAILURE_BASE_DELAY_MS = 1_000; const STARTUP_AUTO_RETRY_HISTORY_FAILURE_MAX_DELAY_MS = 30_000; const MAX_STARTUP_RECOVERY_DEFERRED_ATTEMPTS = 4; +/** + * Retry bound for a stranded resume that keeps failing before its stream starts (pricing gate, + * history read, refused admission): after this many attempts with no stream the marker is + * dropped. Resumes that do start are not counted; each later stranding follows a completed + * model step of real work and is a fresh obligation, so a legitimate turn that awaits several + * monitored processes in a row resumes every time. + */ +const MAX_CONSECUTIVE_STRANDED_TURN_RESUMES = 3; +const FORFEIT_SETTLEMENT_RETRY_DELAY_MS = 1_000; export interface AgentSessionChatEvent { workspaceId: string; @@ -584,7 +699,7 @@ export interface AgentSessionAIService extends BranchSummaryAiService { ): XumToolScope; } -interface AgentSessionOptions { +export interface AgentSessionOptions { workspaceId: string; config: Config; historyService: HistoryService; @@ -626,6 +741,23 @@ interface AgentSessionOptions { * to yield to a manual send that is still awaiting pricing/settings. */ hasExternalSendPreflight?: () => boolean; + /** + * Settles a delegated workspace turn whose owner deferred its stream-end because this session + * advertised an owed continuation that is now forfeited without a successor stream (goal no + * longer admits it, resume retry cap): no later stream-end will arrive for that turn. + */ + settleForfeitedWorkspaceTurnContinuation?: ( + metadata: WorkspaceTurnMuxMetadata, + reason: string + ) => Promise; + /** + * Admission for a stranded resume, read once the turn is claimed: the workspace still accepts + * streams (not being removed or archived) and, for a delegated turn, its owner still has the + * turn running. The probe reports a stop that lands after the read (workspace stop epoch). + */ + admitStrandedTurnResume?: ( + correlation: WorkspaceTurnMuxMetadata | undefined + ) => Promise<{ admissible: boolean; admissionStale?: () => boolean }>; } enum TurnPhase { @@ -667,6 +799,8 @@ export class AgentSession { private readonly sanitizeCliWorkspaceRegistration?: AgentSessionOptions["sanitizeCliWorkspaceRegistration"]; private readonly onPostCompactionStateChange?: () => void; private readonly hasExternalSendPreflight?: () => boolean; + private readonly settleForfeitedWorkspaceTurnContinuation?: AgentSessionOptions["settleForfeitedWorkspaceTurnContinuation"]; + private readonly admitStrandedTurnResume?: AgentSessionOptions["admitStrandedTurnResume"]; private readonly emitter = new EventEmitter(); private readonly aiListeners: Array<{ event: string; handler: (...args: unknown[]) => void }> = []; @@ -699,6 +833,26 @@ export class AgentSession { // Track known siblings and reserve soft interruption for that native-only boundary. private queuedProviderToolEndAbortInFlight = false; private readonly activeToolCallIds = new Set(); + // The model loop stops at a tool boundary on behalf of a queued tool-end message. If that + // message never starts a turn (canceled wake, cleared queue, pre-stream failure), the stop + // would strand the turn on an unanswered tool result, so the continuation is owed here from + // the moment the stop is decided until a stream actually starts (setTurnPhase STREAMING). + // Recorded synchronously so the delegating owner's stream-end settlement can see it. + private strandedTurnResume?: StrandedTurnResume; + // Set while the sweep's resume is running (claim through stream end); aborting it cancels only + // the pre-stream window, since StreamManager unlinks the signal once a stream registers. + private strandedTurnResumeInFlight: AbortController | null = null; + /** + * Cuts of delegated turns whose continuation already ran, keyed by the cut stream's message id. + * The owner settles a correlated tool-calls stream-end under its own event lock, so it can ask + * about a cut after the successor stream has come and gone; the successor's own events settle + * the turn, and this evidence keeps the late claim from settling it as failed first. + */ + private readonly consumedContinuationCuts = new Map(); + /** Owner settlements for forfeited continuations that have not landed yet (settleOwedForfeit). */ + private readonly owedForfeitSettlements = new Map(); + private owedForfeitSettlementRetryTimer: ReturnType | null = null; + private consecutiveStrandedResumes = 0; private idleWaiters: Array<() => void> = []; private pendingExternalManualFollowUps = 0; @@ -782,6 +936,8 @@ export class AgentSession { /** Tracks the user message id that initiated the currently active stream (for retry guards). */ private activeStreamUserMessageId?: string; + /** Assistant message id of the active stream, from its stream-start event. */ + private activeStreamMessageId?: string; /** Track user message ids that already retried without post-compaction injection. */ private readonly postCompactionRetryAttempts = new Set(); @@ -868,6 +1024,12 @@ export class AgentSession { goalKind?: GoalSyntheticMessageKind; /** Goal identity matching goalKind, so mid-stream compaction follow-ups stay goal-scoped. */ goalId?: string; + /** Step ceiling this stream runs under when it continues a cut turn (see StrandedTurnResume). */ + stepBudget?: number; + /** Fallback chain this stream continues when it continues a cut turn (see StrandedTurnResume). */ + modelFallbackProgress?: ModelFallbackProgress; + /** Admitted under resumeStream's revalidation; in-session retries of this stream repeat it. */ + revalidateAdmission?: boolean; workspaceTurnMetadata?: Extract; }; @@ -907,6 +1069,8 @@ export class AgentSession { onIdleCompactionOutcome, onPostCompactionStateChange, hasExternalSendPreflight, + settleForfeitedWorkspaceTurnContinuation, + admitStrandedTurnResume, } = options; assert(typeof workspaceId === "string", "workspaceId must be a string"); @@ -935,6 +1099,8 @@ export class AgentSession { this.sanitizeCliWorkspaceRegistration = sanitizeCliWorkspaceRegistration; this.onPostCompactionStateChange = onPostCompactionStateChange; this.hasExternalSendPreflight = hasExternalSendPreflight; + this.settleForfeitedWorkspaceTurnContinuation = settleForfeitedWorkspaceTurnContinuation; + this.admitStrandedTurnResume = admitStrandedTurnResume; this.compactionHandler = new CompactionHandler({ workspaceId: this.workspaceId, @@ -991,6 +1157,10 @@ export class AgentSession { this.activePreparedTurnAbortController?.abort(); this.activePreparedTurnAbortController = null; + // A resume parked in its pre-stream I/O passed streamWithHistory's disposed check already; + // its abort signal is the only thing that stops it registering a stream after teardown. + // Disposal (workspace removal) also produces no successor stream for a deferred delegated turn. + this.forfeitStrandedTurnResume("Stranded turn resume discarded: workspace session disposed."); // Ensure any callers blocked on waitForIdle() can continue during teardown. this.setTurnPhase(TurnPhase.IDLE); @@ -1295,7 +1465,11 @@ export class AgentSession { options: SendMessageOptions | undefined, agentInitiated?: boolean, goalKind?: GoalSyntheticMessageKind, - goalId?: string + goalId?: string, + stepBudget?: number, + revalidateAdmission?: boolean, + modelFallbackProgress?: ModelFallbackProgress, + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata ): void { if (!options) { this.lastAutoRetryResumeRequest = undefined; @@ -1307,6 +1481,10 @@ export class AgentSession { ...(agentInitiated === true ? { agentInitiated: true } : {}), ...(goalKind != null ? { goalKind } : {}), ...(goalId != null ? { goalId } : {}), + ...(stepBudget != null ? { stepBudget } : {}), + ...(revalidateAdmission === true ? { revalidateAdmission: true } : {}), + ...(modelFallbackProgress != null ? { modelFallbackProgress } : {}), + ...(workspaceTurnMetadata != null ? { workspaceTurnMetadata } : {}), }; } @@ -1330,6 +1508,11 @@ export class AgentSession { this.emitRetryEvent({ type: "auto-retry-abandoned", reason: "missing_retry_options" }); return; } + if (request.stepBudget != null && request.stepBudget <= 0) { + // The failed attempt spent the turn's last step; the ceiling ends the turn here. + this.emitRetryEvent({ type: "auto-retry-abandoned", reason: "step_budget_spent" }); + return; + } // Archive only interrupts a live stream; a backoff timer armed before it keeps ticking. if (this.isWorkspaceArchivedOnDisk()) { @@ -1341,8 +1524,20 @@ export class AgentSession { agentInitiated: request.agentInitiated === true ? true : undefined, goalKind: request.goalKind, goalId: request.goalId, + stepBudget: request.stepBudget, + revalidateAdmission: request.revalidateAdmission, + modelFallbackProgress: request.modelFallbackProgress, + workspaceTurnMetadata: request.workspaceTurnMetadata, }); if (result.success) { + if (result.data.refusedBy != null) { + // The goal or delegated turn ended during the backoff; no retry can readmit it. + this.emitRetryEvent({ + type: "auto-retry-abandoned", + reason: `${result.data.refusedBy}_admission_refused`, + }); + return; + } if (!result.data.started) { // resumeStream can defer when a turn is still PREPARING/COMPLETING. // Treat this as retriable so auto-retry keeps progressing instead of @@ -1573,11 +1768,11 @@ export class AgentSession { abortReason: StreamAbortReason | undefined, userMessageId?: string ): Promise { - // "system" and "startup" aborts come from backend-orchestrated flows - // (for example, mid-stream auto-compaction or canceling a pending startup). - // They are not user intent and must not poison startup recovery with a - // persisted non-retryable "aborted" marker. - if (abortReason === "system" || abortReason === "startup") { + // "system", "startup", and "queued-message" aborts come from backend-orchestrated flows + // (for example, mid-stream auto-compaction, canceling a pending startup, or the soft stop + // for a queued message). They are not user intent and must not poison startup recovery + // with a persisted non-retryable "aborted" marker. + if (abortReason === "system" || abortReason === "startup" || abortReason === "queued-message") { return; } @@ -2326,7 +2521,37 @@ export class AgentSession { } const { agentInitiated, goalKind, goalId, ...resumeOptions } = retryRequest; - this.setAutoRetryResumeState(resumeOptions, agentInitiated, goalKind, goalId); + // A row cut for a queued message carries what the turn had left of its ceiling; the retry + // continues that turn, so it runs under the remainder (and is abandoned at zero) rather + // than a fresh ceiling. + const interruptedAssistant = + partial?.role === "assistant" + ? partial + : lastHistoryMessage?.role === "assistant" + ? lastHistoryMessage + : undefined; + // Raw JSON boundary, like the pending follow-up's persisted budget: a present but malformed + // remainder fails closed. The row can no longer state the ceiling its turn ran under, and + // reading it as absent would hand the turn the default one. + const persistedStepsRemaining = interruptedAssistant?.metadata?.stepsRemaining; + if ( + persistedStepsRemaining !== undefined && + !(Number.isInteger(persistedStepsRemaining) && persistedStepsRemaining >= 0) + ) { + log.warn("Startup auto-retry abandoned: malformed persisted step budget", { + workspaceId: this.workspaceId, + messageId: interruptedAssistant?.id, + }); + this.emitRetryEvent({ type: "auto-retry-abandoned", reason: "malformed_step_budget" }); + return "completed"; + } + this.setAutoRetryResumeState( + resumeOptions, + agentInitiated, + goalKind, + goalId, + persistedStepsRemaining + ); } // Disk reads above may race with user actions; retry once the current work settles @@ -3050,6 +3275,20 @@ export class AgentSession { /** Goal identity persisted alongside goalKind so chat-tail reconciliation can scope the row. */ goalId?: string; startStreamInBackground?: boolean; + /** For a send continuing an interrupted turn: what that turn had left of its ceiling. */ + stepBudget?: number; + /** For a send continuing an interrupted turn: the fallback chain state it reached. */ + modelFallbackProgress?: ModelFallbackProgress; + /** For a send continuing an interrupted turn: it ran under resumeStream's revalidation. */ + revalidateAdmission?: boolean; + /** For a send continuing a delegated turn its `muxMetadata` does not carry (a wake). */ + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; + /** + * Launch-boundary admission probe (goal and delegated-turn state), rechecked by + * StreamManager right before the stream registers; unlike admissionStale it must not + * observe this send's own turn. + */ + refuseStreamStart?: () => boolean; onAccepted?: () => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; onCanceled?: (reason: string) => Promise | void; @@ -3156,6 +3395,31 @@ export class AgentSession { ) ); }; + const rollbackRefusedLaunchRows = async (): Promise => { + const historyResult = await this.historyService.getHistoryFromLatestBoundary( + this.workspaceId + ); + const historySequences = historyResult.success + ? historyResult.data + .filter((message) => persistedCancelableMessageIds.includes(message.id)) + .map((message) => message.metadata?.historySequence) + .filter((sequence): sequence is number => isNonNegativeInteger(sequence)) + : []; + if (!(await rollbackPersistedTurnRows())) { + return; + } + if (historySequences.length > 0) { + this.emitChatEvent({ type: "delete", historySequences }); + } + try { + await this.workspaceGoalService?.syncGoalModeWithChatTail(this.workspaceId); + } catch (error) { + log.warn("Failed to resync goal state after a refused launch", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + } + }; let cancellationHandled = false; let cancellationDisabled = false; const cancelBeforeAcceptance = async (): Promise => { @@ -3948,6 +4212,8 @@ export class AgentSession { // leaves no trace for a later human resume to replay into provider context. Past this point // rollback is forbidden by design (goal sync observes the durable row), so a Stop landing in // the remaining pre-stream awaits refuses the turn at the PREPARING gate with rows retained. + // The one exception is a synthetic continuation refused by its own launch probe + // (rollbackRefusedLaunchRows), which re-derives goal state after removing its rows. if (internal?.admissionStale?.() === true) { const rolledBack = await rollbackPersistedTurnRows(); // Probe-carrying sends are peer messages whose caller already returned success when the @@ -3979,6 +4245,14 @@ export class AgentSession { if (cancelSignal != null) { cancellationDisabled = true; } + // The durable row of anything but a continuation of the cut turn (a wake, or the same + // delegated turn) supersedes the owed continuation from here on: a failure below leaves the + // row in place, and the cut turn must not resume over it with its own identity. + if (!this.continuesOwedTurn(typedMuxMetadata)) { + this.forfeitStrandedTurnResume( + "Stranded turn resume dropped: superseded by a later durable message." + ); + } // r54: the pre-turn batch is now irrevocable — rollbackPersistedTurnRows // is never invoked past this point, so even a failure in goal sync or // acceptance leaves the payload + trigger rows durable in the transcript. @@ -4071,7 +4345,16 @@ export class AgentSession { // Same-session retry should resume the exact accepted request we just finalized // in history, even if runtime warmup fails before streamWithHistory() starts. - this.setAutoRetryResumeState(optionsForStream, agentInitiated, goalKind, internal?.goalId); + this.setAutoRetryResumeState( + optionsForStream, + agentInitiated, + goalKind, + internal?.goalId, + internal?.stepBudget, + internal?.revalidateAdmission, + internal?.modelFallbackProgress, + internal?.workspaceTurnMetadata + ); try { await internal?.onAccepted?.(); } catch (error) { @@ -4155,7 +4438,11 @@ export class AgentSession { preparedTurnAbortController.signal, goalKind, internal?.goalId, - turnThinkingOverride + turnThinkingOverride, + internal?.refuseStreamStart, + internal?.stepBudget, + internal?.revalidateAdmission, + internal?.modelFallbackProgress ); if (streamResult.success && preparedTurnAbortController.signal.aborted) { await notifyAcceptedPreStreamFailure( @@ -4164,6 +4451,19 @@ export class AgentSession { ) ); } + // The send's own launch probe refused the turn at the boundary (Ok with the turn still + // PREPARING, nothing registered): the goal or delegated turn this synthetic continuation + // served ended during startup. Its rows would otherwise sit at the tail as a prompt the + // next unrelated request replays, so they go with the launch; goal state re-derives from + // the tail they leave (the one rollback past goal sync, see the horizon note above). + if ( + streamResult.success && + this.turnPhase === TurnPhase.PREPARING && + !hasPreTurnMessages && + internal?.refuseStreamStart?.() === true + ) { + await rollbackRefusedLaunchRows(); + } return streamResult; } finally { // Success should advance via stream events; if startup never emitted any, don't leave the @@ -4224,8 +4524,26 @@ export class AgentSession { async resumeStream( options: SendMessageOptions, - internal?: { agentInitiated?: boolean; goalKind?: GoalSyntheticMessageKind; goalId?: string } - ): Promise> { + internal?: { + agentInitiated?: boolean; + goalKind?: GoalSyntheticMessageKind; + goalId?: string; + /** Cancels the pre-stream window (admission gates, history reads) once the resume is withdrawn. */ + abortSignal?: AbortSignal; + /** + * Revalidate the resume once the turn is claimed: a goal turn against durable goal state + * (the same veto durable goal redispatches apply) and the workspace plus delegated turn + * through admitStrandedTurnResume; a refusal reports `refusedBy`. + */ + revalidateAdmission?: boolean; + /** Step ceiling for the resumed stream when it continues a cut turn (StrandedTurnResume). */ + stepBudget?: number; + /** Fallback chain the resumed stream continues when it continues a cut turn. */ + modelFallbackProgress?: ModelFallbackProgress; + /** Delegated turn to revalidate against when `options.muxMetadata` does not carry it. */ + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; + } + ): Promise> { this.assertNotDisposed("resumeStream"); assert(options, "resumeStream requires options"); @@ -4242,16 +4560,6 @@ export class AgentSession { return Ok({ started: false }); } - if (this.workspaceGoalService) { - const pricingGate = await this.workspaceGoalService.assertPricedModelForBudgetedGoal( - this.workspaceId, - modelForStream - ); - if (!pricingGate.success) { - return Err(pricingGate.error); - } - } - // r40: refuse resume admission while a context-discarding mutation is // mid-flight (see holdTurnAdmission) — checked in the same synchronous // block that sets PREPARING. A non-started resume reads as retriable to @@ -4269,8 +4577,15 @@ export class AgentSession { optionsForStream, internal?.agentInitiated, internal?.goalKind, - internal?.goalId + internal?.goalId, + internal?.stepBudget, + internal?.revalidateAdmission, + internal?.modelFallbackProgress, + internal?.workspaceTurnMetadata ); + // Claim the turn before any await: the admission gates below do I/O, and a manual send + // entering meanwhile must see a busy session rather than start a stream this resume + // would then collide with. this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(optionsForStream.muxMetadata); this.setTurnPhase(TurnPhase.PREPARING); // Open the mid-turn thinking override window for the resumed turn (after @@ -4278,6 +4593,44 @@ export class AgentSession { const turnThinkingOverride: ActiveTurnThinkingOverride = {}; this.activeTurnThinkingOverride = turnThinkingOverride; try { + const withdrawn = (): boolean => internal?.abortSignal?.aborted === true; + if (this.workspaceGoalService) { + const pricingGate = await this.workspaceGoalService.assertPricedModelForBudgetedGoal( + this.workspaceId, + modelForStream + ); + if (!pricingGate.success) { + return Err(pricingGate.error); + } + } + if (withdrawn()) { + return Ok({ started: false }); + } + let refuseStreamStart: (() => boolean) | undefined; + let launchRefusedBy: () => "goal" | "workspace-turn" | undefined = () => undefined; + if (internal?.revalidateAdmission === true) { + const admission = await this.admitResumeLaunch({ + goalKind: internal.goalKind, + goalId: internal.goalId, + muxMetadata: optionsForStream.muxMetadata, + workspaceTurnMetadata: internal.workspaceTurnMetadata, + }); + if (!admission.admissible) { + return Ok({ started: false, refusedBy: admission.refusedBy }); + } + refuseStreamStart = admission.refuseStreamStart; + launchRefusedBy = admission.refusedBy; + } + // Last admission check before the stream's own pre-start I/O, like sendMessage's PREPARING + // gate: a withdrawal that landed during the awaits above refuses the turn here; the abort + // signal covers the history reads that follow. + if (withdrawn()) { + return Ok({ started: false }); + } + if (refuseStreamStart?.() === true) { + return Ok({ started: false, refusedBy: launchRefusedBy() }); + } + // Must await here so the finally block runs after streaming completes, // not immediately when the Promise is returned. const result = await this.streamWithHistory( @@ -4286,16 +4639,26 @@ export class AgentSession { undefined, undefined, internal?.agentInitiated, - undefined, + internal?.abortSignal, internal?.goalKind, internal?.goalId, - turnThinkingOverride + turnThinkingOverride, + refuseStreamStart, + internal?.stepBudget, + internal?.revalidateAdmission, + internal?.modelFallbackProgress ); if (!result.success) { return result; } + const refusedBy = launchRefusedBy(); + if (refusedBy != null) { + return Ok({ started: false, refusedBy }); + } - return Ok({ started: true }); + // A withdrawal inside streamWithHistory returns Ok before any stream registers, so the + // turn is still PREPARING here; after a real stream it has moved on. + return Ok({ started: !(withdrawn() && this.turnPhase === TurnPhase.PREPARING) }); } finally { if (this.turnPhase === TurnPhase.PREPARING) { this.setTurnPhase(TurnPhase.IDLE); @@ -4303,6 +4666,61 @@ export class AgentSession { } } + /** + * Revalidates a resume against durable goal state (the veto durable goal redispatches apply) + * and the workspace plus delegated turn (admitStrandedTurnResume). The probes ride along to + * the stream-admission boundary: a Pause, goal replacement, or workspace stop landing during + * the history reads and request construction has no stream to interrupt, so the launch itself + * rechecks them (StreamManager last, right before registration). Sticky about what refused it. + */ + private async admitResumeLaunch(input: { + goalKind?: GoalSyntheticMessageKind; + goalId?: string; + muxMetadata: unknown; + /** The delegated turn when `muxMetadata` does not carry it (an inherited correlation). */ + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; + }): Promise< + | { admissible: false; refusedBy: "goal" | "workspace-turn" } + | { + admissible: true; + refuseStreamStart?: () => boolean; + refusedBy: () => "goal" | "workspace-turn" | undefined; + } + > { + let goalAdmissionStale: (() => boolean) | undefined; + if (input.goalKind != null && input.goalId != null && this.workspaceGoalService) { + const admission = await this.workspaceGoalService.buildGoalRedispatchAdmission( + this.workspaceId, + input.goalId, + input.goalKind + ); + if (!admission.admissible) { + return { admissible: false, refusedBy: "goal" }; + } + goalAdmissionStale = admission.admissionStale; + } + let turnAdmissionStale: (() => boolean) | undefined; + if (this.admitStrandedTurnResume) { + const admission = await this.admitStrandedTurnResume( + getWorkspaceTurnMuxMetadata(input.muxMetadata) ?? input.workspaceTurnMetadata + ); + if (!admission.admissible) { + return { admissible: false, refusedBy: "workspace-turn" }; + } + turnAdmissionStale = admission.admissionStale; + } + let refusedBy: "goal" | "workspace-turn" | undefined; + const refuseStreamStart = + goalAdmissionStale != null || turnAdmissionStale != null + ? (): boolean => { + refusedBy ??= goalAdmissionStale?.() === true ? "goal" : undefined; + refusedBy ??= turnAdmissionStale?.() === true ? "workspace-turn" : undefined; + return refusedBy != null; + } + : undefined; + return { admissible: true, refuseStreamStart, refusedBy: () => refusedBy }; + } + async setAutoRetryEnabled( enabled: boolean, options?: { persist?: boolean } @@ -4552,12 +4970,20 @@ export class AgentSession { goalId?: string; muxMetadata?: MuxMessageMetadata; workspaceTurnMetadata?: Extract; + stepBudget?: number; + modelFallbackProgress?: ModelFallbackProgress; + revalidateAdmission?: boolean; }): CompactionFollowUpRequest { const followUp: CompactionFollowUpRequest = { text: params.messageText, model: params.modelForStream, agentId: params.options.agentId, ...pickPreservedSendOptions(params.options), + ...(params.stepBudget != null ? { stepBudget: params.stepBudget } : {}), + ...(params.modelFallbackProgress != null + ? { modelFallbackProgress: params.modelFallbackProgress } + : {}), + ...(params.revalidateAdmission === true ? { revalidateAdmission: true } : {}), }; if (params.agentInitiated === true) { @@ -4838,6 +5264,10 @@ export class AgentSession { goalId: streamContext.goalId, modelForStream: streamContext.modelString, muxMetadata: streamContext.workspaceTurnMetadata, + // The abort handler left the interrupted stream's remainder on this context. + stepBudget: streamContext.stepBudget, + modelFallbackProgress: streamContext.modelFallbackProgress, + revalidateAdmission: streamContext.revalidateAdmission, }); // Waterfall hook point: see the on-send compaction.prepare run above. await eventSpine.run("compaction.prepare", { @@ -4919,8 +5349,12 @@ export class AgentSession { }): Promise> { this.assertNotDisposed("interruptStream"); - // Explicit user interruption should immediately stop any pending auto-retry loop. + // Explicit user interruption should immediately stop any pending auto-retry loop and + // withdraw any continuation owed to a stranded turn, including a resume still in its + // pre-stream window, which StreamManager has no stream to abort for (the stream-abort + // handler repeats this for a stop that lands mid-step). this.retryManager.cancel(); + this.withdrawStrandedTurnResume(); if (options?.soft !== true) { this.queuedProviderToolEndAbortInFlight = false; @@ -5004,6 +5438,42 @@ export class AgentSession { // A disposed session must not persist retry/goal state post-teardown. if (outcome.status !== "failed" || this.disposed) return; + // A retry continues the same logical turn: it runs under what the failed attempt left + // of the ceiling, not the budget the attempt started with (auto-retry and the in-session + // context_exceeded retries alike). + if (outcome.stepsRemaining != null) { + const retryRequest = this.lastAutoRetryResumeRequest; + if (retryRequest?.stepBudget != null) { + this.lastAutoRetryResumeRequest = { + ...retryRequest, + stepBudget: outcome.stepsRemaining, + }; + } + if (this.activeStreamContext?.stepBudget != null) { + this.activeStreamContext.stepBudget = outcome.stepsRemaining; + } + } + // Likewise the chain: an attempt that moved down its fallback chain hands the retry the + // model it reached and the refusals so far, so no refused hop is attempted again. + const progress = outcome.modelFallbackProgress; + if (progress != null && progress.refusedModels.length > 0 && outcome.modelString != null) { + const retryRequest = this.lastAutoRetryResumeRequest; + if (retryRequest != null) { + this.lastAutoRetryResumeRequest = { + ...retryRequest, + options: { ...retryRequest.options, model: outcome.modelString }, + modelFallbackProgress: progress, + }; + } + const context = this.activeStreamContext; + if (context != null) { + context.modelString = outcome.modelString; + context.modelFallbackProgress = progress; + if (context.options != null) { + context.options = { ...context.options, model: outcome.modelString }; + } + } + } try { await this.handleStreamError(outcome.streamError); } finally { @@ -5030,13 +5500,22 @@ export class AgentSession { // Session-owned per-turn holder for mid-turn thinking changes. Passed // explicitly (not read from the field) so a preempted turn can never pick // up its replacement's holder. Absent for internal retry paths. - activeTurnThinkingOverride?: ActiveTurnThinkingOverride + activeTurnThinkingOverride?: ActiveTurnThinkingOverride, + // Pull-based admission probe (goal state) with no push into abortSignal; checked wherever + // the signal is, and by StreamManager right before the stream registers. + refuseStreamStart?: () => boolean, + stepBudget?: number, + revalidateAdmission?: boolean, + modelFallbackProgress?: ModelFallbackProgress ): Promise> { // Re-read at every pre-stream checkpoint below: dispose or shutdown can land while a // recovery-initiated stream (which carries no abortSignal) awaits commitPartial, file-change // detection, or history reads, and must not reach the provider afterwards. const isStreamStartAborted = (): boolean => - this.disposed || this.shuttingDown || abortSignal?.aborted === true; + this.disposed || + this.shuttingDown || + abortSignal?.aborted === true || + refuseStreamStart?.() === true; if (isStreamStartAborted()) { return Ok(undefined); @@ -5056,6 +5535,9 @@ export class AgentSession { openaiTruncationModeOverride, ...(goalKind != null ? { goalKind } : {}), ...(goalId != null ? { goalId } : {}), + ...(stepBudget != null ? { stepBudget } : {}), + ...(modelFallbackProgress != null ? { modelFallbackProgress } : {}), + ...(revalidateAdmission === true ? { revalidateAdmission: true } : {}), providersConfig, }; this.activeStreamUserMessageId = undefined; @@ -5125,6 +5607,7 @@ export class AgentSession { // [CONTINUE] sentinel so the model has a valid conversation to respond to. This is // defense-in-depth; callers should prefer sendMessage() which persists a real user message. const lastMsg = requestMessages[requestMessages.length - 1]; + let sentinelMessageId: string | undefined; if (lastMsg?.role === "assistant" && !lastMsg.metadata?.partial) { log.warn("streamWithHistory: trailing non-partial assistant detected, injecting [CONTINUE]", { workspaceId: this.workspaceId, @@ -5135,11 +5618,29 @@ export class AgentSession { synthetic: true, }); await this.historyService.appendToHistory(this.workspaceId, sentinelMessage); + sentinelMessageId = sentinelMessage.id; const refreshed = await this.historyService.getHistoryFromLatestBoundary(this.workspaceId); if (refreshed.success) { requestMessages = filterOrphanedMcpPromptSnapshots(refreshed.data); } } + // A launch refused or withdrawn after the sentinel landed would leave it as an orphan that the + // next unrelated turn sends to the provider; it goes with the launch it was appended for. + const abortStartup = async (): Promise> => { + if (sentinelMessageId != null) { + const removed = await this.historyService.deleteMessage( + this.workspaceId, + sentinelMessageId + ); + if (!removed.success) { + log.warn("Failed to remove the [CONTINUE] sentinel of a refused launch", { + workspaceId: this.workspaceId, + error: removed.error, + }); + } + } + return Ok(undefined); + }; // Capture the current user message id so retries are stable across assistant message ids. // Retry-eligible rows only: startup recovery matches this persisted ID @@ -5159,7 +5660,7 @@ export class AgentSession { ); if (isStreamStartAborted()) { - return Ok(undefined); + return await abortStartup(); } // Check if post-compaction attachments should be injected. @@ -5168,7 +5669,7 @@ export class AgentSession { ? null : await this.getPostCompactionAttachmentsIfNeeded(this.isRlmCompactionEnabled(options)); if (isStreamStartAborted()) { - return Ok(undefined); + return await abortStartup(); } this.activeStreamHadPostCompactionInjection = @@ -5241,6 +5742,9 @@ export class AgentSession { workspaceId: this.workspaceId, modelString, abortSignal, + refuseStreamStart, + stepBudget, + modelFallbackProgress, thinkingLevel: effectiveThinkingLevel, // Orthogonal to thinking level; buildRequestHeaders gates it per model. reasoningMode: options?.reasoningMode, @@ -5268,6 +5772,19 @@ export class AgentSession { disableWorkspaceAgents: options?.disableWorkspaceAgents, strictAgentResolution: options?.strictAgentResolution, hasQueuedMessages: this.hasQueuedMessages.bind(this), + onQueuedMessageStop: (stop) => { + if (this.activeStreamContext != null && this.activeStreamMessageId != null) { + this.strandedTurnResume = buildStrandedTurnResume({ + ...this.activeStreamContext, + modelString: stop.modelString, + cutMessageId: this.activeStreamMessageId, + stepBudget: stop.stepsRemaining, + modelFallbackProgress: stop.modelFallbackProgress, + thinkingLevelAtCut: + activeTurnThinkingOverride?.pending ?? activeTurnThinkingOverride?.applied, + }); + } + }, openaiTruncationModeOverride, // Mid-turn thinking overrides clamp against the same floor as the // send-time level above (single source of truth for the floor). @@ -5284,6 +5801,11 @@ export class AgentSession { ); } + // stream-start moves the turn to STREAMING synchronously inside startStream, so a turn still + // PREPARING here was refused or withdrawn at the launch boundary and registered nothing. + if (this.turnPhase === TurnPhase.PREPARING && isStreamStartAborted()) { + return await abortStartup(); + } this.consumeTurnCompletion(streamResult.data); return Ok(undefined); } @@ -5435,6 +5957,82 @@ export class AgentSession { return provider === "openai" && modelName?.toLowerCase().startsWith("gpt-"); } + /** + * Gate for the in-session context_exceeded retries, which relaunch through streamWithHistory + * directly: a stream admitted under resumeStream's revalidation is revalidated again (a Pause, + * Stop, or delegated-turn interrupt that landed during the cleanup awaits ends the turn rather + * than the retry restarting tool-enabled work), and a spent step budget ends it too. Undefined + * when the retry must not run; otherwise the launch-boundary probe to pass along. + */ + private async admitInSessionRetry(input: { + stepBudget?: number; + revalidateAdmission?: boolean; + goalKind?: GoalSyntheticMessageKind; + goalId?: string; + muxMetadata: unknown; + workspaceTurnMetadata?: WorkspaceTurnMuxMetadata; + retryLabel: string; + }): Promise<{ refuseStreamStart?: () => boolean } | undefined> { + if (input.stepBudget != null && input.stepBudget <= 0) { + log.info(`Skipping ${input.retryLabel}: the turn's step budget is spent`, { + workspaceId: this.workspaceId, + }); + return undefined; + } + if (input.revalidateAdmission !== true) { + return {}; + } + const admission = await this.admitResumeLaunch(input); + if (!admission.admissible) { + log.info(`Skipping ${input.retryLabel}: no longer admitted`, { + workspaceId: this.workspaceId, + refusedBy: admission.refusedBy, + }); + return undefined; + } + return { refuseStreamStart: admission.refuseStreamStart }; + } + + /** + * Runs an in-session retry's launch under PREPARING. False when no stream registered: an Err, + * or a launch refused at the boundary, which returns Ok before any stream registers (like a + * refused resumeStream). Either way the recovery decision stays pending: the terminal path in + * handleStreamError resolves it once settlement state is final, so waiters (task/workspace-turn + * settlement) never observe a transient "retry preparing" that already ended before startup. + */ + private async launchInSessionRetry( + retryLabel: string, + refuseStreamStart: (() => boolean) | undefined, + launch: () => Promise> + ): Promise { + try { + const result = await launch(); + if (!result.success) { + log.error(`${retryLabel} failed to start`, { + workspaceId: this.workspaceId, + error: result.error, + }); + return false; + } + // stream-start moves the turn to STREAMING synchronously inside startStream, so a turn + // still PREPARING here registered nothing. + if ( + this.turnPhase === TurnPhase.PREPARING && + (refuseStreamStart?.() === true || this.disposed || this.shuttingDown) + ) { + log.info(`${retryLabel} refused at the launch boundary`, { + workspaceId: this.workspaceId, + }); + return false; + } + return true; + } finally { + if (this.turnPhase === TurnPhase.PREPARING) { + this.setTurnPhase(TurnPhase.IDLE); + } + } + } + private async maybeRetryCompactionOnContextExceeded(data: { messageId: string; errorType?: string; @@ -5491,6 +6089,10 @@ export class AgentSession { const retryAgentInitiated = this.activeStreamContext?.agentInitiated; const retryGoalKind = this.activeStreamContext?.goalKind; const retryGoalId = this.activeStreamContext?.goalId; + const retryStepBudget = this.activeStreamContext?.stepBudget; + const retryModelFallbackProgress = this.activeStreamContext?.modelFallbackProgress; + const retryRevalidateAdmission = this.activeStreamContext?.revalidateAdmission; + const retryWorkspaceTurnMetadata = this.activeStreamContext?.workspaceTurnMetadata; const retryOptionsForResume = retryOptions ?? { model: context.modelString, agentId: WORKSPACE_DEFAULTS.agentId, @@ -5509,43 +6111,54 @@ export class AgentSession { }); return false; } + const retryAdmission = await this.admitInSessionRetry({ + stepBudget: retryStepBudget, + revalidateAdmission: retryRevalidateAdmission, + goalKind: retryGoalKind, + goalId: retryGoalId, + muxMetadata: retryOptionsForResume.muxMetadata, + workspaceTurnMetadata: retryWorkspaceTurnMetadata, + retryLabel: "compaction retry", + }); + if (retryAdmission == null) { + return false; + } this.setAutoRetryResumeState( retryOptionsForResume, retryAgentInitiated, retryGoalKind, - retryGoalId + retryGoalId, + retryStepBudget, + retryRevalidateAdmission, + retryModelFallbackProgress, + retryWorkspaceTurnMetadata ); this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata( retryOptionsForResume.muxMetadata ); this.setTurnPhase(TurnPhase.PREPARING); - let retryResult: Result; - try { - retryResult = await this.streamWithHistory( - context.modelString, - retryOptions, - isGptClass ? "auto" : undefined, - undefined, - retryAgentInitiated, - undefined, - retryGoalKind, - retryGoalId - ); - } finally { - if (this.turnPhase === TurnPhase.PREPARING) { - this.setTurnPhase(TurnPhase.IDLE); - } - } - if (!retryResult.success) { - // Leave the recovery decision pending: the terminal path in - // handleStreamError resolves it once settlement state is final, so - // waiters (task/workspace-turn settlement) never observe a transient - // "retry preparing" that already failed before stream startup. - log.error("Compaction retry failed to start", { - workspaceId: this.workspaceId, - error: retryResult.error, - }); + const retryStarted = await this.launchInSessionRetry( + "Compaction retry", + retryAdmission.refuseStreamStart, + () => + this.streamWithHistory( + context.modelString, + retryOptions, + isGptClass ? "auto" : undefined, + undefined, + retryAgentInitiated, + undefined, + retryGoalKind, + retryGoalId, + undefined, + retryAdmission.refuseStreamStart, + retryStepBudget, + retryRevalidateAdmission, + retryModelFallbackProgress + ) + ); + if (!retryStarted) { return false; } @@ -5623,36 +6236,43 @@ export class AgentSession { ); return false; } + const retryAdmission = await this.admitInSessionRetry({ + stepBudget: context.stepBudget, + revalidateAdmission: context.revalidateAdmission, + goalKind: context.goalKind, + goalId: context.goalId, + muxMetadata: context.options?.muxMetadata, + workspaceTurnMetadata: context.workspaceTurnMetadata, + retryLabel: "post-compaction retry", + }); + if (retryAdmission == null) { + return false; + } // Retry the same request, but without post-compaction injection. this.preparingWorkspaceTurnMetadata = getWorkspaceTurnMuxMetadata(context.options?.muxMetadata); this.setTurnPhase(TurnPhase.PREPARING); - let retryResult: Result; - try { - retryResult = await this.streamWithHistory( - context.modelString, - context.options, - context.openaiTruncationModeOverride, - true, - context.agentInitiated, - undefined, - context.goalKind, - context.goalId - ); - } finally { - if (this.turnPhase === TurnPhase.PREPARING) { - this.setTurnPhase(TurnPhase.IDLE); - } - } - - if (!retryResult.success) { - // Leave the recovery decision pending: the terminal path in - // handleStreamError resolves it once settlement state is final (see - // maybeRetryCompactionOnContextExceeded). - log.error("Post-compaction retry failed to start", { - workspaceId: this.workspaceId, - error: retryResult.error, - }); + const retryStarted = await this.launchInSessionRetry( + "Post-compaction retry", + retryAdmission.refuseStreamStart, + () => + this.streamWithHistory( + context.modelString, + context.options, + context.openaiTruncationModeOverride, + true, + context.agentInitiated, + undefined, + context.goalKind, + context.goalId, + undefined, + retryAdmission.refuseStreamStart, + context.stepBudget, + context.revalidateAdmission, + context.modelFallbackProgress + ) + ); + if (!retryStarted) { return false; } @@ -5766,6 +6386,7 @@ export class AgentSession { this.activeToolCallIds.clear(); this.activeStreamContext = undefined; this.activeStreamUserMessageId = undefined; + this.activeStreamMessageId = undefined; this.activeStreamStartedAtMs = undefined; this.activeStreamHadPostCompactionInjection = false; this.activeStreamHadAnyDelta = false; @@ -5776,6 +6397,9 @@ export class AgentSession { this.setTurnPhase(TurnPhase.COMPLETING); this.queuedProviderToolEndAbortInFlight = false; + // A stop decided for a queued message can still end as a stream error; the error path + // (auto-retry or a terminal error row) owns what happens next, not the stranded resume. + this.withdrawStrandedTurnResume(); this.clearLiveUsageState(); const hadCompactionRequest = this.activeCompactionRequest !== undefined; if ( @@ -5847,6 +6471,7 @@ export class AgentSession { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; this.preparingWorkspaceTurnMetadata = undefined; + this.activeStreamMessageId = payload.messageId; this.activeStreamStartedAtMs = payload.startTime; // Codex P1 (PRRT_kwDOPxxmWM6cClKS): a new live stream makes mid-stream // setGoal deferral meaningful again — clear the goal service's settled @@ -6000,6 +6625,15 @@ export class AgentSession { }); const preStreamAbortReason = "abortReason" in payload ? payload.abortReason : undefined; + // A hard stop with no registered stream (task_stop / interrupt cascade through + // aiService.stopStream while a stranded resume is still in its pre-stream I/O) must + // withdraw that resume here: StreamManager had nothing to abort, and the STREAMING + // branch below never runs for it. Only the queued-message soft stop keeps its obligation. + if ( + !(preStreamAbortReason === "queued-message" && this.queuedProviderToolEndAbortInFlight) + ) { + this.withdrawStrandedTurnResume(); + } if (this.turnPhase === TurnPhase.PREPARING) { this.clearPreparingRuntimeStatus(); this.setTerminalStreamLifecycle("interrupted", { @@ -6022,7 +6656,9 @@ export class AgentSession { } this.setTurnPhase(TurnPhase.COMPLETING); - const activeModelForAbort = this.activeStreamContext?.modelString; + // Price the abort against the model that produced the usage (a configured fallback may + // differ from the requested one), as stream-end does; the context still names the request. + const activeModelForAbort = payload.metadata?.model ?? this.activeStreamContext?.modelString; if (activeModelForAbort) { this.updateUsageStateFromModelUsage({ model: activeModelForAbort, @@ -6036,9 +6672,33 @@ export class AgentSession { const failedUserMessageId = this.activeStreamUserMessageId; const hadCompactionRequest = this.activeCompactionRequest !== undefined; + const abortedStreamContext = this.activeStreamContext; + // Whoever continues the interrupted turn from this context (mid-stream compaction holds a + // reference to it) continues under what the aborted stream left of the ceiling and from + // the chain state it reached, as a resume or retry would. + if (abortedStreamContext != null) { + if (payload.metadata?.stepsRemaining != null) { + abortedStreamContext.stepBudget = payload.metadata.stepsRemaining; + } + const abortedProgress = payload.metadata?.modelFallbackProgress; + if (abortedProgress != null && payload.metadata?.model != null) { + abortedStreamContext.modelFallbackProgress = abortedProgress; + abortedStreamContext.modelString = payload.metadata.model; + } + } const abortReason = "abortReason" in payload ? payload.abortReason : undefined; + // The soft stop is recognized by its own reason, not by the in-flight flag alone: a hard + // "system" stop (task_stop, interrupt cascade, workflow timeout) can land while the soft + // stop is pending. The flag still gates it so a hard user interrupt that reset it cancels + // the dispatch even when the soft stop wins the event race. const isQueuedProviderToolEndAbort = - this.queuedProviderToolEndAbortInFlight && abortReason !== "user"; + abortReason === "queued-message" && this.queuedProviderToolEndAbortInFlight; + // Only the queued-message soft stop owes the cut turn a continuation (rebuilt below from + // the aborted context). A user Stop must leave the session idle even though the queue it + // restores to the composer pokes the resume sweep; other aborts belong to auto-retry. + if (!isQueuedProviderToolEndAbort) { + this.withdrawStrandedTurnResume(); + } if (abortReason === "user") { await this.workspaceGoalService?.recordUserStoppedStream(this.workspaceId); } @@ -6075,10 +6735,21 @@ export class AgentSession { } await this.updateStartupAutoRetryAbandonFromAbort(abortReason, failedUserMessageId); this.emitChatEvent(payload); - const dispatchedQueuedMessage = - this.dispatchQueuedProviderToolEndMessageAfterAbort(abortReason); + // Re-read the in-flight flag: a hard stop that landed during the awaits above (synthetic + // system abort while COMPLETING, or a task hard stop clearing the queue) reset it and + // withdrew the marker, so the soft stop sampled at entry no longer owes a continuation. + const dispatchedQueuedMessage = this.dispatchQueuedProviderToolEndMessageAfterAbort( + isQueuedProviderToolEndAbort && this.queuedProviderToolEndAbortInFlight, + abortedStreamContext, + payload.metadata?.model, + payload.messageId, + payload.metadata?.stepsRemaining, + payload.metadata?.modelFallbackProgress, + payload.metadata?.requiredToolSatisfied + ); if (!dispatchedQueuedMessage) { this.setTurnPhase(TurnPhase.IDLE); + this.resumeStrandedTurnIfIdle(); } }); forward("runtime-status", (payload) => { @@ -6197,12 +6868,24 @@ export class AgentSession { // Clear the queued-message signal while the edit flow owns the next dispatch. this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); // Do not dispatch stream-end follow-ups while the edit flow is waiting - // for IDLE; truncation must run before any synthetic turn resumes. + // for IDLE; truncation must run before any synthetic turn resumes. The edit + // rewrites the interrupted turn, so nothing is owed to it either. + this.withdrawStrandedTurnResume(); } else { + if (handled) { + this.strandedTurnResume = undefined; + } + // The queued dispatch below normally consumes the owed continuation (STREAMING + // transition); it stays owed only when that message does not start a turn. this.sendQueuedMessages(); } - if (!handled && !this.deferQueuedFlushUntilAfterEdit && !hadQueuedMessages) { + if ( + !handled && + !this.deferQueuedFlushUntilAfterEdit && + !hadQueuedMessages && + this.strandedTurnResume == null + ) { const sendOptions = activeStreamOptions ?? { model: streamEndPayload.metadata.model, agentId: WORKSPACE_DEFAULTS.agentId, @@ -6258,6 +6941,7 @@ export class AgentSession { if (this.turnPhase === TurnPhase.COMPLETING) { this.resetActiveStreamState(); this.setTurnPhase(TurnPhase.IDLE); + this.resumeStrandedTurnIfIdle(); if (goalContinuationRequest != null) { await this.workspaceGoalService?.requestContinuationAfterStreamEnd({ workspaceId: this.workspaceId, @@ -6341,6 +7025,25 @@ export class AgentSession { this.emitStreamLifecycleIfChanged(); + if (next === TurnPhase.STREAMING) { + // Any stream that actually starts is the continuation the stranded turn was waiting + // for. PREPARING is not enough: a dequeued entry can still be canceled before acceptance. + const consumed = this.strandedTurnResume; + const consumedCorrelation = getWorkspaceTurnMuxMetadata(consumed?.options.muxMetadata); + if (consumed != null && consumed.claimed !== true && consumedCorrelation != null) { + this.consumedContinuationCuts.set(consumed.cutMessageId, consumedCorrelation); + // Bounded like the recovery decisions: unclaimed cuts (owner gone) must not accumulate. + for (const key of this.consumedContinuationCuts.keys()) { + if (this.consumedContinuationCuts.size <= MAX_RETAINED_CONTINUATION_CUTS) break; + this.consumedContinuationCuts.delete(key); + } + } + this.strandedTurnResume = undefined; + // "Consecutive" counts only resume attempts that never got this far: a stream that starts + // (the resume's own included) consumed the marker, so a later stranding is new work. + this.consecutiveStrandedResumes = 0; + } + if (next === TurnPhase.IDLE) { this.dispatchingQueuedEntry = false; this.dispatchingQueuedEntryMuxMetadata = undefined; @@ -6401,6 +7104,11 @@ export class AgentSession { async discardAutoRetryForContextMutation(): Promise> { this.retryManager.cancel(); this.setAutoRetryResumeState(undefined); + // The discarded transcript yields no successor stream, so a delegated turn whose stream-end + // the owner deferred on the owed continuation is settled here rather than left running. + this.forfeitStrandedTurnResume( + "Stranded turn resume discarded: workspace context was mutated." + ); const deleteResult = await this.historyService.deletePartial(this.workspaceId); if (!deleteResult.success) { return Err(deleteResult.error); @@ -6438,14 +7146,14 @@ export class AgentSession { // Entries left queued while the block was held have no stream-end // drain to dispatch them (the session stayed idle throughout) — // drain now, mirroring the edit-admission release. Only when entries - // exist: releases from a session that never queued must stay - // side-effect free. - if ( - this.turnAdmissionBlocks === 0 && - this.turnPhase === TurnPhase.IDLE && - !this.messageQueue.isEmpty() - ) { - this.sendQueuedMessages(); + // (or an owed stranded resume) exist: releases from a session that + // never queued must stay side-effect free. + if (this.turnAdmissionBlocks === 0 && this.turnPhase === TurnPhase.IDLE) { + if (!this.messageQueue.isEmpty()) { + this.sendQueuedMessages(); + } else { + this.resumeStrandedTurnIfIdle(); + } } }, }; @@ -6594,15 +7302,29 @@ export class AgentSession { return nextDispatchableMode ?? null; } - clearQueue(cancelReason = "Queued message cleared before dispatch."): void { + clearQueue( + cancelReason = "Queued message cleared before dispatch.", + options?: { hardStop?: boolean } + ): void { this.assertNotDisposed("clearQueue"); + // A task hard stop (task_stop, interrupt cascade, workflow timeout) clears the queue and + // then stops the stream through StreamManager directly. That stop emits nothing when the + // stream has already completed or has not registered yet, so the queue clear is the only + // session-visible boundary at which the owed continuation and a pending provider-tool soft + // stop can be forfeited. A user clearing the queue keeps them: the cut turn still resumes. + // Forfeit rather than withdraw: with the stream already completed, the owner that deferred on + // the marker gets no stream event, so this settles the delegated turn (a no-op when the stop + // already did). + if (options?.hardStop === true) { + this.queuedProviderToolEndAbortInFlight = false; + this.forfeitStrandedTurnResume("Stranded turn resume dropped: task hard stop."); + } + const heldOwedContinuation = this.queueHoldsOwedTurnContinuation(); const callbackSets = this.messageQueue.getClearCallbacks(); this.messageQueue.clear(); this.emitQueuedMessageChanged(); this.backgroundProcessManager.setMessageQueued(this.workspaceId, false); - for (const callbacks of callbackSets) { - this.notifyQueuedMessageCleared(callbacks, cancelReason); - } + this.cancelRemovedEntries(callbackSets, cancelReason, heldOwedContinuation); } setQueuedMessageDispatchMode(mode: "tool-end" | "turn-end"): boolean { @@ -6628,7 +7350,7 @@ export class AgentSession { onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; }, cancelReason: string - ): void { + ): Promise { const notify = async () => { if (callbacks.onCanceled != null) { await callbacks.onCanceled(cancelReason); @@ -6636,7 +7358,7 @@ export class AgentSession { } await callbacks.onAcceptedPreStreamFailure?.(createUnknownSendMessageError(cancelReason)); }; - notify().catch((error: unknown) => { + return notify().catch((error: unknown) => { log.error("Queued message clear callback failed", { workspaceId: this.workspaceId, error: getErrorMessage(error), @@ -6644,9 +7366,58 @@ export class AgentSession { }); } + /** + * Whether the queue holds an entry continuing the owed marker's delegated turn. The owner + * defers that turn's settlement on such an entry (claimWorkspaceTurnContinuation), and the + * entry's onCanceled settles the handle if it is removed unstarted. + */ + private queueHoldsOwedTurnContinuation(): boolean { + const correlation = getWorkspaceTurnMuxMetadata(this.strandedTurnResume?.options.muxMetadata); + return correlation != null && this.messageQueue.hasWorkspaceTurn(correlation.taskHandleId); + } + + /** + * Whether a send with this metadata continues the owed cut turn (a wake inherits its + * correlation from history; a workspace-turn entry must carry the same correlation) rather than + * supersede it. Vacuously true when nothing is owed. + */ + private continuesOwedTurn(muxMetadata: MuxMessageMetadata | undefined): boolean { + const owed = this.strandedTurnResume; + if (owed == null || muxMetadata?.type === "bash-monitor-wake") { + return true; + } + return hasSameWorkspaceTurnCorrelation( + getWorkspaceTurnMuxMetadata(muxMetadata), + getWorkspaceTurnMuxMetadata(owed.options.muxMetadata) + ); + } + + /** + * Runs removed entries' cancellation and sweeps for the owed continuation only once it has + * settled, as the dequeue path does: a canceled workspace-turn entry settles its handle in + * onCanceled, and a sweep admitted before that lands would resume against an interrupted + * handle. An entry that continued the owed turn was that turn's terminal path, so the + * continuation is forfeited before its cancellation runs. + */ + private cancelRemovedEntries( + callbackSets: QueueClearCallbacks[], + cancelReason: string, + heldOwedContinuation: boolean + ): void { + if (heldOwedContinuation && !this.queueHoldsOwedTurnContinuation()) { + this.forfeitStrandedTurnResume( + "Stranded turn resume dropped: its queued continuation was canceled." + ); + } + void Promise.all( + callbackSets.map((callbacks) => this.notifyQueuedMessageCleared(callbacks, cancelReason)) + ).then(() => this.resumeStrandedTurnIfIdle()); + } + removeQueuedMessagesByDedupeKeyPrefix(prefix: string, cancelReason: string): number { this.assertNotDisposed("removeQueuedMessagesByDedupeKeyPrefix"); assert(prefix.length > 0, "removeQueuedMessagesByDedupeKeyPrefix requires prefix"); + const heldOwedContinuation = this.queueHoldsOwedTurnContinuation(); const removal = this.messageQueue.removeByDedupeKeyPrefix(prefix); if (removal.removedCount === 0) { return 0; @@ -6656,9 +7427,7 @@ export class AgentSession { this.workspaceId, this.messageQueue.getNextDispatchableMode() === "tool-end" ); - for (const callbacks of removal.callbacks) { - this.notifyQueuedMessageCleared(callbacks, cancelReason); - } + this.cancelRemovedEntries(removal.callbacks, cancelReason, heldOwedContinuation); return removal.removedCount; } @@ -6675,6 +7444,7 @@ export class AgentSession { removeQueuedWorkspaceTurn(handleId: string, cancelReason: string): boolean { this.assertNotDisposed("removeQueuedWorkspaceTurn"); assert(handleId.length > 0, "removeQueuedWorkspaceTurn requires handleId"); + const heldOwedContinuation = this.queueHoldsOwedTurnContinuation(); const callbacks = this.messageQueue.removeWorkspaceTurn(handleId); if (callbacks == null) { return false; @@ -6684,7 +7454,7 @@ export class AgentSession { this.workspaceId, this.messageQueue.getNextDispatchableMode() === "tool-end" ); - this.notifyQueuedMessageCleared(callbacks, cancelReason); + this.cancelRemovedEntries([callbacks], cancelReason, heldOwedContinuation); return true; } @@ -6740,13 +7510,12 @@ export class AgentSession { * Whether a bash-monitor-wake continuation is pending dispatch: the next * queued entry is a wake, or a dequeued wake is mid-dispatch (dequeue → * stream start). Wake sends are the only input that inherits an open - * delegated workspace turn's correlation, so TaskService uses this — not - * generic queued/preparing state — to decide whether a correlated - * "tool-calls" queue cut will be continued rather than superseded. Once the - * wake's stream starts, TaskService matches the active stream's inherited - * correlation instead (see hasSameTurnWakeContinuation). + * delegated workspace turn's correlation, so claimWorkspaceTurnContinuation + * treats one as the turn's continuation rather than a superseding entry. Once + * the wake's stream starts, the owner matches the active stream's inherited + * correlation instead. */ - hasPendingBashMonitorWakeContinuation(): boolean { + private hasPendingBashMonitorWakeContinuation(): boolean { if (this.messageQueue.isNextEntryBashMonitorWake()) { return true; } @@ -6755,10 +7524,29 @@ export class AgentSession { } /** - * Whether a queued or dispatching entry continues the exact workspace-turn correlation. + * The delegated turn owner's settlement decision for a correlated "tool-calls" stream-end: + * true when a continuation of that exact turn is pending (the owner defers), false when the + * cut superseded it (the owner settles the turn now). A false answer binds the owed + * continuation: the marker for that cut is voided here, so the turn cannot resume as orphaned + * work no matter how the superseding entry later leaves the queue. The owner reads under its + * own event lock, so this is the only point where its view and the marker are the same. */ - hasPendingWorkspaceTurnContinuation( - metadata: Extract + claimWorkspaceTurnContinuation( + metadata: Extract, + streamEndMessageId: string + ): boolean { + const deferred = this.answerWorkspaceTurnContinuationClaim(metadata, streamEndMessageId); + // The owner claims each stream-end once, so a cut it deferred on here needs no evidence + // retained for a late claim (consumedContinuationCuts). + if (deferred && this.strandedTurnResume?.cutMessageId === streamEndMessageId) { + this.strandedTurnResume.claimed = true; + } + return deferred; + } + + private answerWorkspaceTurnContinuationClaim( + metadata: Extract, + streamEndMessageId: string ): boolean { if (hasSameWorkspaceTurnCorrelation(this.preparingWorkspaceTurnMetadata, metadata)) { return true; @@ -6775,12 +7563,177 @@ export class AgentSession { } const dispatching = this.dispatchingQueuedEntryMuxMetadata as MuxMessageMetadata | undefined; - return ( + if ( dispatching?.type === "workspace-turn-task" && dispatching.taskHandleId === metadata.taskHandleId && dispatching.ownerWorkspaceId === metadata.ownerWorkspaceId && dispatching.turnId === metadata.turnId + ) { + return true; + } + + // A wake send inherits the open delegated turn's correlation from history, so a wake at + // the queue head continues the turn even though the entry carries no correlation itself. + if (this.hasPendingBashMonitorWakeContinuation()) { + return true; + } + + // The continuation of this exact cut already ran (and may already have ended): its own + // stream events settle the turn, so the late claim defers rather than settling it as failed. + const consumedCorrelation = this.consumedContinuationCuts.get(streamEndMessageId); + if (consumedCorrelation != null) { + this.consumedContinuationCuts.delete(streamEndMessageId); + if (hasSameWorkspaceTurnCorrelation(consumedCorrelation, metadata)) { + return true; + } + } + + const owed = this.strandedTurnResume; + if ( + owed == null || + !hasSameWorkspaceTurnCorrelation( + getWorkspaceTurnMuxMetadata(owed.options.muxMetadata), + metadata + ) + ) { + return false; + } + // A marker from a later cut of this turn proves a continuation already ran after the + // stream-end being settled (the owner is processing an older event): defer, and leave the + // newer cut's continuation to its own stream-end. + if (owed.cutMessageId !== streamEndMessageId) { + return true; + } + // Past the retry cap the marker is no longer advertised; the sweep forfeits it. + if (this.owedStrandedTurnResume() == null) { + return false; + } + // Nothing dispatchable queued to take the turn (withdrawn entries drain as no-ops): the + // stranded resume will carry this correlation. + if (this.messageQueue.getNextDispatchableMode() == null && !this.dispatchingQueuedEntry) { + return true; + } + // A queued or dispatching entry that failed the checks above supersedes the turn; the + // owner settles it on this answer, so the continuation must not outlive that entry. + this.withdrawStrandedTurnResume(); + return false; + } + + /** + * The continuation still owed to a stranded turn. Past the retry cap the marker is + * forfeited: nothing may advertise it (the owner would defer settlement for a resume that + * never starts) and the next sweep drops it. + */ + private owedStrandedTurnResume(): StrandedTurnResume | undefined { + return this.consecutiveStrandedResumes < MAX_CONSECUTIVE_STRANDED_TURN_RESUMES + ? this.strandedTurnResume + : undefined; + } + + /** + * The continuation is given up with no successor stream (sweep cap or goal refusal, context + * discard, session disposal). A delegated turn's owner may have deferred its stream-end on the + * strength of this marker (claimWorkspaceTurnContinuation), and no later stream-end will + * arrive for it, so the owner settles that turn here. + */ + private forfeitStrandedTurnResume(reason: string): void { + const correlation = getWorkspaceTurnMuxMetadata(this.strandedTurnResume?.options.muxMetadata); + // Retain the owner's only terminal path before dropping the marker that advertised it. + const owed = this.recordOwedForfeit(correlation, reason); + this.withdrawStrandedTurnResume(); + if (owed != null) { + void this.settleOwedForfeit(owed); + } + } + + /** + * A delegated turn given up outside the marker (a dropped compaction follow-up that continued + * it): the same settlement, since nothing else will end the turn for its owner. Resolves true + * once the owner has the terminal record (or nothing was owed), false when the attempt failed + * and the settlement stays owed. + */ + private forfeitWorkspaceTurnContinuation( + correlation: WorkspaceTurnMuxMetadata | undefined, + reason: string + ): Promise { + const owed = this.recordOwedForfeit(correlation, reason); + return owed == null ? Promise.resolve(true) : this.settleOwedForfeit(owed); + } + + private recordOwedForfeit( + correlation: WorkspaceTurnMuxMetadata | undefined, + reason: string + ): OwedForfeitSettlement | undefined { + if (correlation == null || this.settleForfeitedWorkspaceTurnContinuation == null) { + return undefined; + } + const owed: OwedForfeitSettlement = { + key: `${correlation.ownerWorkspaceId}/${correlation.taskHandleId}/${correlation.turnId}`, + correlation, + reason, + }; + this.owedForfeitSettlements.set(owed.key, owed); + return owed; + } + + private settleOwedForfeits(): void { + for (const owed of this.owedForfeitSettlements.values()) { + void this.settleOwedForfeit(owed); + } + } + + /** + * The settlement is the owner's only remaining path to a terminal record for that turn, so a + * failed attempt (task store I/O) stays owed and retries on its own (including after session + * disposal) as well as from idle sweeps. Settlement is idempotent on the owner's side. + */ + private settleOwedForfeit(owed: OwedForfeitSettlement): Promise { + if (owed.inFlight != null) { + return owed.inFlight; + } + const settle = this.settleForfeitedWorkspaceTurnContinuation; + assert(settle != null, "an owed forfeit settlement requires a settler"); + owed.inFlight = settle(owed.correlation, owed.reason).then( + () => { + if (this.owedForfeitSettlements.get(owed.key) === owed) { + this.owedForfeitSettlements.delete(owed.key); + } + return true; + }, + (error: unknown) => { + owed.inFlight = undefined; + log.warn("Failed to settle forfeited workspace turn continuation; retrying", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + this.scheduleOwedForfeitSettlementRetry(); + return false; + } ); + return owed.inFlight; + } + + private scheduleOwedForfeitSettlementRetry(): void { + if (this.owedForfeitSettlementRetryTimer != null) { + return; + } + // Do not cancel this on dispose: workspace removal is itself one of the boundaries that can + // forfeit a delegated turn, and its owner still requires the terminal task-store write. + this.owedForfeitSettlementRetryTimer = setTimeout(() => { + this.owedForfeitSettlementRetryTimer = null; + this.settleOwedForfeits(); + }, FORFEIT_SETTLEMENT_RETRY_DELAY_MS); + this.owedForfeitSettlementRetryTimer.unref(); + } + + /** + * Nothing is owed anymore (user Stop, superseding input): drop the marker and cancel a resume + * still in its pre-stream window, which already copied the marker. The owner of a delegated + * turn learns of it from the stream event or hard stop that caused the withdrawal. + */ + private withdrawStrandedTurnResume(): void { + this.strandedTurnResume = undefined; + this.strandedTurnResumeInFlight?.abort(); } /** @@ -6842,7 +7795,7 @@ export class AgentSession { this.queuedProviderToolEndAbortInFlight = true; const result = await this.streamManager.stopStream(this.workspaceId, { soft: true, - abortReason: "system", + abortReason: "queued-message", }); if (!result.success) { this.queuedProviderToolEndAbortInFlight = false; @@ -6854,20 +7807,44 @@ export class AgentSession { } private dispatchQueuedProviderToolEndMessageAfterAbort( - abortReason: StreamAbortReason | undefined + isQueuedProviderToolEndAbort: boolean, + abortedStreamContext: AgentSession["activeStreamContext"], + abortedModelString: string | undefined, + abortedMessageId: string, + abortedStepsRemaining: number | undefined, + abortedModelFallbackProgress: ModelFallbackProgress | undefined, + abortedRequiredToolSatisfied: boolean | undefined ): boolean { - if (!this.queuedProviderToolEndAbortInFlight) { + this.queuedProviderToolEndAbortInFlight = false; + if (!isQueuedProviderToolEndAbort || this.deferQueuedFlushUntilAfterEdit) { return false; } - // Physical check: withdrawn entries must still drain so their onCanceled fires. - const shouldDispatch = - abortReason !== "user" && - !this.deferQueuedFlushUntilAfterEdit && - !this.messageQueue.isEmpty(); - this.queuedProviderToolEndAbortInFlight = false; - - if (!shouldDispatch) { + // The soft stop was made on behalf of the queued message; if that message has been + // withdrawn (or is withdrawn after dequeue), the interrupted turn must resume instead. + // Only under the steps the cut stream had left: at zero the ceiling ended the turn, and an + // abort that reports no budget must not hand the resume a fresh one. A successful required + // tool in the cut step ended the turn too (the loop's stop condition would have, one tool + // result later), as in the loop's own queued-message stop. + if ( + abortedStreamContext != null && + abortedStepsRemaining != null && + abortedStepsRemaining > 0 && + abortedRequiredToolSatisfied !== true + ) { + this.strandedTurnResume = buildStrandedTurnResume({ + ...abortedStreamContext, + modelString: abortedModelString ?? abortedStreamContext.modelString, + cutMessageId: abortedMessageId, + stepBudget: abortedStepsRemaining, + modelFallbackProgress: abortedModelFallbackProgress, + thinkingLevelAtCut: + this.activeTurnThinkingOverride?.pending ?? this.activeTurnThinkingOverride?.applied, + }); + } + // Physical check: withdrawn entries must still drain so their onCanceled fires; that drain's + // idle transition is what sweeps the continuation registered above. + if (this.messageQueue.isEmpty()) { return false; } @@ -6875,6 +7852,115 @@ export class AgentSession { return true; } + /** + * One idempotent sweep for the owed continuation (see strandedTurnResume). Cheap enough to + * run from every idle transition and queue removal; only the first eligible call acts. + */ + private resumeStrandedTurnIfIdle(): void { + this.settleOwedForfeits(); + const resume = this.owedStrandedTurnResume(); + if (resume == null) { + if (this.strandedTurnResume != null) { + log.warn("Leaving stranded turn idle: consecutive resume cap reached", { + workspaceId: this.workspaceId, + cap: MAX_CONSECUTIVE_STRANDED_TURN_RESUMES, + }); + this.forfeitStrandedTurnResume("Stranded turn resume gave up: retry cap reached."); + } + return; + } + if ( + this.strandedTurnResumeInFlight || + this.disposed || + this.turnAdmissionBlocks > 0 || + this.hasActiveOrPendingTurnWork() || + !this.messageQueue.isEmpty() || + this.hasPendingAutoRetry() || + this.deferQueuedFlushUntilAfterEdit || + // A manual send inside WorkspaceService preflight is not queued yet and the session + // reads idle; the user's message supersedes the owed continuation (it drains here + // through drainQueuedMessagesIfIdle if it settles without a turn). + this.hasExternalSendPreflight?.() === true + ) { + return; + } + + this.consecutiveStrandedResumes += 1; + log.info("Resuming turn stranded by a withdrawn queued message", { + workspaceId: this.workspaceId, + attempt: this.consecutiveStrandedResumes, + }); + + // The owed continuation is cleared by the STREAMING transition, not here: a resume that + // fails before its stream starts (pricing gate, history read) stays owed for the next + // sweep, bounded by the cap above. + const inFlight = new AbortController(); + this.strandedTurnResumeInFlight = inFlight; + this.resumeStream(resume.options, { + agentInitiated: resume.agentInitiated, + goalKind: resume.goalKind, + goalId: resume.goalId, + abortSignal: inFlight.signal, + revalidateAdmission: true, + stepBudget: resume.stepBudget, + modelFallbackProgress: resume.modelFallbackProgress, + }) + .then((result) => { + if (!result.success) { + log.warn("Stranded turn resume failed", { + workspaceId: this.workspaceId, + error: result.error, + }); + return false; + } + if (result.data.refusedBy != null) { + // A Pause or terminal goal transition, or a stop on the workspace or delegated turn, + // landed while the cut turn waited: nothing is owed to it anymore. + log.info("Dropping stranded turn: no longer admitted", { + workspaceId: this.workspaceId, + refusedBy: result.data.refusedBy, + goalKind: resume.goalKind, + }); + this.forfeitStrandedTurnResume( + result.data.refusedBy === "goal" + ? "Stranded turn resume dropped: goal no longer admits it." + : "Stranded turn resume dropped: workspace or delegated turn no longer admits it." + ); + return false; + } + if (!result.data.started) { + log.warn("Stranded turn resume did not start", { workspaceId: this.workspaceId }); + return false; + } + return true; + }) + .catch((error: unknown) => { + log.warn("Stranded turn resume threw", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + return false; + }) + .then((started) => { + this.strandedTurnResumeInFlight = null; + // resumeStream settles only after its stream ends, so a stranding of that resumed + // stream can find the flag still set; sweep again once it clears. A resume that failed + // before its stream is swept again too: an idle session gets no later poke, and the cap's + // forfeit settles a delegated owner instead of leaving it waiting on a resume that never + // runs. Anything queued behind the failed PREPARING claim has no stream end to wait for. + this.resumeStrandedTurnIfIdle(); + if (!started) { + this.dispatchQueuedMessagesIfIdle(); + } + }) + .catch((error: unknown) => { + log.error("Stranded turn resume sweep failed", { + workspaceId: this.workspaceId, + error: getErrorMessage(error), + }); + }); + } + async waitForPendingCompactionCompletionDecision(messageId: string): Promise { if (!this.compactionCompletionDecisions.has(messageId)) { if (this.activeCompactionRequest == null) return false; @@ -6983,6 +8069,11 @@ export class AgentSession { * failed-startup drains elsewhere in this file. */ drainQueuedMessagesIfIdle(): void { + this.resumeStrandedTurnIfIdle(); + this.dispatchQueuedMessagesIfIdle(); + } + + private dispatchQueuedMessagesIfIdle(): void { if ( this.hasActiveOrPendingTurnWork() || this.deferQueuedFlushUntilAfterEdit || @@ -7087,6 +8178,8 @@ export class AgentSession { } this.sendQueuedMessages(); }); + } else { + this.resumeStrandedTurnIfIdle(); } } @@ -7252,6 +8345,26 @@ export class AgentSession { imageParts?: FilePart[]; }; + // The delegated turn this follow-up continues, if any: stamped on the follow-up itself by + // mid-stream compaction, or beside a wake follow-up by on-send compaction. Every drop below + // settles it, since the compaction abort and the compact stream end it for nobody. + const continuedTurn = + parsePersistedWorkspaceTurnMetadata(followUp.muxMetadata) ?? + parsePersistedWorkspaceTurnMetadata(followUp.workspaceTurnMetadata); + const dropFollowUp = async (reason: string): Promise => { + // The owner's terminal record lands before the follow-up leaves history: the owed + // settlement is memory, so a crash between the two would strand the handle for good. A + // failed attempt keeps the follow-up pending for the next startup to re-drop and re-settle. + const settled = await this.forfeitWorkspaceTurnContinuation( + continuedTurn, + `Compaction follow-up dropped: ${reason}` + ); + if (settled) { + await this.clearPendingFollowUpFromSummary(lastMessage); + } + return false; + }; + // Compaction summaries are unchecked chat.jsonl. Reject malformed persisted // goal attribution instead of forwarding it into goal-service assertions or // repeatedly crashing startup recovery on the same row. @@ -7265,8 +8378,7 @@ export class AgentSession { workspaceId: this.workspaceId, summaryMessageId: lastMessage.id, }); - await this.clearPendingFollowUpFromSummary(lastMessage); - return false; + return dropFollowUp("malformed goal attribution."); } // Codex P1 (PRRT_kwDOPxxmWM6cS8Bq): pre-upgrade summaries persisted @@ -7282,8 +8394,39 @@ export class AgentSession { summaryMessageId: lastMessage.id, goalKind: persistedGoalKind, }); - await this.clearPendingFollowUpFromSummary(lastMessage); - return false; + return dropFollowUp("legacy goal follow-up without goal identity."); + } + + // Same raw JSON boundary. A present but malformed remainder fails closed: the interrupted turn + // ran under a ceiling this row can no longer state, and an absent-legacy reading would hand a + // nearly spent autonomous turn the default ceiling instead. The chain state is only a + // preference order, so a malformed one falls back to the model's own chain. + const persistedStepBudget = followUp.stepBudget; + if ( + persistedStepBudget !== undefined && + !(Number.isInteger(persistedStepBudget) && persistedStepBudget >= 0) + ) { + log.warn("Discarding pending follow-up with a malformed persisted step budget", { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + }); + return dropFollowUp("malformed persisted step budget."); + } + const persistedFallbackProgress = + followUp.modelFallbackProgress != null + ? ModelFallbackProgressSchema.safeParse(followUp.modelFallbackProgress) + : undefined; + // A follow-up continuing a delegated turn is admitted and retried like a stranded resume even + // when the interrupted turn itself was not one (a delegated turn's first compaction). + const revalidateAdmission = followUp.revalidateAdmission === true || continuedTurn != null; + // The interrupted turn spent its last step before compaction: the ceiling ended it, and the + // loop's stop condition is only evaluated after a step, so a follow-up would run one more. + if (persistedStepBudget === 0) { + log.info("Discarding pending follow-up: the interrupted turn's step budget is spent", { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + }); + return dropFollowUp("the interrupted turn's step budget is spent."); } // Codex P1 (PRRT_kwDOPxxmWM6cPuMw): goal-loop follow-ups were originally @@ -7346,12 +8489,30 @@ export class AgentSession { workspaceId: this.workspaceId, goalKind: persistedGoalKind, }); - await this.clearPendingFollowUpFromSummary(lastMessage); - return false; + return dropFollowUp("goal no longer admits it."); } goalAdmissionStale = admission.admissionStale; } + // Admitted like a stranded resume (admitResumeLaunch): the workspace must still accept + // streams and a delegated turn's owner must still have it running. The probe rides along to + // the launch boundary below. + let turnAdmissionStale: (() => boolean) | undefined; + if (revalidateAdmission && this.admitStrandedTurnResume) { + const admission = await this.admitStrandedTurnResume(continuedTurn); + if (!admission.admissible) { + log.info( + "Skipping pending follow-up: the workspace or delegated turn no longer admits it", + { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + } + ); + return dropFollowUp("the workspace or delegated turn no longer admits it."); + } + turnAdmissionStale = admission.admissionStale; + } + // Codex P1 (PRRT_kwDOPxxmWM6cQt3j): the queue/busy sample above ages // across the awaited goal read and the send's own preflight. Re-evaluate // the idle rule through the send-admission gates — all of them run before @@ -7364,9 +8525,14 @@ export class AgentSession { this.hasExternalSendPreflight?.() === true || (this.isBusy() && this.turnPhase !== TurnPhase.COMPLETING) : undefined; + // Launch-safe probes only (the idle rule would trip on this send's own PREPARING turn). + const launchAdmissionStale = + goalAdmissionStale != null || turnAdmissionStale != null + ? () => goalAdmissionStale?.() === true || turnAdmissionStale?.() === true + : undefined; const followUpAdmissionStale = - idleRuleStale != null || goalAdmissionStale != null - ? () => idleRuleStale?.() === true || goalAdmissionStale?.() === true + idleRuleStale != null || launchAdmissionStale != null + ? () => idleRuleStale?.() === true || launchAdmissionStale?.() === true : undefined; log.debug("Dispatching pending follow-up from compaction summary", { @@ -7454,7 +8620,11 @@ export class AgentSession { options, followUp.agentInitiated, persistedGoalKind, - persistedGoalId + persistedGoalId, + persistedStepBudget, + revalidateAdmission, + persistedFallbackProgress?.success ? persistedFallbackProgress.data : undefined, + continuedTurn ); // Await sendMessage to ensure the follow-up is persisted before returning. @@ -7474,11 +8644,32 @@ export class AgentSession { // Codex P1 (PRRT_kwDOPxxmWM6cPuMw): re-derived admission guard for the // redispatched goal turn (see buildGoalRedispatchAdmission above). admissionStale: followUpAdmissionStale, + refuseStreamStart: launchAdmissionStale, + stepBudget: persistedStepBudget, + modelFallbackProgress: persistedFallbackProgress?.success + ? persistedFallbackProgress.data + : undefined, + revalidateAdmission, + // The wake's own metadata does not carry the delegated turn; its retries must still + // revalidate against that turn's handle. + workspaceTurnMetadata: continuedTurn, }); + // The goal, the workspace, or the delegated turn stopped admitting the follow-up during its + // preflight (an Err) or at the launch boundary (StreamManager refuses as a startup-aborted Ok + // handle, like a refused stranded resume): no successor stream, so the turn is settled and + // the follow-up dropped. + if (launchAdmissionStale?.() === true) { + log.info("Pending follow-up refused at admission: goal, workspace, or delegated turn", { + workspaceId: this.workspaceId, + summaryMessageId: lastMessage.id, + goalRefused: goalAdmissionStale?.() === true, + }); + return dropFollowUp("no longer admitted by its goal, workspace, or delegated turn."); + } if (!sendResult.success) { - // A stale-admission refusal is the idle rule (or a goal transition) - // working as intended, not a recovery failure: route it through the - // same skip path as the pre-send check instead of throwing. + // A stale-admission refusal is the idle rule working as intended, not a + // recovery failure: route it through the same skip path as the pre-send + // check instead of throwing. if (followUpAdmissionStale?.() === true) { log.info("Pending follow-up refused at send admission; skipping it", { workspaceId: this.workspaceId, diff --git a/src/node/services/aiService.test.ts b/src/node/services/aiService.test.ts index f11e5f3c4e..2db61e8647 100644 --- a/src/node/services/aiService.test.ts +++ b/src/node/services/aiService.test.ts @@ -1421,6 +1421,46 @@ describe("AIService.streamMessage compaction boundary slicing", () => { ); }); + it("a stream continuing a cut turn keeps that turn's fallback chain, not its model's", async () => { + using xumHome = new DisposableTempDir("ai-service-fallback-continuation"); + const projectPath = path.join(xumHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-fallback-continuation"; + const requestedModel = KNOWN_MODELS.SONNET.id; + const cutModel = KNOWN_MODELS.GPT.id; + const nextModel = KNOWN_MODELS.GEMINI_FLASH.id; + // The resumed model has a chain of its own that would lead back to the model that refused. + await writeMainConfig(xumHome.path, { + modelFallbacks: { + [requestedModel]: { models: [cutModel, nextModel] }, + [cutModel]: { models: [requestedModel] }, + }, + }); + const harness = createHarness( + xumHome.path, + createLocalWorkspaceMetadata(workspaceId, projectPath), + { useRequestedModelString: true } + ); + const progress = { + requestedModel, + refusedModels: [requestedModel], + chain: [cutModel, nextModel], + }; + + const result = await harness.service.streamMessage({ + messages: [createMuxMessage("latest-user", "user", "fix the issue")], + workspaceId, + modelString: cutModel, + thinkingLevel: "off", + modelFallbackProgress: progress, + }); + expect(result.success).toBe(true); + + expect(harness.startStreamCalls[0]?.modelFallback?.chain).toEqual([cutModel, nextModel]); + expect(harness.startStreamCalls[0]?.modelFallbackProgress).toEqual(progress); + }); + it("emits startup breadcrumbs as runtime-status events before stream start", async () => { using xumHome = new DisposableTempDir("ai-service-startup-breadcrumbs"); const projectPath = path.join(xumHome.path, "project"); diff --git a/src/node/services/messageQueue.test.ts b/src/node/services/messageQueue.test.ts index 82106e398c..6822743cc1 100644 --- a/src/node/services/messageQueue.test.ts +++ b/src/node/services/messageQueue.test.ts @@ -604,6 +604,61 @@ describe("MessageQueue", () => { ).toBe(false); }); + it("ignores withdrawn predecessors when checking that every entry continues the turn", () => { + const withdrawn = new AbortController(); + queue.add( + "withdrawn wake", + { + model: "gpt-4", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + }, + { synthetic: true, agentInitiated: true, cancelSignal: withdrawn.signal } + ); + queue.add("Follow up", { model: "gpt-4", agentId: "exec", muxMetadata: metadata }); + + expect( + queue.hasAllWorkspaceTurnContinuations("wst_followup", "parent-workspace", "turn-1") + ).toBe(false); + + // Withdrawn but not yet drained: no longer pending work, so it supersedes nothing. + withdrawn.abort(); + expect( + queue.hasAllWorkspaceTurnContinuations("wst_followup", "parent-workspace", "turn-1") + ).toBe(true); + }); + + it("reads the next entry past withdrawn ones for wake, correlation, and cut candidate", () => { + const withdrawn = new AbortController(); + queue.add( + "withdrawn wake", + { + model: "gpt-4", + agentId: "exec", + muxMetadata: { type: "bash-monitor-wake", records: [] }, + queueDispatchMode: "tool-end", + }, + { synthetic: true, agentInitiated: true, cancelSignal: withdrawn.signal } + ); + expect(queue.isNextEntryBashMonitorWake()).toBe(true); + + withdrawn.abort(); + expect(queue.isNextEntryBashMonitorWake()).toBe(false); + expect(queue.getNextQueueCutCandidate()).toBeUndefined(); + + queue.add("Follow up", { + model: "gpt-4", + agentId: "exec", + muxMetadata: metadata, + queueDispatchMode: "turn-end", + }); + expect(queue.isNextEntryBashMonitorWake()).toBe(false); + expect( + queue.hasNextWorkspaceTurnContinuation("wst_followup", "parent-workspace", "turn-1") + ).toBe(true); + expect(queue.getNextQueueCutCandidate()?.dispatchMode).toBe("turn-end"); + }); + it("exposes the head entry's metadata and dispatch mode as the queue-cut candidate", () => { expect(queue.getNextQueueCutCandidate()).toBeUndefined(); diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index 93593b9f5d..c6b4912be6 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -153,7 +153,7 @@ interface QueuedMessageInternalOptions { admissionStale?: () => boolean; } -type QueueClearCallbacks = Pick< +export type QueueClearCallbacks = Pick< QueuedMessageInternalOptions, "onCanceled" | "onAcceptedPreStreamFailure" >; @@ -272,48 +272,54 @@ export class MessageQueue { } /** - * Dispatch mode of the first entry whose cancel signal has not fired, or undefined - * when none remains. Aborted entries still drain FIFO (as no-ops that fire - * onCanceled), but they are not pending work and must not arm a tool-end stop. + * The first entry whose cancel signal has not fired. Aborted entries still drain FIFO (as + * no-ops that fire onCanceled), but they are not pending work: they must not arm a tool-end + * stop, count as a turn's continuation, or be attributed a cut. */ + private nextDispatchableEntry(): QueueEntry | undefined { + return this.entries.find((entry) => entry.cancelSignal?.aborted !== true); + } + + /** Dispatch mode of the next dispatchable entry, or undefined when none remains. */ getNextDispatchableMode(): QueueDispatchMode | undefined { - return this.entries.find((entry) => entry.cancelSignal?.aborted !== true)?.dispatchMode; + return this.nextDispatchableEntry()?.dispatchMode; } /** - * Whether every queued entry continues the exact workspace turn correlation. + * Whether every pending queued entry continues the exact workspace turn correlation. * * The caller uses this for a new continuation that has not entered the queue. - * An unrelated entry anywhere ahead of it supersedes the correlation. + * An unrelated pending entry anywhere ahead of it supersedes the correlation; a withdrawn + * entry still draining is not pending work (see nextDispatchableEntry) and supersedes nothing. */ hasAllWorkspaceTurnContinuations( taskHandleId: string, ownerWorkspaceId: string, turnId: string ): boolean { - return ( - this.entries.length > 0 && - this.entries.every((entry) => { - const metadata = entry.muxMetadata; - return ( - isWorkspaceTurnMetadata(metadata) && - metadata.taskHandleId === taskHandleId && - metadata.ownerWorkspaceId === ownerWorkspaceId && - metadata.turnId === turnId - ); - }) - ); + return this.entries.every((entry) => { + if (entry.cancelSignal?.aborted === true) { + return true; + } + const metadata = entry.muxMetadata; + return ( + isWorkspaceTurnMetadata(metadata) && + metadata.taskHandleId === taskHandleId && + metadata.ownerWorkspaceId === ownerWorkspaceId && + metadata.turnId === turnId + ); + }); } /** - * Whether the next entry continues the exact workspace turn correlation. + * Whether the next dispatchable entry continues the exact workspace turn correlation. */ hasNextWorkspaceTurnContinuation( taskHandleId: string, ownerWorkspaceId: string, turnId: string ): boolean { - const metadata = this.entries[0]?.muxMetadata; + const metadata = this.nextDispatchableEntry()?.muxMetadata; return ( isWorkspaceTurnMetadata(metadata) && metadata.taskHandleId === taskHandleId && @@ -323,7 +329,7 @@ export class MessageQueue { } /** - * FIFO head entry's cut-attribution view: its first muxMetadata plus dispatch mode. + * Next dispatchable entry's cut-attribution view: its first muxMetadata plus dispatch mode. * * Soundness of metadata-based cut attribution rests on the sealing invariant * (see class docblock): workspace-turn entries are sealed at add time and @@ -334,7 +340,7 @@ export class MessageQueue { getNextQueueCutCandidate(): | { muxMetadata: unknown; dispatchMode: QueueDispatchMode } | undefined { - const head = this.entries[0]; + const head = this.nextDispatchableEntry(); if (head == null) { return undefined; } @@ -342,13 +348,13 @@ export class MessageQueue { } /** - * Whether the next entry to dispatch is a bash-monitor wake. Wake sends are + * Whether the next dispatchable entry is a bash-monitor wake. Wake sends are * the only queued input that continues an open delegated workspace turn * (see AgentSession.inheritOpenWorkspaceTurnMetadata); any other head entry * supersedes the turn when it dispatches. */ isNextEntryBashMonitorWake(): boolean { - const muxMetadata = this.entries[0]?.muxMetadata; + const muxMetadata = this.nextDispatchableEntry()?.muxMetadata; if (typeof muxMetadata !== "object" || muxMetadata === null) return false; return (muxMetadata as Record).type === "bash-monitor-wake"; } diff --git a/src/node/services/streamManager.modelOnlyNotifications.test.ts b/src/node/services/streamManager.modelOnlyNotifications.test.ts index 93369e4db4..3d8f688f79 100644 --- a/src/node/services/streamManager.modelOnlyNotifications.test.ts +++ b/src/node/services/streamManager.modelOnlyNotifications.test.ts @@ -88,6 +88,8 @@ describe("StreamManager - model-only tool notifications", () => { lastStepUsage: undefined, lastStepProviderMetadata: undefined, toolModelUsages: [], + request: { messages: [], providerOptions: undefined }, + stepCount: 0, }; const method = Reflect.get(streamManager, "processStreamWithCleanup") as unknown; @@ -182,6 +184,8 @@ describe("StreamManager - model-only tool notifications", () => { lastStepUsage: undefined, lastStepProviderMetadata: undefined, toolModelUsages: [], + request: { messages: [], providerOptions: undefined }, + stepCount: 0, }; const method = Reflect.get(streamManager, "processStreamWithCleanup") as unknown; diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index ff51f1e4db..0024b319dc 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -8,6 +8,7 @@ import type { ProvidersConfigMap } from "@/common/orpc/types"; import { StreamEndEventSchema, ToolCallStartEventSchema } from "@/common/orpc/schemas/stream"; import type { CompletedMessagePart, + ModelFallbackProgress, ToolCallEndEvent, ToolCallExecutionStartEvent, ToolCallStartEvent, @@ -256,6 +257,7 @@ function createStreamInfoForTests( didRetryPreviousResponseIdAtStep: false, receivedTerminalEvent: false, currentStepStartIndex: 0, + stepCount: 0, stepTracker: {}, ...overrides, }; @@ -1226,9 +1228,13 @@ describe("StreamManager - stream resource scope", () => { describe("StreamManager - stopWhen configuration", () => { type StopWhenCondition = (options: { steps: unknown[] }) => boolean; type BuildStopWhenCondition = (request: { + modelString: string; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + onQueuedMessageStop?: (stop: { modelString: string; stepsRemaining: number }) => void; + stepBudget?: number; toolPolicy?: ToolPolicy; }) => StopWhenCondition[]; + const TEST_MODEL_STRING = "anthropic:claude-sonnet-4-5"; function buildStopWhenForTests(streamManager = new StreamManager(historyService)) { return getPrivateMethodForTests( @@ -1239,6 +1245,7 @@ describe("StreamManager - stopWhen configuration", () => { function requiredToolConditionForTests(toolPolicy: ToolPolicy): StopWhenCondition { const [, , requiredToolCondition] = buildStopWhenForTests()({ + modelString: TEST_MODEL_STRING, hasQueuedMessages: () => false, toolPolicy, }); @@ -1251,7 +1258,10 @@ describe("StreamManager - stopWhen configuration", () => { test("returns step-cap and queued-message conditions with no policy", () => { let queued = false; - const stopWhen = buildStopWhenForTests()({ hasQueuedMessages: () => queued }); + const stopWhen = buildStopWhenForTests()({ + modelString: TEST_MODEL_STRING, + hasQueuedMessages: () => queued, + }); expect(stopWhen).toHaveLength(3); const [maxStepCondition, queuedMessageCondition, requiredToolCondition] = stopWhen; @@ -1266,6 +1276,81 @@ describe("StreamManager - stopWhen configuration", () => { ); }); + test("queued-message stop reports itself only when no required tool completed", () => { + let queued = false; + let stopsForQueuedMessage = 0; + let stoppedModel: string | undefined; + const [, queuedMessageCondition] = buildStopWhenForTests()({ + modelString: "openai:gpt-5-fallback", + hasQueuedMessages: () => queued, + onQueuedMessageStop: ({ modelString }) => { + stopsForQueuedMessage += 1; + stoppedModel = modelString; + }, + toolPolicy: [{ regex_match: "agent_report", action: "require" }], + }); + const bashStep = stepsWithToolResult("bash", { success: true }); + + expect(queuedMessageCondition(bashStep)).toBe(false); + expect(stopsForQueuedMessage).toBe(0); + + queued = true; + expect(queuedMessageCondition(bashStep)).toBe(true); + expect(stopsForQueuedMessage).toBe(1); + // The stop names the request's own model, which is the fallback's after a model swap. + expect(stoppedModel).toBe("openai:gpt-5-fallback"); + + expect(queuedMessageCondition(stepsWithToolResult("agent_report", { success: true }))).toBe( + true + ); + expect(stopsForQueuedMessage).toBe(1); + + expect(queuedMessageCondition(stepsWithToolResult("agent_report", { success: false }))).toBe( + true + ); + expect(stopsForQueuedMessage).toBe(2); + }); + + test("queued-message stop does not report itself once the step cap is reached", () => { + let stopsForQueuedMessage = 0; + const [maxStepCondition, queuedMessageCondition] = buildStopWhenForTests()({ + modelString: TEST_MODEL_STRING, + hasQueuedMessages: () => true, + onQueuedMessageStop: () => { + stopsForQueuedMessage += 1; + }, + }); + const cappedSteps = { steps: new Array(100000).fill({}) }; + + expect(maxStepCondition(cappedSteps)).toBe(true); + expect(queuedMessageCondition(cappedSteps)).toBe(true); + expect(stopsForQueuedMessage).toBe(0); + + expect(queuedMessageCondition({ steps: new Array(99999).fill({}) })).toBe(true); + expect(stopsForQueuedMessage).toBe(1); + }); + + test("a step budget replaces the default ceiling and the cut reports what is left", () => { + const stops: number[] = []; + const [maxStepCondition, queuedMessageCondition] = buildStopWhenForTests()({ + modelString: TEST_MODEL_STRING, + hasQueuedMessages: () => true, + onQueuedMessageStop: ({ stepsRemaining }) => { + stops.push(stepsRemaining); + }, + stepBudget: 5, + }); + + expect(maxStepCondition({ steps: new Array(4).fill({}) })).toBe(false); + expect(maxStepCondition({ steps: new Array(5).fill({}) })).toBe(true); + + expect(queuedMessageCondition({ steps: new Array(2).fill({}) })).toBe(true); + expect(stops).toEqual([3]); + // At the budget the ceiling ends the turn; the cut owes nothing. + expect(queuedMessageCondition({ steps: new Array(5).fill({}) })).toBe(true); + expect(stops).toEqual([3]); + }); + const requiredToolCases: Array<{ name: string; toolPolicy: ToolPolicy; @@ -1781,6 +1866,137 @@ describe("StreamManager - fallback construction callbacks", () => { }); }); +describe("StreamManager - fallback chain continuation", () => { + const requestedModel = KNOWN_MODELS.SONNET.id; + const cutModel = KNOWN_MODELS.GPT.id; + const nextModel = KNOWN_MODELS.GEMINI_FLASH.id; + // The requested model refused and the turn was cut while running on the first fallback. + const progress: ModelFallbackProgress = { + requestedModel, + refusedModels: [requestedModel], + chain: [cutModel, nextModel], + }; + + async function runContinuationForTests( + workspaceId: string, + streams: Array<() => AsyncGenerator> + ) { + const streamManager = new StreamManager(historyService); + Reflect.set(streamManager, "tokenTracker", { + setModel: () => Promise.resolve(undefined), + countTokens: () => Promise.resolve(0), + }); + Reflect.set(streamManager, "createTempDirForStream", () => + Promise.resolve("/tmp/fallback-continuation-tempdir") + ); + Reflect.set(streamManager, "cleanupStreamTempDir", (): void => undefined); + const errorEvents: unknown[] = []; + const streamEndEvents: Array<{ + metadata?: { + model?: string; + modelFallback?: { requestedModel: string; refusedModels: string[] }; + }; + }> = []; + onTurnEngineEvent(streamManager, "error", (data) => errorEvents.push(data)); + onTurnEngineEvent(streamManager, "stream-end", (data) => { + streamEndEvents.push(data as (typeof streamEndEvents)[number]); + }); + + const messageId = `${workspaceId}-message`; + await appendPartialAssistantForTests(workspaceId, messageId, 1); + const createStreamResult = mock( + (_request: { modelFallbackProgress?: ModelFallbackProgress }) => { + const nextStream = streams.shift(); + if (nextStream == null) { + throw new Error("createStreamResult called more often than the test provided streams"); + } + return createStreamResultForTests(nextStream(), { + inputTokens: 5, + outputTokens: 3, + totalTokens: 8, + }); + } + ); + expect(Reflect.set(streamManager, "createStreamResult", createStreamResult)).toBe(true); + const prepare = mock((nextModelString: string) => + Promise.resolve( + Ok({ + model: createTestLanguageModel(`fallback-${nextModelString}`), + modelString: nextModelString, + messages: [], + system: "fallback system", + tools: undefined, + }) + ) + ); + + const result = await streamManager.startStream( + testStartOptions({ + workspaceId, + messageId, + model: createTestLanguageModel("cut-model"), + modelString: cutModel, + tools: {}, + modelFallback: { chain: progress.chain, prepare }, + modelFallbackProgress: progress, + }) + ); + expect(result.success).toBe(true); + if (!result.success) { + throw new Error("Expected the continuation stream to start"); + } + await result.data.completion; + return { errorEvents, streamEndEvents, createStreamResult, prepare }; + } + + const refusal = () => + (async function* () { + await Promise.resolve(); + yield { type: "finish", finishReason: "content-filter", rawFinishReason: "refusal" }; + })(); + const answer = () => + (async function* () { + await Promise.resolve(); + yield { type: "text-delta", text: "answer" }; + yield { type: "finish", finishReason: "stop" }; + })(); + + test("a refusal on the resumed stream moves on to the entry after the resumed model", async () => { + const run = await runContinuationForTests("fallback-continuation-refusal-workspace", [ + refusal, + answer, + ]); + + expect(run.errorEvents).toHaveLength(0); + // Not back to the chain's first entry (the resumed model itself) or a chain of its own. + expect(run.prepare.mock.calls.map((call) => call[0])).toEqual([nextModel]); + // Each request carries the chain state a further cut would report: the cut turn's at + // first, then the hop's. + expect(run.createStreamResult.mock.calls[0]?.[0].modelFallbackProgress).toEqual(progress); + expect(run.createStreamResult.mock.calls[1]?.[0].modelFallbackProgress).toEqual({ + ...progress, + refusedModels: [requestedModel, cutModel], + }); + expect(run.streamEndEvents[0]?.metadata?.model).toBe(nextModel); + expect(run.streamEndEvents[0]?.metadata?.modelFallback).toEqual({ + requestedModel, + refusedModels: [requestedModel, cutModel], + }); + }); + + test("a resumed stream that answers records the cut turn's fallback", async () => { + const run = await runContinuationForTests("fallback-continuation-answer-workspace", [answer]); + + expect(run.errorEvents).toHaveLength(0); + expect(run.prepare).not.toHaveBeenCalled(); + expect(run.streamEndEvents[0]?.metadata?.model).toBe(cutModel); + expect(run.streamEndEvents[0]?.metadata?.modelFallback).toEqual({ + requestedModel, + refusedModels: [requestedModel], + }); + }); +}); + describe("StreamManager - sequential tool execution", () => { interface Deferred { promise: Promise; @@ -2376,6 +2592,8 @@ describe("StreamManager - turn completion", () => { createStreamResult?: (request: unknown, abortController: AbortController) => unknown; sink?: (event: TurnEngineEvent) => void | Promise; events?: TurnEngineEvent[]; + toolPolicy?: ToolPolicy; + stepBudget?: number; }) { const streamManager = new StreamManager( historyService, @@ -2400,6 +2618,8 @@ describe("StreamManager - turn completion", () => { messageId: input.messageId, model: createTestLanguageModel(), providedRuntimeTempDir: "", + toolPolicy: input.toolPolicy, + stepBudget: input.stepBudget, }) ); expect(result.success).toBe(true); @@ -2407,6 +2627,83 @@ describe("StreamManager - turn completion", () => { return { streamManager, handle: result.data }; } + // The soft stop for a queued tool-end message lands right after a provider-executed tool + // result, before the loop's step-end stop conditions run; the abort reports whether that step + // already completed a required tool so the session knows the turn was over anyway. + for (const requiredToolCase of [ + { requiredTool: "web_search", output: { ok: true }, satisfied: true }, + { requiredTool: "web_search", output: { ok: false }, satisfied: false }, + { requiredTool: "agent_report", output: { ok: true }, satisfied: false }, + ]) { + test(`a queued-message soft stop reports a satisfied required tool: ${requiredToolCase.requiredTool} -> ${requiredToolCase.output.ok} is ${requiredToolCase.satisfied}`, async () => { + const workspaceId = `soft-stop-required-${requiredToolCase.requiredTool}-${requiredToolCase.output.ok}`; + const events: TurnEngineEvent[] = []; + const managerRef: { current?: StreamManager } = {}; + let releaseToolResult!: () => void; + const toolResultGate = new Promise((resolve) => { + releaseToolResult = resolve; + }); + const started = await startWithStreamResult({ + workspaceId, + messageId: "soft-stop-required-message", + stepBudget: 5, + toolPolicy: [{ regex_match: requiredToolCase.requiredTool, action: "require" }], + sink: (event) => { + events.push(event); + if (event.type === "tool-call-end") { + // AgentSession asks for the soft stop from this event, synchronously. + void managerRef.current?.stopStream(workspaceId, { + soft: true, + abortReason: "queued-message", + }); + } + }, + createStreamResult: (_request, abortController) => + createStreamResultForTests( + (async function* () { + // Armed before the tool result: the soft stop aborts while that result is handled. + const aborted = new Promise((resolve) => { + abortController.signal.addEventListener("abort", () => resolve(), { once: true }); + }); + yield { type: "start-step" }; + yield { + type: "tool-call", + toolCallId: "call-1", + toolName: "web_search", + input: { query: "x" }, + providerExecuted: true, + }; + await toolResultGate; + yield { + type: "tool-result", + toolCallId: "call-1", + toolName: "web_search", + output: requiredToolCase.output, + providerExecuted: true, + }; + await aborted; + })() + ), + }); + managerRef.current = started.streamManager; + releaseToolResult(); + + expect(await started.handle.completion).toEqual({ + status: "aborted", + abortReason: "queued-message", + }); + const abort = events.find((event) => event.type === "stream-abort"); + expect(abort?.type).toBe("stream-abort"); + if (abort?.type !== "stream-abort") throw new Error("Expected a stream-abort event"); + expect(abort.metadata?.requiredToolSatisfied).toBe( + requiredToolCase.satisfied ? true : undefined + ); + // The cut's remainder rides on the committed partial for a startup retry after a crash. + const partial = await historyService.readPartial(workspaceId); + expect(partial?.metadata?.stepsRemaining).toBe(4); + }); + } + test("pre-start failures return Err while successful startup owns an aborted completion", async () => { const streamManager = new StreamManager(historyService); const model = createTestLanguageModel(); @@ -2750,6 +3047,89 @@ describe("StreamManager - Concurrent Stream Prevention", () => { } }); + test("refuses registration when the caller's admission probe turns stale during setup", async () => { + const workspaceId = "test-workspace-refuse-before-create"; + + let createCalled = false; + let streamStartEmitted = false; + let refused = false; + + onTurnEngineEvent(streamManager, "stream-start", () => { + streamStartEmitted = true; + }); + Reflect.set(streamManager, "createTempDirForStream", (): Promise => { + // A goal Pause lands during startup I/O: nothing aborts the signal, only the probe knows. + refused = true; + return Promise.resolve("/tmp/mock-stream-temp"); + }); + Reflect.set(streamManager, "cleanupStreamTempDir", (): void => undefined); + Reflect.set(streamManager, "createStreamAtomically", (): never => { + createCalled = true; + throw new Error("createStreamAtomically should not be called"); + }); + + const result = await streamManager.startStream( + testStartOptions({ + workspaceId, + messageId: "test-msg-refuse", + model: createTestLanguageModel(), + runtime, + refuseStreamStart: () => refused, + tools: {}, + }) + ); + expect(result.success).toBe(true); + if (!result.success) throw new Error("Expected aborted startup handle"); + expect(await result.data.completion).toEqual({ status: "aborted", abortReason: "startup" }); + expect(createCalled).toBe(false); + expect(streamStartEmitted).toBe(false); + expect(streamManager.isStreaming(workspaceId)).toBe(false); + }); + + test("refuses processing when the admission probe turns stale during the envelope write", async () => { + const workspaceId = "test-workspace-refuse-after-construct"; + + let processCalled = false; + let streamStartEmitted = false; + let refused = false; + + onTurnEngineEvent(streamManager, "stream-start", () => { + streamStartEmitted = true; + }); + Reflect.set( + streamManager, + "createTempDirForStream", + (): Promise => Promise.resolve("/tmp/mock-stream-temp") + ); + Reflect.set(streamManager, "cleanupStreamTempDir", (): void => undefined); + Reflect.set(streamManager, "processStreamWithCleanup", (): Promise => { + processCalled = true; + return Promise.resolve(); + }); + + const result = await streamManager.startStream( + testStartOptions({ + workspaceId, + messageId: "test-msg-refuse-after-construct", + model: createTestLanguageModel(), + runtime, + refuseStreamStart: () => refused, + // The stream is registered by now; a goal Pause lands while the envelope is written. + onStreamConstructed: () => { + refused = true; + return Promise.resolve(); + }, + tools: {}, + }) + ); + expect(result.success).toBe(true); + if (!result.success) throw new Error("Expected aborted startup handle"); + expect(await result.data.completion).toEqual({ status: "aborted", abortReason: "startup" }); + expect(processCalled).toBe(false); + expect(streamStartEmitted).toBe(false); + expect(streamManager.isStreaming(workspaceId)).toBe(false); + }); + test("should honor abortSignal before atomic stream creation", async () => { const workspaceId = "test-workspace-abort-before-create"; @@ -3566,6 +3946,100 @@ describe("StreamManager - empty stream completions", () => { expect(swappedRequest.system).toBe("fallback system"); }); + test("a fallback hop runs under the refused stream's remaining step budget, none once spent", async () => { + const runRefusalWithFallback = async (stepBudget: number) => { + const streamManager = new StreamManager(historyService); + const errorEvents: unknown[] = []; + onTurnEngineEvent(streamManager, "error", (data) => errorEvents.push(data)); + Reflect.set(streamManager, "tokenTracker", { + setModel: () => Promise.resolve(undefined), + countTokens: () => Promise.resolve(0), + }); + + const workspaceId = `fallback-step-budget-${stepBudget}-workspace`; + const messageId = "fallback-step-budget-message"; + await appendPartialAssistantForTests(workspaceId, messageId, 1); + const processStreamWithCleanup = getProcessStreamWithCleanupForTests(streamManager); + + const createStreamResult = mock((_request: { stepBudget?: number }) => + createStreamResultForTests( + (async function* () { + await Promise.resolve(); + yield { type: "text-delta", text: "fallback answer" }; + yield { type: "finish", finishReason: "stop" }; + })(), + { inputTokens: 5, outputTokens: 3, totalTokens: 8 } + ) + ); + expect(Reflect.set(streamManager, "createStreamResult", createStreamResult)).toBe(true); + const prepare = mock((nextModelString: string) => + Promise.resolve( + Ok({ + model: createTestLanguageModel("fallback-model"), + modelString: nextModelString, + messages: [], + system: "fallback system", + tools: {}, + thinkingLevel: "off", + }) + ) + ); + + const startTime = Date.now() - 250; + const streamInfo = createStreamInfoForTests({ + streamResult: createStreamResultForTests( + (async function* () { + await Promise.resolve(); + // The refused step is a step the turn spent. + yield { type: "start-step" }; + yield { + type: "finish-step", + usage: { inputTokens: 30, outputTokens: 0, totalTokens: 30 }, + }; + yield { type: "finish", finishReason: "content-filter", rawFinishReason: "refusal" }; + })(), + { inputTokens: 30, outputTokens: 0, totalTokens: 30 } + ), + messageId, + startTime, + lastPartTimestamp: startTime, + model: KNOWN_MODELS.SONNET.id, + metadataModel: KNOWN_MODELS.SONNET.id, + historySequence: 1, + initialMetadata: { agentId: "plan" }, + runtime, + request: { + model: createTestLanguageModel("refused-model"), + messages: [], + providerOptions: undefined, + stepBudget, + }, + modelFallback: { + options: { chain: [KNOWN_MODELS.GPT.id], prepare }, + requestedModel: KNOWN_MODELS.SONNET.id, + refusedModels: [], + original: { maxOutputTokens: undefined }, + }, + }); + + await processStreamWithCleanup.call(streamManager, workspaceId, streamInfo, 1); + return { errorEvents, prepare, createStreamResult }; + }; + + // Three steps allowed and one spent on the refusal: the hop's own loop gets the other two. + const hop = await runRefusalWithFallback(3); + expect(hop.errorEvents).toHaveLength(0); + expect(hop.createStreamResult).toHaveBeenCalledTimes(1); + expect(hop.createStreamResult.mock.calls[0]?.[0].stepBudget).toBe(2); + + // The refusal spent the last step: the ceiling ended the turn, so no hop is bought. + const spent = await runRefusalWithFallback(1); + expect(spent.prepare).not.toHaveBeenCalled(); + expect(spent.createStreamResult).not.toHaveBeenCalled(); + expect(spent.errorEvents).toHaveLength(1); + expect(spent.errorEvents[0]).toMatchObject({ errorType: "model_refusal" }); + }); + test("partial refusal with a configured fallback continues from cloned partial output", async () => { const streamManager = new StreamManager(historyService); const errorEvents: unknown[] = []; @@ -5131,6 +5605,7 @@ describe("StreamManager - previousResponseId recovery", () => { stepTracker: { latestMessages: stepMessages }, didRetryPreviousResponseIdAtStep: false, currentStepStartIndex: 1, + stepCount: 1, request: { model, messages: [{ role: "user", content: "original" }], diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index d12db74e0a..c31eea5f8a 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -35,6 +35,7 @@ import type { ReasoningDeltaEvent, ReasoningEndEvent, CompletedMessagePart, + ModelFallbackProgress, WorkflowRunAttachedEvent, } from "@/common/types/stream"; @@ -114,6 +115,8 @@ const EMPTY_STREAM_OUTPUT_ERROR_MESSAGE = "The model ended the stream before producing any assistant-visible output. This usually means the upstream stream was dropped rather than completed normally. Xum will retry automatically when possible, and if retries keep failing you should try again or switch models."; const MAX_EMPTY_STREAM_RECOVERY_ATTEMPTS = 1; +/** Hard per-stream step cap; the practical limit is the model's own finish. */ +const MAX_STREAM_STEPS = 100_000; /** Drop reason for a partial that never reaches chat.jsonl. */ type DroppedStreamSource = "aborted_stream" | "errored_stream"; @@ -204,7 +207,21 @@ export type TurnEngineEventSink = (event: TurnEngineEvent) => void | Promise boolean; + onQueuedMessageStop?: (stop: QueuedMessageStop) => void; + stepBudget?: number; + modelFallbackProgress?: ModelFallbackProgress; headers?: Record; onChunk?: StreamTextOnChunk; onStepMessages?: (messages: ModelMessage[]) => void; @@ -262,6 +293,8 @@ export interface TurnExecutionOptions extends StreamRequestOptions { runtime: Runtime; messageId: string; abortSignal?: AbortSignal; + /** Startup-only admission probe; a true answer at registration time refuses the stream like an abort. */ + refuseStreamStart?: () => boolean; initialMetadata?: Partial; providedStreamToken?: StreamToken; workspaceName?: string; @@ -280,6 +313,8 @@ interface StepMessageTracker { } interface StreamRequestConfig { model: LanguageModel; + /** Canonical model string of `model` (the fallback's once a fallback request replaces this). */ + modelString: string; messages: ModelMessage[]; /** Provider-ready system instructions from TurnContextAssembler. */ system?: string | SystemModelMessage; @@ -290,6 +325,23 @@ interface StreamRequestConfig { maxOutputTokens?: number; streamCallSettings?: Omit; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + /** + * Invoked when the loop stops on behalf of a queued tool-end message (and not + * because a required tool completed). The session uses it to resume the turn + * if that queued message is later withdrawn instead of starting a turn. + */ + onQueuedMessageStop?: (stop: QueuedMessageStop) => void; + /** + * Step ceiling for this stream instead of MAX_STREAM_STEPS. A stream resuming a turn cut + * for a queued message inherits the cut stream's remaining steps, so cut plus resumes + * share one turn's ceiling; without it, every resume would restart the full cap. + */ + stepBudget?: number; + /** + * Fallback chain state this request runs under (the fallback's request replaces the + * original's), reported at a queued-message cut so the resumed stream continues the chain. + */ + modelFallbackProgress?: ModelFallbackProgress; /** Optional hook for callers that need chunk-level visibility during streaming. */ onChunk?: StreamTextOnChunk; /** Optional hook for callers that need the live prepared step transcript. */ @@ -409,6 +461,19 @@ export interface ModelFallbackOptions { ) => Promise>; } +/** Snapshot of a stream's fallback chain state for a resumed stream to continue from. */ +function modelFallbackProgressOf( + state: WorkspaceStreamInfo["modelFallback"] +): ModelFallbackProgress | undefined { + return state == null + ? undefined + : { + requestedModel: state.requestedModel, + refusedModels: [...state.refusedModels], + chain: state.options.chain, + }; +} + function isKnownProviderName(provider: string): provider is keyof typeof PROVIDER_DEFINITIONS { return Object.hasOwn(PROVIDER_DEFINITIONS, provider); } @@ -595,6 +660,29 @@ function zeroTokenUsage(): LanguageModelV2Usage { return { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; } +/** + * Completion-tool success check: completion/routing tools use explicit success/ok markers + * (agent_report, propose_plan). When a marker is present, respect it (success:false means the tool + * should be retried, so the turn goes on). When no marker is present (MCP tools, arbitrary + * required tools), treat non-null object results as successful completion unless error-shaped. + */ +function isSuccessfulRequiredToolOutput(output: unknown): boolean { + if (typeof output !== "object" || output === null) { + return false; + } + const parsedOutput = output as Record; + if ("success" in parsedOutput) { + return parsedOutput.success === true; + } + if ("ok" in parsedOutput) { + return parsedOutput.ok === true; + } + if (parsedOutput.error != null || parsedOutput.isError === true) { + return false; + } + return true; +} + function hasIncompleteToolCallPart(parts: CompletedMessagePart[]): boolean { return parts.some((part) => part.type === "dynamic-tool" && part.state !== "output-available"); } @@ -623,6 +711,16 @@ interface WorkspaceStreamInfo { // original start timestamp even after they gain output. toolCompletionTimestamps: Map; + // Steps started by the current SDK loop, the in-progress one included: the budget left at + // an abort counts a step the model already began. Reset when the loop restarts under + // request.stepBudget (restartStepBudget). + stepCount: number; + + // A required completion tool (request.toolPolicy) succeeded in the step in progress. The + // step-end stop condition would end the turn on it, so a queued-message soft stop that lands + // first (after a provider-executed tool result) owes the turn no continuation. + requiredToolSatisfied?: boolean; + // Workflow tools can create the durable run before their stream part is stored. Keep the exact // attachment and apply it as soon as the matching dynamic-tool part lands. pendingWorkflowRunAttachments: Map; @@ -1868,12 +1966,21 @@ export class StreamManager { streamInfo ); + // A queued-message cut owes the turn a continuation under what it left of the ceiling. The + // committed partial is that remainder's only durable carrier: a process exit before the + // in-memory resume starts leaves startup recovery to retry the row from history. + const stepsRemaining = + abortReason === "queued-message" ? this.remainingStepBudget(streamInfo) : undefined; + // Stamp the aborted turn's usage onto the partial message BEFORE emitting // stream-abort (whose handler commits the partial to chat.jsonl). Analytics // prices history rows from metadata.usage, so without this every // interrupted turn — user Esc, queued tool-end preemption, monitor wakes — // would ingest as $0 even though the provider billed all completed steps. - if (!abandonPartial && (usage !== undefined || streamInfo.toolModelUsages.length > 0)) { + if ( + !abandonPartial && + (usage !== undefined || streamInfo.toolModelUsages.length > 0 || stepsRemaining !== undefined) + ) { try { await this.awaitPendingPartialWrite(streamInfo); const partialMessage = this.buildPartialAssistantMessage(streamInfo, { @@ -1886,6 +1993,7 @@ export class StreamManager { ...(streamInfo.toolModelUsages.length > 0 ? { toolModelUsages: streamInfo.toolModelUsages.map(clonePersistedToolModelUsage) } : {}), + ...(stepsRemaining !== undefined ? { stepsRemaining } : {}), }, }); await this.historyService.writePartial(workspaceId as string, partialMessage); @@ -1931,7 +2039,17 @@ export class StreamManager { const abortDelivery = this.emitStreamAbort( workspaceId, streamInfo.messageId, - { usage, contextUsage, duration, providerMetadata, contextProviderMetadata }, + { + usage, + contextUsage, + duration, + providerMetadata, + contextProviderMetadata, + model: streamInfo.model, + stepsRemaining: this.remainingStepBudget(streamInfo), + modelFallbackProgress: modelFallbackProgressOf(streamInfo.modelFallback), + ...(streamInfo.requiredToolSatisfied === true ? { requiredToolSatisfied: true } : {}), + }, abortReason, abandonPartial, streamInfo.initialMetadata?.acpPromptId @@ -2069,6 +2187,9 @@ export class StreamManager { callSettingsOverrides, toolPolicy, hasQueuedMessages, + onQueuedMessageStop, + stepBudget, + modelFallbackProgress, headers, onChunk, onStepMessages, @@ -2108,6 +2229,7 @@ export class StreamManager { return { model, + modelString, messages, system, // Keep provider-level parallel tool planning enabled, but serialize sibling @@ -2119,6 +2241,9 @@ export class StreamManager { streamCallSettings: Object.keys(streamCallSettings).length > 0 ? streamCallSettings : undefined, hasQueuedMessages, + onQueuedMessageStop, + stepBudget, + modelFallbackProgress, onChunk, onStepMessages, toolPolicy, @@ -2131,31 +2256,17 @@ export class StreamManager { } private createStopWhenCondition( - request: Pick + request: Pick< + StreamRequestConfig, + | "hasQueuedMessages" + | "onQueuedMessageStop" + | "toolPolicy" + | "modelString" + | "stepBudget" + | "modelFallbackProgress" + > ): Array> { - // Completion-tool stop check: completion/routing tools use explicit - // success/ok markers (agent_report, propose_plan). - // When a marker is present, respect it — success:false means the tool - // should be retried, so don't stop. When no marker is present (e.g., - // MCP tools, arbitrary required tools), treat non-null object results - // as successful completion unless the result is error-shaped. - const isSuccessfulOutput = (output: unknown): boolean => { - if (typeof output !== "object" || output === null) { - return false; - } - const parsedOutput = output as Record; - if ("success" in parsedOutput) { - return parsedOutput.success === true; - } - if ("ok" in parsedOutput) { - return parsedOutput.ok === true; - } - if (parsedOutput.error != null || parsedOutput.isError === true) { - return false; - } - return true; - }; - + const stepBudget = request.stepBudget ?? MAX_STREAM_STEPS; const requiredPatterns = buildRequiredToolPatterns(request.toolPolicy); const hasSuccessfulRequiredToolResult: ReturnType = ({ steps }) => { @@ -2167,19 +2278,33 @@ export class StreamManager { lastStep?.toolResults?.some( (toolResult) => requiredPatterns.some((pattern) => pattern.test(toolResult.toolName)) && - isSuccessfulOutput(toolResult.output) + isSuccessfulRequiredToolOutput(toolResult.output) ) ?? false ); }; - return [ - stepCountIs(100000), - // The SDK evaluates stop conditions only after every sibling tool result in the - // model's current step settles. Do not move this to individual tool-call-end events: - // that would abort the remaining calls the model emitted in the same batch. - () => request.hasQueuedMessages?.("tool-end") ?? false, - hasSuccessfulRequiredToolResult, - ]; + // The SDK evaluates stop conditions only after every sibling tool result in the + // model's current step settles. Do not move this to individual tool-call-end events: + // that would abort the remaining calls the model emitted in the same batch. + const hasQueuedToolEndMessage: ReturnType = (state) => { + if (!(request.hasQueuedMessages?.("tool-end") ?? false)) { + return false; + } + // The step cap and a successful required tool result each end the turn on their + // own; only a stop made purely for the queued message may need resuming later. A cut + // spends at least one step, so a chain of cuts and resumes always runs the budget down. + const stepsSpent = Math.max(1, state.steps.length); + if (stepsSpent < stepBudget && !hasSuccessfulRequiredToolResult(state)) { + request.onQueuedMessageStop?.({ + modelString: request.modelString, + stepsRemaining: stepBudget - stepsSpent, + modelFallbackProgress: request.modelFallbackProgress, + }); + } + return true; + }; + + return [stepCountIs(stepBudget), hasQueuedToolEndMessage, hasSuccessfulRequiredToolResult]; } /** @@ -2377,8 +2502,23 @@ export class StreamManager { } = options; const stepTracker: StepMessageTracker = {}; const metadataModel = this.resolveMetadataModel(modelString, options.providersConfigSnapshot); + // A stream continuing a cut turn picks the chain up where the cut left it: the requested + // model and refusals are the cut turn's, and a refusal here moves on to the next entry. + const carried = options.modelFallbackProgress; + const modelFallbackState: WorkspaceStreamInfo["modelFallback"] = + modelFallback && modelFallback.chain.length > 0 + ? { + options: modelFallback, + requestedModel: carried?.requestedModel ?? normalizeToCanonical(modelString), + refusedModels: [...(carried?.refusedModels ?? [])], + // Pre-wrap inputs (NOT request.maxOutputTokens, which may already + // carry call-settings overrides for the original model). + original: { maxOutputTokens }, + } + : undefined; const request = this.buildStreamRequestConfig({ ...options, + modelFallbackProgress: modelFallbackProgressOf(modelFallbackState), onToolExecutionStart: (toolCallId) => this.handleToolExecutionStart(workspaceId, messageId, toolCallId), }); @@ -2405,28 +2545,29 @@ export class StreamManager { startTime, lastPartTimestamp: startTime, toolCompletionTimestamps: new Map(), + stepCount: 0, pendingWorkflowRunAttachments: new Map(), pendingNestedCalls: new Map(), pendingToolExecutionStarts: new Map(), model: modelString, metadataModel, thinkingLevel, - initialMetadata, + // The resumed message answers on a fallback because the cut turn's requested model + // refused; record that as the swap would have. + initialMetadata: + modelFallbackState != null && modelFallbackState.refusedModels.length > 0 + ? { + ...initialMetadata, + modelFallback: { + requestedModel: modelFallbackState.requestedModel, + refusedModels: [...modelFallbackState.refusedModels], + }, + } + : initialMetadata, toolModelUsages: [], didRetryPreviousResponseIdAtStep: false, didRetryAfterEmptyOutput: false, - ...(modelFallback && modelFallback.chain.length > 0 - ? { - modelFallback: { - options: modelFallback, - requestedModel: normalizeToCanonical(modelString), - refusedModels: [], - // Pre-wrap inputs (NOT request.maxOutputTokens, which may already - // carry call-settings overrides for the original model). - original: { maxOutputTokens }, - }, - } - : {}), + ...(modelFallbackState != null ? { modelFallback: modelFallbackState } : {}), stepTracker, receivedTerminalEvent: false, currentStepStartIndex: 0, @@ -2559,6 +2700,15 @@ export class StreamManager { output, providerExecuted ); + if ( + streamInfo.requiredToolSatisfied !== true && + isSuccessfulRequiredToolOutput(output) && + buildRequiredToolPatterns(streamInfo.request.toolPolicy).some((pattern) => + pattern.test(toolName) + ) + ) { + streamInfo.requiredToolSatisfied = true; + } await this.checkSoftCancelStream(workspaceId, streamInfo); } @@ -3095,6 +3245,14 @@ export class StreamManager { }; } + const stepBudget = this.restartStepBudget(streamInfo); + if (stepBudget == null) { + return { + kind: "terminal", + terminalNote: "Model fallback was skipped because the turn's step budget is spent.", + }; + } + const workspaceLog = this.getWorkspaceLogger(workspaceId, streamInfo); // A throw out of prepare() must not escape to the generic stream-error path: // it would be categorized as a retryable api/unknown error and re-enter the @@ -3156,6 +3314,9 @@ export class StreamManager { callSettingsOverrides: prepared.data.callSettingsOverrides, toolPolicy: streamInfo.request.toolPolicy, hasQueuedMessages: streamInfo.request.hasQueuedMessages, + onQueuedMessageStop: streamInfo.request.onQueuedMessageStop, + stepBudget, + modelFallbackProgress: modelFallbackProgressOf(fallbackState), headers: prepared.data.headers, onChunk: streamInfo.request.onChunk, onStepMessages: streamInfo.request.onStepMessages, @@ -3246,12 +3407,23 @@ export class StreamManager { // refused model (e.g. an OpenAI WS transport socket) would leak per hop. runLanguageModelCleanup(streamInfo.request.model); streamInfo.request = nextRequest; + streamInfo.stepCount = 0; streamInfo.streamResult = nextStreamResult; await this.tokenTracker.setModel(streamInfo.model, streamInfo.metadataModel); return { kind: "swapped" }; } + /** + * Step ceiling for an SDK loop restarted under this stream (fallback swap, same-model + * retry): the new loop counts its steps from zero, so it inherits what the stream has + * left rather than a fresh ceiling. Undefined when the budget is spent. + */ + private restartStepBudget(streamInfo: WorkspaceStreamInfo): number | undefined { + const remaining = (streamInfo.request.stepBudget ?? MAX_STREAM_STEPS) - streamInfo.stepCount; + return remaining > 0 ? remaining : undefined; + } + private async handleTruncatedStreamCompletion( workspaceId: WorkspaceId, streamInfo: WorkspaceStreamInfo @@ -3302,6 +3474,11 @@ export class StreamManager { return false; } + const stepBudget = this.restartStepBudget(streamInfo); + if (stepBudget == null) { + return false; + } + const workspaceLog = this.getWorkspaceLogger(workspaceId, streamInfo); workspaceLog.warn("Retrying stream after empty-output completion", { messageId: streamInfo.messageId, @@ -3317,6 +3494,8 @@ export class StreamManager { workspaceLog, }); streamInfo.currentStepStartIndex = 0; + streamInfo.request = { ...streamInfo.request, stepBudget }; + streamInfo.stepCount = 0; streamInfo.streamResult = this.createStreamResult( streamInfo.request, streamInfo.abortController, @@ -3372,6 +3551,8 @@ export class StreamManager { switch (part.type) { case "start-step": { streamInfo.currentStepStartIndex = streamInfo.parts.length; + streamInfo.stepCount += 1; + streamInfo.requiredToolSatisfied = false; break; } @@ -4108,7 +4289,21 @@ export class StreamManager { const errorPayload = this.buildStreamErrorPayload(streamInfo, error); const persistedPayload = await this.persistStreamError(workspaceId, streamInfo, errorPayload); - streamInfo.terminalCompletion = { status: "failed", streamError: persistedPayload }; + streamInfo.terminalCompletion = { + status: "failed", + streamError: persistedPayload, + stepsRemaining: this.remainingStepBudget(streamInfo), + modelString: streamInfo.model, + modelFallbackProgress: modelFallbackProgressOf(streamInfo.modelFallback), + }; + } + + /** Steps left under the stream's ceiling when it ends early; the step it was in is spent. */ + private remainingStepBudget(streamInfo: WorkspaceStreamInfo): number { + return Math.max( + 0, + (streamInfo.request.stepBudget ?? MAX_STREAM_STEPS) - Math.max(1, streamInfo.stepCount) + ); } private buildStreamErrorPayload( @@ -4479,6 +4674,11 @@ export class StreamManager { return false; } + const stepBudget = this.restartStepBudget(streamInfo); + if (stepBudget == null) { + return false; + } + const workspaceLog = this.getWorkspaceLogger(workspaceId, streamInfo); this.recordLostResponseIdIfApplicable(workspaceId, error, streamInfo, workspaceLog); @@ -4508,7 +4708,9 @@ export class StreamManager { ...streamInfo.request, ...(stepMessages ? { messages: stepMessages } : {}), providerOptions, + stepBudget, }; + streamInfo.stepCount = 0; streamInfo.streamResult = this.createStreamResult( streamInfo.request, streamInfo.abortController, @@ -4732,6 +4934,7 @@ export class StreamManager { runtime, messageId, abortSignal, + refuseStreamStart, providedStreamToken, providedRuntimeTempDir, onStreamConstructed, @@ -4823,7 +5026,9 @@ export class StreamManager { ) ); - if (streamAbortController.signal.aborted) { + // The caller's pull-based admission probe (goal state) has no signal to abort; this is + // its last read before the stream becomes real. + if (streamAbortController.signal.aborted || refuseStreamStart?.() === true) { return settleStartupAbort(); } @@ -4859,10 +5064,12 @@ export class StreamManager { // stream may already occupy this workspace's slot. Launching // processing now would emit stream-start after the abort and its // cleanup would later delete that replacement. Bail out; the finally - // block releases this never-processed stream's resources. + // block releases this never-processed stream's resources. The caller's + // admission probe has no signal to abort with, so it is re-read here too. if ( streamAbortController.signal.aborted || - this.workspaceStreams.get(typedWorkspaceId) !== streamInfo + this.workspaceStreams.get(typedWorkspaceId) !== streamInfo || + refuseStreamStart?.() === true ) { if (this.workspaceStreams.get(typedWorkspaceId) === streamInfo) { this.workspaceStreams.delete(typedWorkspaceId); @@ -5369,7 +5576,13 @@ export class StreamManager { }); // Debug-injected failures bypass handleStreamFailure, so record the failed // completion here or cleanup would never settle the turn handle. - streamInfo.terminalCompletion = { status: "failed", streamError: persistedPayload }; + streamInfo.terminalCompletion = { + status: "failed", + streamError: persistedPayload, + stepsRemaining: this.remainingStepBudget(streamInfo), + modelString: streamInfo.model, + modelFallbackProgress: modelFallbackProgressOf(streamInfo.modelFallback), + }; // Wait for the stream processing to complete (cleanup) await streamInfo.processingPromise; diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 29715309dd..c75d5f86a1 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -347,8 +347,7 @@ describe("TaskService", () => { isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; - hasPendingBashMonitorWakeContinuation?: ReturnType; - hasPendingWorkspaceTurnContinuation?: ReturnType; + claimWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; waitForPendingStreamErrorRecoveryDecision?: ReturnType; @@ -12474,7 +12473,7 @@ describe("TaskService", () => { reason: "timed out", }); - expect(clearQueue).toHaveBeenCalledWith(childTaskId); + expect(clearQueue).toHaveBeenCalledWith(childTaskId, { hardStop: true }); expect(stopStream).toHaveBeenCalledWith(childTaskId, { abandonPartial: true, abortReason: "system", @@ -12792,8 +12791,8 @@ describe("TaskService", () => { const interruptedTaskIds = await taskService.terminateAllDescendantAgentTasks(rootWorkspaceId); expect(interruptedTaskIds).toEqual([childTaskId, parentTaskId]); - expect(clearQueue).toHaveBeenNthCalledWith(1, childTaskId); - expect(clearQueue).toHaveBeenNthCalledWith(2, parentTaskId); + expect(clearQueue).toHaveBeenNthCalledWith(1, childTaskId, { hardStop: true }); + expect(clearQueue).toHaveBeenNthCalledWith(2, parentTaskId, { hardStop: true }); expect(stopStream).toHaveBeenNthCalledWith( 1, childTaskId, @@ -24366,11 +24365,11 @@ describe("TaskService", () => { // A queued bash-monitor wake cuts the correlated stream at a tool boundary // (finishReason "tool-calls") while the child seamlessly continues the // same turn — the handle must stay running. - const hasPendingBashMonitorWakeContinuation = mock( + const claimWorkspaceTurnContinuation = mock( (workspaceId: string) => workspaceId === "childworkspace" ); const { parentId, taskService } = await startWorkspaceTurnForTest({ - hasPendingBashMonitorWakeContinuation, + claimWorkspaceTurnContinuation, }); const internal = taskService as unknown as { handleStreamEnd: (event: StreamEndEvent) => Promise; @@ -24393,6 +24392,12 @@ describe("TaskService", () => { const running = await workspaceTurnSnapshot(taskService, parentId); expect(running).toMatchObject({ status: "running", workspaceId: "childworkspace" }); expect(running?.error).toBeUndefined(); + // The claim is bound to the exact cut it settles. + expect(claimWorkspaceTurnContinuation).toHaveBeenCalledWith( + "childworkspace", + correlation, + "msg_queue_cut" + ); // The continuation stream inherits the correlation metadata (see // AgentSession.inheritOpenWorkspaceTurnMetadata); its terminal stream-end @@ -24418,8 +24423,41 @@ describe("TaskService", () => { }); }); + test("workspace-turn continuation admission tracks the handle and later stops", async () => { + const { parentId, taskService } = await startWorkspaceTurnForTest(); + const correlation = workspaceTurnMuxMetadata(parentId); + + // Running handle on this workspace: admitted, and the probe stays fresh until a stop lands. + const admitted = await taskService.getWorkspaceTurnContinuationAdmission( + "childworkspace", + correlation + ); + expect(admitted.admissible).toBe(true); + expect(admitted.admissionStale()).toBe(false); + + // A stop on the workspace after the read (interruptWorkspaceTurn bumps the stop epoch inside + // its settlement boundary) turns the earlier probe stale and refuses a fresh read. + const stopped = await workspaceTurnManagerFor(taskService).interruptWorkspaceTurn( + parentId, + correlation.taskHandleId + ); + expect(stopped.success).toBe(true); + expect(admitted.admissionStale()).toBe(true); + const refused = await taskService.getWorkspaceTurnContinuationAdmission( + "childworkspace", + correlation + ); + expect(refused.admissible).toBe(false); + + // A different workspace or turn never matches the handle. + expect( + (await taskService.getWorkspaceTurnContinuationAdmission("otherworkspace", correlation)) + .admissible + ).toBe(false); + }); + test("nested agent progress preserves workspace-turn correlation", async () => { - const hasPendingWorkspaceTurnContinuation = mock( + const claimWorkspaceTurnContinuation = mock( ( workspaceId: string, metadata: { taskHandleId: string; ownerWorkspaceId: string; turnId: string } @@ -24429,7 +24467,7 @@ describe("TaskService", () => { metadata.turnId === "turn" ); const { config, parentId, taskService, workspaceMocks } = await startWorkspaceTurnForTest({ - hasPendingWorkspaceTurnContinuation, + claimWorkspaceTurnContinuation, }); const correlation = workspaceTurnMuxMetadata(parentId); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index b4d5ab7706..a77c0e32dc 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -5452,7 +5452,7 @@ export class TaskService implements AgentTaskIntegration { }); } - const clearQueueResult = this.workspaceService.clearQueue(id); + const clearQueueResult = this.workspaceService.clearQueue(id, { hardStop: true }); if (!clearQueueResult.success) { log.debug("stopDescendantAgentTask: clearQueue failed", { taskId: id, @@ -5931,7 +5931,7 @@ export class TaskService implements AgentTaskIntegration { // Best-effort: clear queue first. AgentSession stream-end cleanup auto-flushes // queued messages, so descendants must not keep pending input after a hard interrupt. try { - const clearQueueResult = this.workspaceService.clearQueue(id); + const clearQueueResult = this.workspaceService.clearQueue(id, { hardStop: true }); if (!clearQueueResult.success) { log.debug("terminateAllDescendantAgentTasks: clearQueue failed", { taskId: id, @@ -8281,7 +8281,7 @@ export class TaskService implements AgentTaskIntegration { return; } try { - const clearQueueResult = this.workspaceService.clearQueue(taskId); + const clearQueueResult = this.workspaceService.clearQueue(taskId, { hardStop: true }); if (!clearQueueResult.success) { log.debug("failAgentTaskForHardTimeout: clearQueue failed", { taskId, @@ -9636,6 +9636,43 @@ export class TaskService implements AgentTaskIntegration { return blocking; } + async settleWorkspaceTurnContinuationFailure( + workspaceId: string, + muxMetadata: Extract, + status: "interrupted" | "error", + error: string + ): Promise { + await this.getWorkspaceTurnManager().settleWorkspaceTurnContinuationFailure( + workspaceId, + muxMetadata, + status, + error + ); + } + + async getWorkspaceTurnContinuationAdmission( + workspaceId: string, + muxMetadata: Extract + ): Promise<{ admissible: boolean; admissionStale: () => boolean }> { + // Every stop on the workspace (interruptWorkspaceTurn, task hard-stop cascades) bumps the + // epoch synchronously inside its settlement boundary, before the store write lands. + const stopEpoch = this.getWorkspaceStopEpoch(workspaceId); + const record = await this.getWorkspaceTurnManager().getWorkspaceTurnRecord( + muxMetadata.ownerWorkspaceId, + muxMetadata.taskHandleId + ); + const admissible = + record?.workspaceId === workspaceId && + record.turnId === muxMetadata.turnId && + isActiveWorkspaceTurnTaskStatus(record.status) && + !this.isWorkspaceStopInProgress(workspaceId); + return { + admissible, + admissionStale: () => + this.getWorkspaceStopEpoch(workspaceId) !== stopEpoch || + this.isWorkspaceStopInProgress(workspaceId), + }; + } async noteWorkspaceUnarchived(workspaceId: string): Promise { assert(workspaceId.length > 0, "noteWorkspaceUnarchived requires workspaceId"); // Archived owners park workflow terminal wakes unsettled (the drain drops the in-memory diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index 5b81af0977..48a408caed 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -14,8 +14,7 @@ export function makeWorkspaceHostFake(overrides: Partial = {}): W hasQueuedMessages: () => false, hasPendingQueuedOrPreparingTurn: () => false, hasPendingAutoRetry: () => false, - hasPendingBashMonitorWakeContinuation: () => false, - hasPendingWorkspaceTurnContinuation: () => false, + claimWorkspaceTurnContinuation: () => false, hasQueuedWorkspaceTurn: () => false, removeQueuedWorkspaceTurn: () => Ok(true), removeQueuedMessagesByDedupeKeyPrefix: () => Ok(0), @@ -75,6 +74,9 @@ export function makeAgentTaskIntegrationFake( latchHardInterruptCascade: () => undefined, terminateAllDescendantAgentTasks: () => Promise.resolve([]), noteWorkspaceUnarchived: () => Promise.resolve(), + settleWorkspaceTurnContinuationFailure: () => Promise.resolve(), + getWorkspaceTurnContinuationAdmission: () => + Promise.resolve({ admissible: true, admissionStale: () => false }), ...overrides, }; } diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index ecd8810187..930c88d053 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -374,7 +374,10 @@ export interface WorkspaceTurnHost { options: SendMessageOptions, internal?: { allowQueuedAgentTask?: boolean; agentInitiated?: boolean } ): Promise>; - clearQueue(workspaceId: string, options?: { cancelReason?: string }): Result; + clearQueue( + workspaceId: string, + options?: { cancelReason?: string; hardStop?: boolean } + ): Result; replaceHistory( workspaceId: string, summaryMessage: MuxMessage, @@ -399,10 +402,10 @@ export interface TurnAdmissionHost { hasQueuedMessages(workspaceId: string, dispatchMode?: "tool-end" | "turn-end"): boolean; hasPendingQueuedOrPreparingTurn(workspaceId: string): boolean; hasPendingAutoRetry(workspaceId: string): boolean; - hasPendingBashMonitorWakeContinuation(workspaceId: string): boolean; - hasPendingWorkspaceTurnContinuation( + claimWorkspaceTurnContinuation( workspaceId: string, - metadata: Extract + metadata: Extract, + streamEndMessageId: string ): boolean; hasQueuedWorkspaceTurn(workspaceId: string, handleId: string): boolean; removeQueuedWorkspaceTurn( @@ -527,6 +530,20 @@ export interface AgentTaskIntegration { options?: { workflowRunId?: string } ): Promise; noteWorkspaceUnarchived(workspaceId: string): Promise; + settleWorkspaceTurnContinuationFailure( + workspaceId: string, + muxMetadata: Extract, + status: "interrupted" | "error", + error: string + ): Promise; + /** + * Whether the delegated turn is still active on this workspace, with a probe that turns stale + * once a stop lands on the workspace after the read. + */ + getWorkspaceTurnContinuationAdmission( + workspaceId: string, + muxMetadata: Extract + ): Promise<{ admissible: boolean; admissionStale: () => boolean }>; } export interface WorkspaceTurnTaskHost { diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index e851b13662..604c08cbbf 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -50,6 +50,7 @@ import type { StreamManager } from "./streamManager"; import { markProviderMetadataCostsIncluded, type ModelFallbackOptions, + type QueuedMessageStop, type StreamTextOnChunk, type TurnCompletion, type TurnExecutionOptions, @@ -220,7 +221,7 @@ export function resolveXumToolScope( } import type { PostCompactionAttachment } from "@/common/types/attachment"; -import type { ErrorEvent } from "@/common/types/stream"; +import type { ErrorEvent, ModelFallbackProgress } from "@/common/types/stream"; import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; import type { FileState } from "@/node/services/agentSession"; import type { ActiveTurnThinkingOverride } from "@/node/services/thinkingOverride"; @@ -273,6 +274,20 @@ export interface StreamMessageOptions { workspaceGoalService?: WorkspaceGoalService; disableWorkspaceAgents?: boolean; hasQueuedMessages?: (dispatchMode?: "tool-end" | "turn-end") => boolean; + /** Fires when the model loop stops solely on behalf of a queued tool-end message. */ + onQueuedMessageStop?: (stop: QueuedMessageStop) => void; + /** Step ceiling for this stream; a stream resuming a cut turn runs under the cut's remainder. */ + stepBudget?: number; + /** + * Fallback chain of the cut turn this stream continues, used in place of the chain `model` + * would resolve: the resumed model may be a fallback whose own chain is unrelated. + */ + modelFallbackProgress?: ModelFallbackProgress; + /** + * Pull-based startup refusal (a goal admission probe with no push into abortSignal), rechecked + * by StreamManager right before the stream registers. + */ + refuseStreamStart?: () => boolean; muxMetadata?: MuxMessageMetadata; openaiTruncationModeOverride?: "auto" | "disabled"; /** @@ -737,6 +752,10 @@ export class TurnRequestBuilder { workspaceGoalService, disableWorkspaceAgents, hasQueuedMessages, + onQueuedMessageStop, + stepBudget, + modelFallbackProgress, + refuseStreamStart, openaiTruncationModeOverride, muxMetadata, minThinkingLevel: providedMinThinkingLevel, @@ -2705,13 +2724,16 @@ export class TurnRequestBuilder { // a cross-typed Coder instance (coder:openai/x, type anthropic) must use // its own gateway-scoped chain, never the direct provider's. Task // children can opt out via taskOnRefusal: "fail" (see - // resolveWorkspaceModelFallbackChain). - const modelFallbackChain = resolveWorkspaceModelFallbackChain( - this.dependencies.config.loadConfigOrDefault(), - workspaceId, - modelString, - this.dependencies.providerService.getConfig() - ); + // resolveWorkspaceModelFallbackChain). A stream continuing a cut turn keeps that turn's + // chain instead. + const modelFallbackChain = + modelFallbackProgress?.chain ?? + resolveWorkspaceModelFallbackChain( + this.dependencies.config.loadConfigOrDefault(), + workspaceId, + modelString, + this.dependencies.providerService.getConfig() + ); // Lazily rebuilds the per-model slice of this pipeline (model creation, // provider-specific message prep, provider options, headers, parameter @@ -2857,6 +2879,10 @@ export class TurnRequestBuilder { toolPolicy: effectiveToolPolicy, providedStreamToken: streamToken, hasQueuedMessages, + onQueuedMessageStop, + stepBudget, + modelFallbackProgress, + refuseStreamStart, workspaceName: metadata.name, thinkingLevel: streamThinkingLevel, headers: requestHeaders, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 66bc71107a..dea6e61627 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -252,6 +252,35 @@ describe("WorkspaceService bash monitor wake reconciler wiring", () => { return { config, service, events, cleanup }; } + test("a rename refuses the stranded resume at read and at the launch boundary", async () => { + const { config, service, cleanup } = await createWakeWiringService(); + const workspaceId = "renaming-resume-owner"; + await config.addWorkspace("/tmp/renaming-resume-project", { + id: workspaceId, + name: workspaceId, + projectName: "renaming-resume-project", + projectPath: "/tmp/renaming-resume-project", + runtimeConfig: { type: "local" }, + }); + try { + const session = service.getOrCreateSession(workspaceId); + const admit = Reflect.get(session, "admitStrandedTurnResume") as ( + correlation: undefined + ) => Promise<{ admissible: boolean; admissionStale?: () => boolean }>; + const admitted = await admit(undefined); + expect(admitted.admissible).toBe(true); + expect(admitted.admissionStale?.()).toBe(false); + + // rename() sees no registered stream while the resume is still in its pre-stream window + // and proceeds; the resume must not launch against paths being moved. + addToRenamingWorkspaces(service, workspaceId); + expect(admitted.admissionStale?.()).toBe(true); + expect((await admit(undefined)).admissible).toBe(false); + } finally { + await cleanup(); + } + }); + test("monitor lifecycle and shown-output events poke the reconciler", async () => { const { service, events, cleanup } = await createWakeWiringService(); const scheduleReconcile = mock(() => undefined); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 3e24599c35..b7667a25a4 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4061,6 +4061,43 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // is released at its queue/session handoff so a follow-up dispatched // from within that turn does not veto itself. hasExternalSendPreflight: () => this.hasSessionInvisiblePreflight(workspaceId), + settleForfeitedWorkspaceTurnContinuation: async (metadata, reason) => { + await this.agentTaskIntegration?.settleWorkspaceTurnContinuationFailure( + workspaceId, + metadata, + "interrupted", + reason + ); + }, + // The stranded resume starts a stream from inside the session, so it re-applies the + // stream-start guards WorkspaceService.resumeStream enforces (rename, removal, archive) + // and, for a delegated turn, checks the owner still has it running: a task_stop or + // lifecycle interrupt that found the cut stream already completed had no abort to + // withdraw the marker with. + admitStrandedTurnResume: async (correlation) => { + const workspaceRefused = (): boolean => + this.renamingWorkspaces.has(workspaceId) || + this.removingWorkspaces.has(workspaceId) || + this.archivingWorkspaces.has(workspaceId) || + this.isWorkspaceArchivedInConfig(workspaceId); + if (workspaceRefused()) { + return { admissible: false }; + } + if (correlation == null || this.agentTaskIntegration == null) { + return { admissible: true, admissionStale: workspaceRefused }; + } + const turn = await this.agentTaskIntegration.getWorkspaceTurnContinuationAdmission( + workspaceId, + correlation + ); + if (!turn.admissible) { + return { admissible: false }; + } + return { + admissible: true, + admissionStale: () => workspaceRefused() || turn.admissionStale(), + }; + }, }); } @@ -11801,10 +11838,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } - clearQueue(workspaceId: string, options?: { cancelReason?: string }): Result { + clearQueue( + workspaceId: string, + options?: { cancelReason?: string; hardStop?: boolean } + ): Result { try { const session = this.getOrCreateSession(workspaceId); - session.clearQueue(options?.cancelReason); + session.clearQueue(options?.cancelReason, { hardStop: options?.hardStop }); return Ok(undefined); } catch (error) { const errorMessage = getErrorMessage(error); @@ -11963,24 +12003,21 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } - /** - * Whether a bash-monitor-wake continuation is queued next or mid-dispatch. - * See AgentSession.hasPendingBashMonitorWakeContinuation for semantics. - */ - hasPendingBashMonitorWakeContinuation(workspaceId: string): boolean { - const session = this.sessions.get(workspaceId.trim()); - return session?.hasPendingBashMonitorWakeContinuation() ?? false; + private isWorkspaceArchivedInConfig(workspaceId: string): boolean { + const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); + return ( + entry != null && isWorkspaceArchived(entry.workspace.archivedAt, entry.workspace.unarchivedAt) + ); } - /** - * Whether a queued or dispatching entry continues the exact workspace-turn correlation. - */ - hasPendingWorkspaceTurnContinuation( + /** See AgentSession.claimWorkspaceTurnContinuation for semantics. */ + claimWorkspaceTurnContinuation( workspaceId: string, - metadata: Extract + metadata: Extract, + streamEndMessageId: string ): boolean { const session = this.sessions.get(workspaceId.trim()); - return session?.hasPendingWorkspaceTurnContinuation(metadata) ?? false; + return session?.claimWorkspaceTurnContinuation(metadata, streamEndMessageId) ?? false; } /** diff --git a/src/node/services/workspaceTurnManager.test.ts b/src/node/services/workspaceTurnManager.test.ts index 20588945ad..6e6e7bbe52 100644 --- a/src/node/services/workspaceTurnManager.test.ts +++ b/src/node/services/workspaceTurnManager.test.ts @@ -317,8 +317,7 @@ describe("WorkspaceTurnManager", () => { isStreaming?: ReturnType; hasQueuedMessages?: ReturnType; hasPendingQueuedOrPreparingTurn?: ReturnType; - hasPendingBashMonitorWakeContinuation?: ReturnType; - hasPendingWorkspaceTurnContinuation?: ReturnType; + claimWorkspaceTurnContinuation?: ReturnType; getQueueCutCutter?: ReturnType; hasPendingAutoRetry?: ReturnType; waitForPendingStreamErrorRecoveryDecision?: ReturnType; diff --git a/src/node/services/workspaceTurnManager.ts b/src/node/services/workspaceTurnManager.ts index b74f3b9d6e..745263e070 100644 --- a/src/node/services/workspaceTurnManager.ts +++ b/src/node/services/workspaceTurnManager.ts @@ -4307,24 +4307,23 @@ export class WorkspaceTurnManager { } /** - * Whether a continuation of this exact delegated turn is pending or streaming. - * Pending entries must carry the same correlation metadata as the ended stream. + * Whether a continuation of this exact delegated turn is pending or streaming. The + * session's answer is binding: a false claim voids the continuation it owed to this cut, so + * the settlement made here cannot be followed by an orphaned resume of the same turn. */ private hasSameTurnContinuation( event: StreamEndEvent, correlation: { taskHandleId: string; ownerWorkspaceId: string; turnId: string } ): boolean { if ( - this.workspaceService.hasPendingWorkspaceTurnContinuation(event.workspaceId, { - type: "workspace-turn-task", - ...correlation, - }) + this.workspaceService.claimWorkspaceTurnContinuation( + event.workspaceId, + { type: "workspace-turn-task", ...correlation }, + event.messageId + ) ) { return true; } - if (this.workspaceService.hasPendingBashMonitorWakeContinuation(event.workspaceId)) { - return true; - } const activeStream = this.streamManager?.getStreamInfo(event.workspaceId); if (activeStream == null || activeStream.messageId === event.messageId) { return false;