From 843e84ae382db7e693c7c51fef466d5f4cadaf8f Mon Sep 17 00:00:00 2001 From: Giselle van Dongen Date: Fri, 20 Nov 2026 14:12:32 +0200 Subject: [PATCH 1/2] save state more frequently --- CONTEXT.md | 4 +- README.md | 1 + packages/core/src/agent-run.ts | 77 ++- packages/core/src/conversation.ts | 32 +- packages/core/src/index.ts | 1 + packages/core/src/run-history.ts | 129 +++- packages/core/src/types.ts | 4 + packages/core/src/validation.ts | 58 +- .../examples/src/lessons/03_conversation.ts | 2 + .../src/lessons/09_support_session.ts | 5 +- .../tests/src/conversation-progress.test.ts | 568 ++++++++++++++++++ .../tests/src/conversation-replay.test.ts | 149 +++++ packages/tests/src/errors.test.ts | 43 +- packages/tests/src/runtime.integration.ts | 185 +++++- .../restate-agents/references/agent-basics.md | 3 +- .../references/approvals-and-guardrails.md | 4 +- .../references/conversations-and-controls.md | 31 +- .../references/error-handling.md | 45 +- 18 files changed, 1254 insertions(+), 87 deletions(-) create mode 100644 packages/tests/src/conversation-progress.test.ts create mode 100644 packages/tests/src/conversation-replay.test.ts diff --git a/CONTEXT.md b/CONTEXT.md index d23cdff..460214d 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -56,7 +56,9 @@ _Avoid_: Sub-agent, handoff **Conversation**: The accumulated model context across runs, including user messages, assistant -messages, tool exchanges, and runtime observations. +messages, tool exchanges, and runtime observations. A run saves to it while it +runs, after each model call and tool batch, so a running turn can be read; a +failed run saves what it committed, with a note on the failure. _Avoid_: Public transcript, session **Session**: diff --git a/README.md b/README.md index 0d71ebb..2dacdf1 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ repeat a completed model call. | ---------------------------- | -------------------------------------------------------------------------- | ----------------------------------------------------------- | | Tools with journaled results | `asyncTool`, `tool` | [02](packages/examples/src/lessons/02_tools.ts) | | Conversation history | `run(input, {messages})`, `run(input, {conversation})` | [03](packages/examples/src/lessons/03_conversation.ts), 09 | +| Live and failed-turn history | `{conversation}` saves as the run goes; `messagesOf(error)` | [09](packages/examples/src/lessons/09_support_session.ts) | | Typed output | `agent({output})` | [04](packages/examples/src/lessons/04_structured_output.ts) | | Human approval | `needsApproval`, `onApproval`, `humanApprovalTool` | [06](packages/examples/src/lessons/06_human_approval.ts) | | Steering and interruption | `run(input, {steeringSignal, interruptSignal})` or `{controls}` | [09](packages/examples/src/lessons/09_support_session.ts) | diff --git a/packages/core/src/agent-run.ts b/packages/core/src/agent-run.ts index 9989db6..d412cce 100644 --- a/packages/core/src/agent-run.ts +++ b/packages/core/src/agent-run.ts @@ -1,3 +1,4 @@ +import {CancelledError, TerminalError} from "@restatedev/restate-sdk"; import { handlerRequest, InterruptedError, @@ -21,7 +22,8 @@ import { ToolError, toolErrorCode, } from "./errors.js"; -import {RunHistory, type BranchPoint} from "./run-history.js"; +import {ConversationConflictError} from "./conversation.js"; +import {recordFailedHistory, RunHistory, type BranchPoint} from "./run-history.js"; import {RunTasks, type FutureBranches} from "./run-tasks.js"; import {RunTools} from "./run-tools.js"; import {PARENT_USAGE, recordFailedRun, RunUsageTracker} from "./run-usage.js"; @@ -179,7 +181,10 @@ function resolveControls(options?: RunOptions): RunControls { * - Background results completed during a model request also invalidate its * response: the model must see them before acting. * - * Nothing is saved until finish() persists the turn; failures save nothing. */ + * With a conversation, the run saves as it goes, so a shared handler can read its + * progress: after each model call, each tool batch, and accepted steering or + * background results, then once more when finish() commits the answer. A failed + * run saves its committed history with a note on the failure; see fail(). */ class AgentRun { private step = 0; private steeringSource: SteeringSource | undefined; @@ -187,6 +192,8 @@ class AgentRun { private interrupt: Future | undefined; /** Set once a summary failed; the run truncates from then on. */ private compactionFailed = false; + /** Set once `beforeRun` accepted the input; a fork's input needs no check. */ + private inputChecked: boolean; /** The last request without its messages: the prefix a provider cached. * Its instructions may come from a function. */ private lastRequest: Pick | undefined; @@ -213,6 +220,7 @@ class AgentRun { private readonly origin?: ForkOrigin, ) { this.active = origin?.active ?? root; + this.inputChecked = origin !== undefined; this.skills = origin?.skills; this.catalog = createToolCatalog(this.active.tools, this.runTools()); for (const name of origin?.loaded ?? []) this.catalog.search.loaded.add(name); @@ -252,6 +260,10 @@ class AgentRun { } // Outside the catch: a failing stop propagates instead of stopping twice. return yield* this.stop(stop); + } catch (error) { + yield* this.fail(error); + // The same value: Restate's retry and cancellation handling depend on its type. + throw error; } finally { yield* this.tasks.close(); } @@ -263,6 +275,7 @@ class AgentRun { private *start(): Operation { if (this.origin) return; const rejected = yield* this.checkInput(); + this.inputChecked = true; const source = this.root.skills; const listed = source ? yield* listSkills(source) : []; if (source && listed.length) { @@ -313,11 +326,13 @@ class AgentRun { yield* this.emit({type: "step", step: this.step}); yield* this.compactContext(); const response = yield* this.ask(); - if (!response || this.collectBackground()) continue; + if (!response || (yield* this.collectBackground())) continue; if (response.type === "tool_calls") { yield* this.toolPhase(response); continue; } + // The answer itself waits for review; this saves what came before it, such as the input. + yield* this.history.save(); const result = yield* this.complete(response); if (result) return result; } @@ -608,6 +623,8 @@ class AgentRun { loaded: [...this.catalog.search.loaded], }); const foreground = this.tools.startBatch(action, this.step, this.active, this.catalog); + // The calls are saved before their results, so a shared handler can show them. + yield* this.history.save(); while (Object.keys(foreground).length > 0) { const {tag} = yield* this.race({...foreground, ...this.tools.pendingTasks()}); if (tag === "steering") { @@ -615,10 +632,11 @@ class AgentRun { continue; } delete foreground[tag]; - this.collectBackground(); + yield* this.collectBackground(); } const transfer = this.tools.commitBatch(); - this.collectBackground(); + yield* this.collectBackground(); + yield* this.history.save(); if (transfer) { this.active = transfer(); yield* this.emit({type: "handoff", agent: this.active.name}); @@ -628,11 +646,13 @@ class AgentRun { } } - /** Moves finished background results into history; true when any arrived. */ - private collectBackground(): boolean { + /** Moves finished background results into history and saves them; true when any arrived. */ + private *collectBackground(): Operation { const completed = this.tools.drainCompleted(); + if (!completed.length) return false; this.history.append(...completed); - return completed.length > 0; + yield* this.history.save(); + return true; } private *complete(action: ModelFinalResponse): Operation | undefined> { @@ -641,7 +661,7 @@ class AgentRun { // Its result, or steering, gives the model a reason to reconsider. if (this.step < this.config.maxSteps) { yield* this.race(this.tools.pendingTasks()); - this.collectBackground(); + yield* this.collectBackground(); } return undefined; } @@ -751,6 +771,8 @@ class AgentRun { } } this.history.appendSteering(message); + // Saved before it is reported. Mid-batch, the note waits for the batch's results. + yield* this.history.save(); yield* this.emit({type: "steering", message, ...withData}); return true; } @@ -796,6 +818,43 @@ class AgentRun { return yield* this.finish(yield* this.review(result)); } + /** Records the history of a run that `error` ends, for `messagesOf`, and saves it + * when the error fails the invocation, so the next run knows what happened. + * Running work stops first: calls that never finished get cancelled results, and + * a note tells the model why the run ended. A retryable error saves nothing: + * Restate retries the invocation, and a journaled save would not match a retry + * that takes another path. A conflict means another writer changed the + * conversation, so saving again would fail the same way. A failure to save is + * logged; the caller still receives the run's own error. + * + * A run that fails before `beforeRun` accepted its input records nothing: + * throwing `AgentTerminalError` from `beforeRun` keeps the input out of the + * history. */ + private *fail(error: unknown): Operation { + if (!this.inputChecked) return; + try { + yield* this.tasks.close(); + const reason = error instanceof Error ? error.message : String(error); + const cancelled = error instanceof CancelledError; + const closed = `${cancelled ? "Run cancelled" : "Run failed"}: ${reason}`; + this.tools.commitBatch(closed); + this.history.append(...this.tools.settlePending(new InterruptedError(closed))); + this.history.append({ + role: "user", + content: JSON.stringify({ + source: "agent-runtime", + type: cancelled ? "run-cancelled" : "run-failed", + message: reason, + }), + }); + recordFailedHistory(error, this.history.committed()); + if (error instanceof TerminalError && !(error instanceof ConversationConflictError)) + yield* this.history.save(); + } catch (failure) { + logger().warn("Could not record the failed run's history:", failure); + } + } + /** The schema the run's answer must match. A fork answers in plain text: its * reply is a tool result for the parent, not the run's output. */ private get output(): AgentOptions["output"] { diff --git a/packages/core/src/conversation.ts b/packages/core/src/conversation.ts index d726d9f..f32a29a 100644 --- a/packages/core/src/conversation.ts +++ b/packages/core/src/conversation.ts @@ -2,10 +2,17 @@ import {TerminalError} from "@restatedev/restate-sdk"; import {sharedState, state, type Operation} from "@restatedev/restate-sdk-gen"; import {readable} from "./context-size.js"; import type {Message, Model} from "./types.js"; -import {history, modelResponse} from "./validation.js"; +import {checkProtocol, history, modelResponse, parseMessages, type OpenCall} from "./validation.js"; export type ConversationSnapshot = {messages: Message[]; version: number}; export type ConversationCompactionStatus = "skipped" | "applied" | "stale"; +/** Where a run loads its history and saves its progress. + * + * A run saves while it runs, not only at its end. Each `append` continues the + * messages before it, so it can start with results for tool calls saved by an + * earlier append, and end with tool calls whose results a later append adds: + * the run saves a batch's calls as soon as the model makes them. Between those + * two appends, `load` returns a history that ends with unfinished calls. */ export interface Conversation { load(): Operation; append(messages: readonly Message[], expectedVersion: number): Operation; @@ -23,7 +30,12 @@ export type CompactionPlan = { messages: Message[]; }; export type CompactionResult = {plan: CompactionPlan; summary: string}; -type Meta = {count: number; summary?: {through: number; text: string}}; +type Meta = { + count: number; + summary?: {through: number; text: string}; + /** Calls at the end of the stored messages that have no result yet. */ + open?: OpenCall[]; +}; /** Another writer changed the conversation since this run loaded it. */ export class ConversationConflictError extends TerminalError { constructor(message: string) { @@ -81,9 +93,12 @@ export class LazyStateConversation implements Conversation { return structuredClone(messages); } + /** The history a run continues. It ends with unfinished tool calls while a run's + * batch is running, or after that run was killed; the next run closes them. */ *load(): Operation { const meta = yield* this.meta(); - const messages = history(yield* this.range(meta.summary?.through ?? 0, meta.count)); + const range = yield* this.range(meta.summary?.through ?? 0, meta.count); + const messages = history(range, {unfinished: true}); if (meta.summary) messages.unshift({ role: "user", @@ -93,11 +108,15 @@ export class LazyStateConversation implements Conversation { } *append(messages: readonly Message[], expectedVersion: number): Operation { - const additions = history(messages); + const additions = parseMessages(messages); const meta = yield* this.meta(); if (meta.count !== expectedVersion) throw new ConversationConflictError("Conversation changed since it was loaded"); if (!additions.length) return meta.count; + // The additions may first answer the calls the last append left open. + const open = checkProtocol(additions, {unanswered: meta.open ?? []}); + if (open.length) meta.open = open; + else delete meta.open; let offset = 0; while (offset < additions.length) { const index = Math.floor(meta.count / CHUNK_SIZE); @@ -130,7 +149,8 @@ export class LazyStateConversation implements Conversation { throw new TypeError("Compaction limits must be positive safe integers"); const meta = yield* this.meta(); const baseThrough = meta.summary?.through ?? 0; - const messages = history(yield* this.range(baseThrough, meta.count)); + // A run may be in the middle of a batch: the boundary stays before its calls. + const messages = history(yield* this.range(baseThrough, meta.count), {unfinished: true}); // Measured as a summarizer reads them: encrypted reasoning would inflate the count. const chars = JSON.stringify(messages.map(readable)).length; if (messages.length <= maxMessages && chars <= maxChars) return undefined; @@ -170,7 +190,7 @@ export class LazyStateConversation implements Conversation { if (meta.summary?.through === plan.through && meta.summary.text === summary) return true; if ((meta.summary?.through ?? 0) !== plan.baseThrough) return false; // Verify the retained suffix independently before committing the new boundary. - history(yield* this.range(plan.through, meta.count)); + history(yield* this.range(plan.through, meta.count), {unfinished: true}); yield* this.store.set(this.metaKey, {...meta, summary: {through: plan.through, text: summary}}); return true; } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2e2b594..eb42c72 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -17,6 +17,7 @@ export { isContextOverflow, } from "./errors.js"; export {usageOf} from "./run-usage.js"; +export {messagesOf} from "./run-history.js"; export type { Agent, AgentOptions, diff --git a/packages/core/src/run-history.ts b/packages/core/src/run-history.ts index 1421cf2..4edca5a 100644 --- a/packages/core/src/run-history.ts +++ b/packages/core/src/run-history.ts @@ -4,7 +4,7 @@ import type {Message} from "./types.js"; import {RunInputError} from "./errors.js"; import {estimateTokens} from "./context-size.js"; import {previewMessage, type PreviewSettings} from "./tool-result-preview.js"; -import {checkProtocol, history} from "./validation.js"; +import {checkProtocol, history, UnfinishedToolCallsError} from "./validation.js"; type HistorySource = {conversation?: Conversation; messages?: readonly Message[]}; @@ -32,10 +32,46 @@ type Measured = {readonly mark: ViewMark; readonly inputTokens: number}; type ToolMessage = Extract; export type NoteMessage = Extract; -/** Events arrive in execution order. Model protocols require each batch's tool - * results immediately after its assistant call, ahead of intervening steering. - * Every event was validated when it was recorded, so only the order is checked. */ -function modelTranscript(events: readonly Message[]): Message[] { +/** The result a run records for a call that a killed run left without one. */ +const UNFINISHED_CALL = + "The run that made this call ended before the call returned a result. It may have taken effect."; + +/** Validates a conversation's history for a new run, and closes the tool calls + * that a killed run left without results. Returns the history and how many results + * it added; the run's first save stores them. + * + * A run saves a batch's calls as soon as the model makes them, and their results + * when the batch ends (see `RunHistory.save`). A retry replays the journal and + * finishes the batch, and a failed or cancelled run closes the calls before it + * ends, so only a killed invocation leaves a history that ends with unfinished + * calls. Model providers reject such a history, so the run closes each call with a + * cancelled result. + * + * The repair adds no work for a complete history: it runs only when `checkProtocol` + * reports unfinished calls. That check passes over the whole history on every load, + * here and in `LazyStateConversation.load`. To remove those passes later, keep a + * check of the last assistant message alone: every save stores a prefix of a + * complete transcript, so only the last batch can lack results. */ +function continuedHistory(loaded: readonly Message[]): {messages: Message[]; closed: number} { + try { + return {messages: history(loaded), closed: 0}; + } catch (error) { + if (!(error instanceof UnfinishedToolCallsError)) throw error; + const closing: Message[] = error.calls.map(({id, name}) => ({ + role: "tool", + callId: id, + name, + result: {status: "cancelled", message: UNFINISHED_CALL}, + })); + return {messages: history([...loaded, ...closing]), closed: closing.length}; + } +} + +/** Events in the order model protocols require: each batch's tool results right + * after its assistant call, ahead of the steering and notes that arrived while the + * batch ran. With `stopAtOpenBatch`, the transcript ends with the first batch that + * still waits for results, and the events after it wait until the batch ends. */ +function ordered(events: readonly Message[], stopAtOpenBatch = false): Message[] { const batchByCall = new Map(); for (const [index, message] of events.entries()) { if (message.role !== "assistant") continue; @@ -54,17 +90,28 @@ function modelTranscript(events: readonly Message[]): Message[] { for (const [index, message] of events.entries()) { if (message.role === "tool") continue; transcript.push(message); - if (message.role === "assistant") transcript.push(...(resultsByBatch.get(index) ?? [])); + if (message.role !== "assistant") continue; + const results = resultsByBatch.get(index) ?? []; + transcript.push(...results); + if (stopAtOpenBatch && results.length < (message.calls?.length ?? 0)) break; } - checkProtocol(transcript); + return transcript; +} + +/** Events arrive in execution order; this is the transcript a model accepts. + * Every event was validated when it was recorded, so only the order is checked. */ +function modelTranscript(events: readonly Message[]): Message[] { + const transcript = ordered(events); + if (checkProtocol(transcript).length) + throw new RunInputError("History contains unfinished tool calls"); return transcript; } /** Owns the append-only event history and its persistence cursor. Every message enters * a run as a copy the run owns, validated once. Model requests and hooks share these * messages instead of copying the history, as the AI SDK does: they read them and never - * change them. The run saves - * a model-valid transcript at finalization; earlier failures save nothing. + * change them. The run saves its committed transcript as it goes (see `save`), and a + * failed run saves it with its open calls closed and a note on the failure. * Provisional messages, such as a rejected answer, never leave the run. */ export class RunHistory { private readonly provisional = new Set(); @@ -99,15 +146,18 @@ export class RunHistory { const {conversation} = source; if (conversation && source.messages) throw new RunInputError("Use either conversation or messages"); - const snapshot = conversation ? yield* conversation.load() : undefined; - const messages = history(snapshot?.messages ?? source.messages ?? []); - const run = new RunHistory( - input, - messages, - snapshot?.version ?? 0, - messages.length, - conversation, - ); + let messages: Message[]; + let version = 0; + let closed = 0; + if (conversation) { + const snapshot = yield* conversation.load(); + ({messages, closed} = continuedHistory(snapshot.messages)); + version = snapshot.version; + } else { + // A caller's messages come from a finished or failed run, which closes its calls. + messages = history(source.messages ?? []); + } + const run = new RunHistory(input, messages, version, messages.length - closed, conversation); run.previews = previews; run.append({role: "user", content: input}); return run; @@ -277,10 +327,24 @@ export class RunHistory { this.messages.push(...messages); } + /** What a save stores: the committed transcript up to the batch that is still + * running, with that batch's calls and the results it has so far. Events only + * extend it, so each save appends to what the last one stored. */ + private saveable(): Message[] { + return ordered( + this.messages.filter((message) => !this.provisional.has(message)), + true, + ); + } + + /** Appends what the run committed since its last save. The run saves after each + * model call, after each tool batch, and after steering or background results + * arrive, so a shared handler can read a run's progress, and when it ends. A + * batch's calls are saved before their results; see `continuedHistory`. */ *save(): Operation { if (!this.conversation) return; - const transcript = this.committed(); - if (transcript.length === this.persisted) return; + const transcript = this.saveable(); + if (transcript.length <= this.persisted) return; this.version = yield* this.conversation.append(transcript.slice(this.persisted), this.version); this.persisted = transcript.length; } @@ -290,12 +354,31 @@ export class RunHistory { const result = yield* this.conversation.compact(); if (result === "skipped") return; const compacted = yield* this.conversation.load(); - this.messages.splice(0, this.messages.length, ...history(compacted.messages), { + const {messages, closed} = continuedHistory(compacted.messages); + this.messages.splice(0, this.messages.length, ...messages, { role: "user", content: this.input, }); - this.persisted = compacted.messages.length; - this.inputAt = compacted.messages.length; + this.persisted = messages.length - closed; + this.inputAt = messages.length; this.version = compacted.version; } } + +const failedRuns = new WeakMap(); + +/** The history of the run that threw `error`, or undefined when no run threw it: + * its committed messages, with a cancelled result for each call that never + * finished and a note that says why the run ended. A run with a `conversation` + * saves this history when the error is terminal; a run with `messages` can pass + * it to the next run. When several runs threw the same error, it holds the last + * one's history; for nested runs that is the outermost. */ +export function messagesOf(error: unknown): Message[] | undefined { + const messages = typeof error === "object" && error !== null ? failedRuns.get(error) : undefined; + return messages && structuredClone(messages); +} + +/** Records a failed run's history for `messagesOf` without changing the error. */ +export function recordFailedHistory(error: unknown, messages: Message[]): void { + if (typeof error === "object" && error !== null) failedRuns.set(error, messages); +} diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 64f6176..cfa493c 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -280,6 +280,10 @@ export type RunControls = {steering?: SteeringSource; interrupt?: Future export type RunOptions = { /** Completed prior messages. The SDK copies these and returns the new history. */ messages?: readonly Message[]; + /** Where the run loads its history and saves its progress: after each model call, + * tool batch, and accepted steering, and when it ends. A run that a terminal error + * fails saves what it recorded, with a note on the failure. Use either this or + * `messages`. */ conversation?: import("./conversation.js").Conversation; name?: string; onApproval?: ApprovalHandler; diff --git a/packages/core/src/validation.ts b/packages/core/src/validation.ts index 6cf86de..c669b1d 100644 --- a/packages/core/src/validation.ts +++ b/packages/core/src/validation.ts @@ -105,22 +105,54 @@ export function toolOutput(value: unknown): JsonValue { return result.data; } -/** Clone and reject incomplete histories before making any model call. */ -export function history(value: readonly Message[]): Message[] { +/** A tool call that has no result yet. */ +export type OpenCall = {id: string; name: string}; + +/** A history whose last tool calls have no results. A run saves its tool calls as + * soon as the model makes them and their results when the batch ends, so a + * conversation holds one while a batch runs, and keeps it if the invocation is + * killed before the batch ends. */ +export class UnfinishedToolCallsError extends RunInputError { + constructor(readonly calls: readonly OpenCall[]) { + super("History contains unfinished tool calls"); + this.name = "UnfinishedToolCallsError"; + } +} + +/** Clones and validates messages, without checking the tool-call protocol. */ +export function parseMessages(value: readonly Message[]): Message[] { const parsed = z.array(messageSchema).safeParse(value); if (!parsed.success) throw new RunInputError(`Invalid history: ${parsed.error.message}`); - checkProtocol(parsed.data); return parsed.data; } +/** Clone and reject incomplete histories before making any model call. A history + * whose last calls have no results fails with `UnfinishedToolCallsError`, unless + * `unfinished` allows it, as for a conversation read while a batch runs. */ +export function history( + value: readonly Message[], + options: {unfinished?: boolean} = {}, +): Message[] { + const messages = parseMessages(value); + const open = checkProtocol(messages); + if (open.length && !options.unfinished) throw new UnfinishedToolCallsError(open); + return messages; +} + /** Rejects a history whose tool results do not answer the calls before them, in - * order. Structural only: callers pass messages that were already validated. */ + * order, and returns the calls at its end that have no result yet. A caller that + * needs a complete history rejects those. `unanswered` are calls from messages + * before these, still waiting for the results these messages may start with. + * Structural only: callers pass messages that were already validated. */ export function checkProtocol( messages: readonly Message[], - fail: (message: string) => Error = (message) => new RunInputError(message), -): void { - const ids = new Set(); - const pending = new Map(); + { + fail = (message) => new RunInputError(message), + unanswered = [], + }: {fail?: (message: string) => Error; unanswered?: readonly OpenCall[]} = {}, +): OpenCall[] { + const ids = new Set(unanswered.map((call) => call.id)); + const pending = new Map(unanswered.map((call) => [call.id, call.name])); for (const message of messages) { if (message.role === "tool") { if (pending.get(message.callId) !== message.name) { @@ -138,7 +170,7 @@ export function checkProtocol( } } } - if (pending.size > 0) throw fail("History contains unfinished tool calls"); + return [...pending].map(([id, name]) => ({id, name})); } /** The messages a `beforeStep` hook returned for one request. Messages it kept from @@ -158,10 +190,8 @@ export function stepMessages( throw new TypeError(`beforeStep returned an invalid message: ${parsed.error.message}`); return parsed.data; }); - checkProtocol( - messages, - (problem) => - new TypeError(`beforeStep returned messages that break the tool-call protocol: ${problem}`), - ); + const fail = (problem: string) => + new TypeError(`beforeStep returned messages that break the tool-call protocol: ${problem}`); + if (checkProtocol(messages, {fail}).length) throw fail("History contains unfinished tool calls"); return messages; } diff --git a/packages/examples/src/lessons/03_conversation.ts b/packages/examples/src/lessons/03_conversation.ts index f1810a4..1938c68 100644 --- a/packages/examples/src/lessons/03_conversation.ts +++ b/packages/examples/src/lessons/03_conversation.ts @@ -36,6 +36,8 @@ export const chatSession = restate.object({ function* ({message}) { const messages = (yield* restate.state().get("messages")) ?? []; const result = yield* assistant.run(message, {messages}); + // Saved when the run returns. For a run that throws, `messagesOf(error)` + // returns its history up to the failure, with a note on why it failed. restate.state().set("messages", result.messages); if (result.status !== "completed") return result.output ?? result.reason; return result.output; diff --git a/packages/examples/src/lessons/09_support_session.ts b/packages/examples/src/lessons/09_support_session.ts index 9f3efc1..2a3c082 100644 --- a/packages/examples/src/lessons/09_support_session.ts +++ b/packages/examples/src/lessons/09_support_session.ts @@ -56,7 +56,9 @@ export const aiSession = restate.object({ output: z.string(), }, function* ({message}) { - // The conversation lives in this object's state; long ones are summarized. + // The conversation lives in this object's state. The turn saves to it as it + // runs, so the shared `history` handler shows a running turn, and a failed + // turn stays in it with a note on the failure. Long ones are summarized. const conversation = new LazyStateConversation({ compaction: {model, policy: {maxMessages: 64, retainMessages: 12}}, }); @@ -84,6 +86,7 @@ export const aiSession = restate.object({ approval: yield* restate.sharedState().get("approval"), }; }, + // The saved conversation, including the turn that is running now. *history() { return yield* new LazyStateConversation().page(); }, diff --git a/packages/tests/src/conversation-progress.test.ts b/packages/tests/src/conversation-progress.test.ts new file mode 100644 index 0000000..6cc81a3 --- /dev/null +++ b/packages/tests/src/conversation-progress.test.ts @@ -0,0 +1,568 @@ +// A run with a conversation saves as it goes, and a failed run saves what it +// committed with a note on the failure. These tests read the store the way a +// shared handler reads it, while the run continues. +import assert from "node:assert/strict"; +import {test} from "node:test"; +import {CancelledError, TerminalError} from "@restatedev/restate-sdk"; +import * as r from "@restatedev/restate-sdk-gen"; +import {z} from "zod"; +import { + agent, + AgentTerminalError, + ConversationConflictError, + LazyStateConversation, + messagesOf, + tool, + type CompactionPlan, + type Conversation, + type Message, + type ModelResponse, + type ToolResult, +} from "@restatedev/agentlib"; +import {history} from "../../core/src/validation.js"; +import {memoryStore, testContext} from "./helpers.js"; + +const ctx = () => testContext("conversation-progress"); +const base = {description: "fixture", input: z.object({})}; +const done: ModelResponse = {type: "final", text: "done"}; +const batch = (...ids: string[]): ModelResponse => ({ + type: "tool_calls", + calls: ids.map((id) => ({id, name: id, input: {}})), +}); +const user = (content: string): Message => ({role: "user", content}); +const answer = (content: string): Message => ({role: "assistant", content}); +const calls = (...ids: string[]): Message => ({ + role: "assistant", + content: "", + calls: ids.map((id) => ({id, name: id, input: {}})), +}); +const resultOf = (id: string, result: ToolResult): Message => ({ + role: "tool", + callId: id, + name: id, + result, +}); +const success = (id: string, output: string) => resultOf(id, {status: "success", output}); +const cancelled = (id: string, message: string) => resultOf(id, {status: "cancelled", message}); +const note = (type: string, message: string): Message => + user(JSON.stringify({source: "agent-runtime", type, message})); +const pairs = (n: number): Message[] => + Array.from({length: n}, (_, i) => [user(`question ${i}`), answer(`answer ${i}`)]).flat(); +const stop = (message: string): never => { + throw new AgentTerminalError(message); +}; + +/** Runs `body` to completion in the in-process scheduler. */ +function execute(body: () => r.Operation): Promise { + return r.execute(ctx(), body()); +} + +/** The error a run throws, from inside the scheduler. */ +function* failure(run: r.Operation): r.Operation { + try { + yield* run; + } catch (error) { + return error; + } + return assert.fail("the run must fail"); +} + +void test("a shared reader sees each model call and tool batch while the run continues", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + // Another instance on the same store stands in for a shared handler. + const reader = new LazyStateConversation({store}); + const seen: Message[][] = []; + await execute(function* () { + const a = agent({ + tools: { + lookup: tool({ + ...base, + *execute() { + seen.push((yield* reader.page()).messages); + // A history read mid-batch ends with the call, which has no result yet. + assert.deepEqual((yield* reader.load()).messages, [user("go"), calls("lookup")]); + return "found"; + }, + }), + }, + *model(request) { + seen.push((yield* reader.page()).messages); + return request.step === 1 ? batch("lookup") : done; + }, + }); + const result = yield* a.run("go", {conversation}); + assert.deepEqual(seen, [ + [], + [user("go"), calls("lookup")], + [user("go"), calls("lookup"), success("lookup", "found")], + ]); + assert.deepEqual((yield* reader.page()).messages, result.messages); + }); +}); + +void test("background results are saved when the run collects them", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + const reader = new LazyStateConversation({store}); + await execute(function* () { + const gate = r.channel(); + let step = 0; + const a = agent({ + tools: { + job: tool({ + ...base, + background: true, + *execute() { + yield* gate.receive; + return "finished"; + }, + }), + }, + *model() { + if (++step === 1) return batch("job"); + if (step === 2) { + // The batch ended with the call's pending acknowledgment. + const saved = (yield* reader.page()).messages; + assert.deepEqual(saved, [ + user("go"), + calls("job"), + resultOf("job", {status: "pending", operationId: "job"}), + ]); + yield* gate.send(undefined); + return {type: "final", text: "too early"}; + } + const saved = (yield* reader.page()).messages; + assert.match(JSON.stringify(saved.at(-1)), /background-tool-result.*finished/); + return done; + }, + }); + const result = yield* a.run("go", {conversation}); + assert.equal(step, 3); + assert.deepEqual((yield* reader.page()).messages, result.messages); + }); +}); + +void test("a run that fails mid-batch saves its finished results, closes the rest, and says why", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + await execute(function* () { + const a = agent({ + tools: { + fast: tool({ + ...base, + *execute() { + return "fast"; + }, + }), + halt: tool({ + ...base, + *execute() { + return stop("payment provider rejected the account"); + }, + }), + slow: tool({ + ...base, + *execute() { + yield* r.channel().receive; + return "unreachable"; + }, + }), + }, + *model() { + return batch("fast", "halt", "slow"); + }, + }); + const error = yield* failure(a.run("go", {conversation})); + assert.ok(error instanceof AgentTerminalError); + const reason = "payment provider rejected the account"; + const saved = (yield* conversation.load()).messages; + history(saved); + assert.deepEqual(saved, [ + user("go"), + calls("fast", "halt", "slow"), + success("fast", "fast"), + cancelled("halt", `Run failed: ${reason}`), + cancelled("slow", `Run failed: ${reason}`), + note("run-failed", reason), + ]); + assert.deepEqual(messagesOf(error), saved); + }); +}); + +void test("a model error fails the run, and the next run reads why", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + await execute(function* () { + const failing = agent({ + tools: { + lookup: tool({ + ...base, + *execute() { + return "found"; + }, + }), + }, + *model(request) { + if (request.step === 1) return batch("lookup"); + throw new TerminalError("model unavailable"); + }, + }); + const error = yield* failure(failing.run("go", {conversation})); + // The original error, unchanged. + assert.ok(error instanceof TerminalError); + assert.equal(error.message, "model unavailable"); + const failed = [ + user("go"), + calls("lookup"), + success("lookup", "found"), + note("run-failed", "model unavailable"), + ]; + assert.deepEqual((yield* conversation.load()).messages, failed); + const next = agent({ + *model(request) { + assert.deepEqual(request.messages, [...failed, user("again")]); + return done; + }, + }); + const result = yield* next.run("again", {conversation}); + assert.deepEqual((yield* conversation.load()).messages, result.messages); + }); +}); + +void test("a retryable error saves no failure note: Restate retries the invocation", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + await execute(function* () { + const a = agent({ + tools: { + lookup: tool({ + ...base, + *execute() { + return "found"; + }, + }), + }, + *model(request) { + if (request.step === 1) return batch("lookup"); + throw new Error("connection reset"); + }, + }); + const error = yield* failure(a.run("go", {conversation})); + assert.ok(!(error instanceof TerminalError)); + // Saved as the run went; a retry replays these writes and continues. + const progress = [user("go"), calls("lookup"), success("lookup", "found")]; + assert.deepEqual((yield* conversation.load()).messages, progress); + assert.deepEqual(messagesOf(error), [...progress, note("run-failed", "connection reset")]); + }); +}); + +void test("a cancelled invocation saves its closed calls with a cancellation note", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + await execute(function* () { + const a = agent({ + tools: { + wait: tool({ + ...base, + *execute() { + // What Restate delivers to a cancelled invocation's pending work. + throw new CancelledError(); + }, + }), + }, + *model() { + return batch("wait"); + }, + }); + const error = yield* failure(a.run("go", {conversation})); + assert.ok(error instanceof CancelledError); + const saved = (yield* conversation.load()).messages; + history(saved); + assert.deepEqual(saved, [ + user("go"), + calls("wait"), + cancelled("wait", `Run cancelled: ${error.message}`), + note("run-cancelled", error.message), + ]); + }); +}); + +void test("steering is saved when it arrives, and mid-batch after the batch's results", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + const reader = new LazyStateConversation({store}); + await execute(function* () { + // Channels are one-shot: one per steering message. + const first = r.channel(); + const second = r.channel(); + const never = r.channel(); + const gate = r.channel(); + const sources = [first, second]; + let step = 0; + const snapshots: Message[][] = []; + const a = agent({ + tools: { + write: tool({ + ...base, + *execute() { + return "written"; + }, + }), + steer: tool({ + ...base, + *execute() { + yield* second.send("Read it back"); + yield* gate.receive; + return "steered"; + }, + }), + }, + *model() { + step++; + if (step === 1) { + yield* first.send("Use the new address"); + yield* r.channel().receive; + return {type: "final", text: "stale"}; + } + return step === 2 ? batch("write", "steer") : done; + }, + }); + const result = yield* a.run("go", { + conversation, + controls: {steering: () => (sources.shift() ?? never).receive}, + *onProgress(event) { + if (event.type !== "steering") return; + snapshots.push((yield* reader.page()).messages); + if (snapshots.length === 2) yield* gate.send(undefined); + }, + }); + assert.deepEqual(snapshots, [ + // During the first model call: the input and the steering note, at once. + [user("go"), note("steering", "Use the new address")], + // Mid-batch: the results so far are saved; the note waits until the batch ends. + [ + user("go"), + note("steering", "Use the new address"), + calls("write", "steer"), + success("write", "written"), + ], + ]); + const saved = (yield* reader.page()).messages; + history(saved); + assert.deepEqual(saved, [ + user("go"), + note("steering", "Use the new address"), + calls("write", "steer"), + success("write", "written"), + success("steer", "steered"), + note("steering", "Read it back"), + answer("done"), + ]); + assert.deepEqual(saved, result.messages); + }); +}); + +void test("an interrupted run saves its calls as they start and closes them when it stops", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + const reader = new LazyStateConversation({store}); + await execute(function* () { + const interrupt = r.channel(); + const a = agent({ + finalize: false, + tools: { + slow: tool({ + ...base, + *execute() { + assert.deepEqual((yield* reader.page()).messages, [user("go"), calls("slow")]); + yield* interrupt.send("user left"); + yield* r.channel().receive; + return "unreachable"; + }, + }), + }, + *model() { + return batch("slow"); + }, + }); + const result = yield* a.run("go", {conversation, controls: {interrupt: interrupt.receive}}); + assert.equal(result.status, "interrupted"); + const saved = (yield* reader.page()).messages; + assert.deepEqual(saved, [user("go"), calls("slow"), cancelled("slow", "user left")]); + assert.deepEqual(saved, result.messages); + }); +}); + +void test("the next run closes the calls a killed run left without results", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + await execute(function* () { + // A run saved its calls and one result, then its invocation was killed. + yield* conversation.append([user("pay"), calls("charge", "email"), success("charge", "ok")], 0); + const closed = cancelled( + "email", + "The run that made this call ended before the call returned a result. It may have taken effect.", + ); + const a = agent({ + *model(request) { + assert.deepEqual(request.messages, [ + user("pay"), + calls("charge", "email"), + success("charge", "ok"), + closed, + user("status?"), + ]); + return done; + }, + }); + const result = yield* a.run("status?", {conversation}); + const saved = (yield* conversation.load()).messages; + // The repair is saved with the run, so the stored history is complete again. + history(saved); + assert.deepEqual(saved, result.messages); + assert.deepEqual(saved.slice(3, 5), [closed, user("status?")]); + }); +}); + +void test("compaction prepared mid-run stays before the open batch and applies after the run", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + // Another handler prepares and applies compaction. + const other = new LazyStateConversation({store}); + await execute(function* () { + yield* conversation.append(pairs(6), 0); + let plan: CompactionPlan | undefined; + const a = agent({ + tools: { + lookup: tool({ + ...base, + *execute() { + plan = yield* other.prepareCompaction({maxMessages: 8, retainMessages: 2}); + return "found"; + }, + }), + }, + *model(request) { + return request.step === 1 ? batch("lookup") : done; + }, + }); + yield* a.run("go", {conversation}); + assert.ok(plan); + // The boundary stops at the run's input: the open batch stays verbatim. + assert.equal(plan.through, 12); + history(plan.messages); + assert.equal(yield* other.applyCompaction({plan, summary: "six earlier questions"}), true); + const loaded = yield* conversation.load(); + history(loaded.messages); + assert.match(loaded.messages[0]!.role === "user" ? loaded.messages[0].content : "", /six/); + assert.deepEqual(loaded.messages.slice(1), [ + user("go"), + calls("lookup"), + success("lookup", "found"), + answer("done"), + ]); + // The next run continues from the compacted history, and its saves pass the version check. + const next = yield* agent({ + *model() { + return done; + }, + }).run("again", {conversation}); + assert.equal(next.status, "completed"); + assert.deepEqual((yield* other.page(16)).messages, [user("again"), answer("done")]); + }); +}); + +void test("another writer fails the run's next save, which then saves no failure note", async () => { + const {store} = memoryStore(); + const conversation = new LazyStateConversation({store}); + const writer = new LazyStateConversation({store}); + await execute(function* () { + const a = agent({ + *model() { + yield* writer.append([user("someone else")], 0); + return batch("missing"); + }, + }); + const error = yield* failure(a.run("go", {conversation})); + assert.ok(error instanceof ConversationConflictError); + assert.deepEqual((yield* writer.page()).messages, [user("someone else")]); + assert.match(JSON.stringify(messagesOf(error)?.at(-1)), /run-failed/); + }); +}); + +void test("a failed run with messages exposes its history for the next run", async () => { + const prior = [user("hi"), answer("hello")]; + await execute(function* () { + const failing = agent({ + tools: { + halt: tool({ + ...base, + *execute() { + return stop("quota exhausted"); + }, + }), + }, + *model() { + return batch("halt"); + }, + }); + const error = yield* failure(failing.run("go", {messages: prior})); + const after = messagesOf(error); + assert.ok(after); + history(after); + assert.deepEqual(after, [ + ...prior, + user("go"), + calls("halt"), + cancelled("halt", "Run failed: quota exhausted"), + note("run-failed", "quota exhausted"), + ]); + const next = agent({ + *model(request) { + assert.deepEqual(request.messages, [...after, user("again")]); + return done; + }, + }); + assert.equal((yield* next.run("again", {messages: after})).status, "completed"); + }); +}); + +void test("a run saves at most once per model call and tool batch", async () => { + const {store} = memoryStore(); + const stored = new LazyStateConversation({store}); + const appended: number[] = []; + const conversation: Conversation = { + load: () => stored.load(), + *append(messages, version) { + appended.push(messages.length); + return yield* stored.append(messages, version); + }, + }; + await execute(function* () { + const a = agent({ + tools: { + lookup: tool({ + ...base, + *execute() { + return "found"; + }, + }), + }, + *model(request) { + return request.step === 1 ? batch("lookup") : done; + }, + }); + yield* a.run("go", {conversation}); + // The input and call, the result, then the answer. + assert.deepEqual(appended, [2, 1, 1]); + appended.length = 0; + yield* agent({ + *model() { + return done; + }, + }).run("again", {conversation}); + // The input after the model call, then the reviewed answer. + assert.deepEqual(appended, [1, 1]); + }); +}); diff --git a/packages/tests/src/conversation-replay.test.ts b/packages/tests/src/conversation-replay.test.ts new file mode 100644 index 0000000..aca8589 --- /dev/null +++ b/packages/tests/src/conversation-replay.test.ts @@ -0,0 +1,149 @@ +// A run's saves are journaled writes, so a replay must make the same writes in +// the same order. The in-process harness journals only run steps, so this store +// journals its reads and writes as run steps; Restate journals state the same way. +// Each test resumes the invocation from every journal entry and checks that it +// ends with the same output and the same stored conversation. +import assert from "node:assert/strict"; +import {test} from "node:test"; +import {TerminalError, type Context} from "@restatedev/restate-sdk"; +import * as r from "@restatedev/restate-sdk-gen"; +import {z} from "zod"; +import { + agent, + journaledModel, + LazyStateConversation, + tool, + type ConversationStore, + type ModelResponse, +} from "@restatedev/agentlib"; +import {runHandler} from "./replay-harness.mjs"; + +type Store = Map; + +/** A store whose reads and writes are journaled run steps over `state.values`. */ +function journaledStore(state: {values: Store; writes: number}): ConversationStore { + return { + *get(key: string) { + const value = yield* r.run(async () => structuredClone(state.values.get(key) ?? null), { + name: `get ${key}`, + }); + return (value ?? undefined) as T | undefined; + }, + *set(key, value) { + yield* r.run( + async () => { + state.writes++; + state.values.set(key, structuredClone(value)); + return null; + }, + {name: `set ${key}`}, + ); + }, + }; +} + +/** Runs `handler` live, then on full replay and resumed from every journal entry. */ +async function checkReplay( + state: {values: Store; writes: number}, + handler: (ctx: Context) => Promise, +) { + // The store after each run step: a resumed attempt starts from what the replayed steps wrote. + const after: Store[] = []; + const live = await runHandler(handler, { + onProposal: () => after.push(structuredClone(state.values)), + }); + const final = structuredClone(state.values); + const writes = state.writes; + assert.ok(writes > 0); + + // A full replay repeats no write: every one comes from the journal. + state.values = new Map(); + const replay = await runHandler(handler, {replay: live.journal}); + assert.deepEqual(replay.output, live.output); + assert.equal(state.writes, writes); + assert.equal(state.values.size, 0); + + for (let entries = 1; entries < live.journal.length; entries++) { + // Entries after the input are run commands, each followed by its completion. + const completed = Math.floor((entries - 1) / 2); + state.values = completed ? structuredClone(after[completed - 1]!) : new Map(); + const resumed = await runHandler(handler, {replay: live.journal.slice(0, entries)}); + assert.deepEqual(resumed.output, live.output, `resumed after ${entries} entries`); + assert.deepEqual(state.values, final, `stored conversation after ${entries} entries`); + } + return live.output; +} + +const lookup = tool({ + description: "Look up an order", + input: z.object({}), + *execute() { + return "shipped"; + }, +}); + +void test("replay makes a run's saves again in the same order", {timeout: 20000}, async () => { + const state = {values: new Map() as Store, writes: 0}; + const a = agent({ + tools: {lookup}, + model: journaledModel(async (request): Promise => { + if (request.step === 1) + return {type: "tool_calls", calls: [{id: "l", name: "lookup", input: {}}]}; + return {type: "final", text: "It has shipped."}; + }), + }); + const handler = (ctx: Context) => + r.execute( + ctx, + (function* () { + const conversation = new LazyStateConversation({store: journaledStore(state)}); + const result = yield* a.run("Where is my order?", {conversation}); + return {status: result.status, saved: (yield* conversation.page()).messages}; + })(), + ); + const output = (await checkReplay(state, handler)) as {status: string; saved: unknown[]}; + assert.equal(output.status, "completed"); + assert.equal(output.saved.length, 4); +}); + +void test( + "replay makes a failed run's saves again, with its failure note", + {timeout: 20000}, + async () => { + const state = {values: new Map() as Store, writes: 0}; + const a = agent({ + tools: {lookup}, + model: journaledModel(async (request): Promise => { + if (request.step === 1) + return {type: "tool_calls", calls: [{id: "l", name: "lookup", input: {}}]}; + throw new TerminalError("model unavailable"); + }), + }); + const handler = (ctx: Context) => + r.execute( + ctx, + (function* () { + const conversation = new LazyStateConversation({store: journaledStore(state)}); + let error: string | undefined; + try { + yield* a.run("Where is my order?", {conversation}); + } catch (failure) { + // Replay restores the journaled failure as a plain TerminalError. + if (!(failure instanceof TerminalError)) throw failure; + error = failure.message; + } + return {error, saved: (yield* conversation.page()).messages}; + })(), + ); + const output = (await checkReplay(state, handler)) as {error: string; saved: unknown[]}; + assert.equal(output.error, "model unavailable"); + assert.deepEqual(output.saved.at(-1), { + role: "user", + content: JSON.stringify({ + source: "agent-runtime", + type: "run-failed", + message: "model unavailable", + }), + }); + }, +); diff --git a/packages/tests/src/errors.test.ts b/packages/tests/src/errors.test.ts index 4ee2323..bf72727 100644 --- a/packages/tests/src/errors.test.ts +++ b/packages/tests/src/errors.test.ts @@ -20,6 +20,7 @@ import { isContextOverflow, journaledModel, LazyStateConversation, + messagesOf, RunFailedError, RunInputError, tool, @@ -32,6 +33,7 @@ import { type Tool, type ToolResult, } from "@restatedev/agentlib"; +import {history} from "../../core/src/validation.js"; import {runHandler} from "./replay-harness.mjs"; import {memoryStore, testContext} from "./helpers.js"; @@ -588,7 +590,7 @@ void test("a TerminalError in beforeSteering rejects the message, and the run co ]); }); -void test("an AgentTerminalError fails the run from anywhere, and nothing is saved", async () => { +void test("an AgentTerminalError fails the run from anywhere, and the conversation keeps the failed turn", async () => { const sites: Record>> = { "a tool": { tools: { @@ -652,7 +654,7 @@ void test("an AgentTerminalError fails the run from anywhere, and nothing is sav }, }; for (const [site, options] of Object.entries(sites)) { - const {store, values} = memoryStore(); + const {store} = memoryStore(); const conversation = new LazyStateConversation({store}); const a = agent({ *model(request) { @@ -660,12 +662,45 @@ void test("an AgentTerminalError fails the run from anywhere, and nothing is sav }, ...options, }); + let failure: unknown; await assert.rejects( r.execute(ctx(), a.run("go", {conversation})), - {name: "AgentTerminalError", message: "stop now"}, + (error) => { + failure = error; + return error instanceof AgentTerminalError && error.message === "stop now"; + }, + site, + ); + const saved = await r.execute( + ctx(), + (function* () { + return (yield* conversation.load()).messages; + })(), + ); + if (site === "beforeRun") { + // Stopping in beforeRun keeps the input out of the history. + assert.deepEqual(saved, [], site); + assert.equal(messagesOf(failure), undefined, site); + continue; + } + // Every saved call has a result, and the next run reads why this one failed. + history(saved); + assert.deepEqual(saved, messagesOf(failure), site); + assert.deepEqual(saved[0], {role: "user", content: "go"}, site); + assert.deepEqual( + saved.at(-1), + { + role: "user", + content: JSON.stringify({source: "agent-runtime", type: "run-failed", message: "stop now"}), + }, + site, + ); + const results = saved.flatMap((message) => (message.role === "tool" ? [message.result] : [])); + assert.deepEqual( + results, + options.tools ? [{status: "cancelled", message: "Run failed: stop now"}] : [], site, ); - assert.equal(values.size, 0, site); } }); diff --git a/packages/tests/src/runtime.integration.ts b/packages/tests/src/runtime.integration.ts index 407c884..d2348d0 100644 --- a/packages/tests/src/runtime.integration.ts +++ b/packages/tests/src/runtime.integration.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import {after, before, test} from "node:test"; -import {TerminalError} from "@restatedev/restate-sdk"; +import {InvocationIdParser, TerminalError} from "@restatedev/restate-sdk"; import {connect} from "@restatedev/restate-sdk-clients"; import {RestateContainer, RestateTestEnvironment} from "@restatedev/restate-sdk-testcontainers"; import * as r from "@restatedev/restate-sdk-gen"; @@ -629,7 +629,7 @@ const runner = r.service({ const session = r.object({ name: "SdkConversation", handlers: { - *persistTurn(ending: "completed" | "stopped" | "interrupted" | "failed") { + *persistTurn(ending: TurnEnding) { const stored = new LazyStateConversation(); let loads = 0; let appends = 0; @@ -656,11 +656,28 @@ const session = r.object({ return "written"; }, }), + halt: tool({ + ...base, + *execute() { + // Ends the run after `write` has returned. + yield* r.sleep(200); + throw new AgentTerminalError("halted"); + }, + }), }, *model(request): r.Operation { - assert.deepEqual((yield* stored.page()).messages, []); - if (request.step === 1) - return {type: "tool_calls", calls: [{id: "write", name: "write", input: {}}]}; + const saved = (yield* stored.page()).messages; + if (request.step === 1) { + assert.deepEqual(saved, []); + const calls = [{id: "write", name: "write", input: {}}]; + if (ending === "failed mid-batch") calls.push({id: "halt", name: "halt", input: {}}); + return {type: "tool_calls", calls}; + } + // The call and its result were saved when the batch ended. + assert.deepEqual( + saved.map((message) => message.role), + ["user", "assistant", "tool"], + ); if (ending === "failed") throw new TerminalError("failure after effect"); if (ending === "interrupted") { r.invocation(r.handlerRequest().id).signal("interrupt").resolve("stop"); @@ -776,6 +793,39 @@ const session = r.object({ *history() { return yield* new LazyStateConversation().page(); }, + /** A turn whose tool waits for `release`, so a shared handler can read it mid-run. */ + *waitingTurn() { + const conversation = new LazyStateConversation(); + const result = yield* agent({ + finalize: false, + tools: { + wait: tool({ + ...base, + *execute() { + return yield* r.signal("release"); + }, + }), + }, + *model(request): r.Operation { + if (request.step === 1) + return {type: "tool_calls", calls: [{id: "wait", name: "wait", input: {}}]}; + return final; + }, + }).run("go", {conversation}); + return {status: result.status, messages: result.messages}; + }, + *progress() { + return yield* new LazyStateConversation().page(); + }, + *release(invocationId: string) { + r.invocation(invocationId).signal("release").resolve("released"); + }, + *cancelTurn(invocationId: string) { + r.cancel(InvocationIdParser.fromString(invocationId)); + }, + }, + options: { + handlers: {progress: {shared: true}, release: {shared: true}, cancelTurn: {shared: true}}, }, }); // A Restate interface (contract) and its implementation, exercised through serviceTool. @@ -1299,7 +1349,55 @@ void test( }, ); -for (const ending of ["completed", "stopped", "interrupted", "failed"] as const) { +const input = {role: "user", content: "go"}; +const call = (...ids: string[]) => ({ + role: "assistant", + content: "", + calls: ids.map((id) => ({id, name: id, input: {}})), +}); +const written = { + role: "tool", + callId: "write", + name: "write", + result: {status: "success", output: "written"}, +}; +const runtimeNote = (type: string, message: string) => ({ + role: "user", + content: JSON.stringify({source: "agent-runtime", type, message}), +}); +// The saves a turn makes: after the model's call, after the batch, and at the end. +const turnEndings = { + completed: { + appends: 3, + messages: [input, call("write"), written, {role: "assistant", content: "done"}], + }, + stopped: {appends: 2, messages: [input, call("write"), written]}, + interrupted: {appends: 2, messages: [input, call("write"), written]}, + failed: { + appends: 3, + messages: [input, call("write"), written, runtimeNote("run-failed", "failure after effect")], + }, + "failed mid-batch": { + appends: 2, + messages: [ + input, + call("write", "halt"), + written, + { + role: "tool", + callId: "halt", + name: "halt", + result: {status: "cancelled", message: "Run failed: halted"}, + }, + runtimeNote("run-failed", "halted"), + ], + }, +}; +type TurnEnding = keyof typeof turnEndings; +for (const [ending, expected] of Object.entries(turnEndings) as [ + TurnEnding, + (typeof turnEndings)[TurnEnding], +][]) { void test( `native always-replay: turn persistence after ${ending}`, {timeout: 20000}, @@ -1307,16 +1405,75 @@ for (const ending of ["completed", "stopped", "interrupted", "failed"] as const) const result = await connect({url: env.baseUrl()}) .client(session, `persist-${ending}`) .persistTurn(ending); - assert.equal(result.status, ending); + assert.equal(result.status, ending.startsWith("failed") ? "failed" : ending); assert.equal(result.loads, 1); assert.equal(result.effect, "written"); - if (ending === "failed") { - assert.equal(result.appends, 0); - assert.deepEqual(result.messages, []); - } else { - assert.equal(result.appends, 1); - assert.match(JSON.stringify(result.messages), /written/); - } + assert.deepEqual(result.messages, expected.messages); + assert.equal(result.appends, expected.appends); }, ); } + +/** Reads until `ready` accepts the value, or the deadline passes. */ +async function poll(read: () => Promise, ready: (value: T) => boolean): Promise { + const deadline = Date.now() + 15000; + for (;;) { + const value = await read(); + if (ready(value) || Date.now() > deadline) return value; + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + +void test( + "native always-replay: a shared handler reads a running turn's progress", + {timeout: 30000}, + async () => { + const ingress = connect({url: env.baseUrl()}); + const send = await ingress.sendClient(session, "progress").waitingTurn(); + const turn = ingress.client(session, "progress"); + // The call is saved before it runs; the tool waits for the release. + const running = await poll( + () => turn.progress(), + (page) => page.messages.length === 2, + ); + assert.deepEqual(running.messages, [input, call("wait")]); + await turn.release(send.invocationId); + const finished = await poll( + () => turn.progress(), + (page) => page.messages.length === 4, + ); + assert.deepEqual(finished.messages, [ + input, + call("wait"), + {role: "tool", callId: "wait", name: "wait", result: {status: "success", output: "released"}}, + {role: "assistant", content: "done"}, + ]); + }, +); + +void test( + "native always-replay: a cancelled turn saves its closed calls and a cancellation note", + {timeout: 30000}, + async () => { + const ingress = connect({url: env.baseUrl()}); + const send = await ingress.sendClient(session, "cancel").waitingTurn(); + const turn = ingress.client(session, "cancel"); + await poll( + () => turn.progress(), + (page) => page.messages.length === 2, + ); + await turn.cancelTurn(send.invocationId); + const saved = ( + await poll( + () => turn.progress(), + (page) => page.messages.length === 4, + ) + ).messages; + assert.deepEqual(saved.slice(0, 2), [input, call("wait")]); + const [closed, cancelled] = saved.slice(2); + assert.ok(closed?.role === "tool" && closed.result.status === "cancelled"); + assert.match(closed.result.message, /^Run cancelled: /); + assert.ok(cancelled?.role === "user"); + assert.match(cancelled.content, /"type":"run-cancelled"/); + }, +); diff --git a/plugins/restate-agents/skills/restate-agents/references/agent-basics.md b/plugins/restate-agents/skills/restate-agents/references/agent-basics.md index 0507114..7348b4a 100644 --- a/plugins/restate-agents/skills/restate-agents/references/agent-basics.md +++ b/plugins/restate-agents/skills/restate-agents/references/agent-basics.md @@ -80,7 +80,8 @@ try { } ``` -`models-and-testing.md` lists what the totals include. +`models-and-testing.md` lists what the totals include. `messagesOf(error)` +returns the failed run's history the same way; see `error-handling.md`. ## Typed output diff --git a/plugins/restate-agents/skills/restate-agents/references/approvals-and-guardrails.md b/plugins/restate-agents/skills/restate-agents/references/approvals-and-guardrails.md index 10fa10a..bf21c9b 100644 --- a/plugins/restate-agents/skills/restate-agents/references/approvals-and-guardrails.md +++ b/plugins/restate-agents/skills/restate-agents/references/approvals-and-guardrails.md @@ -156,8 +156,8 @@ skips the summary. The run's history is what the model sees in later steps, what summaries see, `result.messages`, and what a `conversation` saves. A run saves its history -when it finishes, completed or stopped; a run that fails saves nothing. The -model also sees the notes about rejections and the answers `afterRun` +as it goes and when it ends; a run that fails saves what it recorded, with a +note on the failure (see `conversations-and-controls.md`). The model also sees the notes about rejections and the answers `afterRun` rejected, for the rest of the run, but they are never recorded. | Hook | Allowed: history keeps | Rejected: history keeps | Replaced: history keeps | diff --git a/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md b/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md index 4172fe3..fe11068 100644 --- a/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md +++ b/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md @@ -8,7 +8,9 @@ Lessons: ## Remember earlier requests A conversation belongs to a key, so keep it in a virtual object. Load the -messages, pass them in, and save `result.messages`: +messages, pass them in, and save `result.messages`. A run that throws returns +no result; `messagesOf(error)` returns its history up to the failure, with a +note on why it failed: ```ts import {agent, type Message} from "@restatedev/agentlib"; @@ -45,6 +47,33 @@ const result = yield* support.run(message, {conversation}); Read it from a shared handler with `yield* new LazyStateConversation().page()`. +### When a run saves + +A run saves to its `conversation` while it runs, so `page()` in a shared +handler shows the turn that is running: + +- **After each model call.** A tool step's calls are saved before they run. A + final answer is saved once `afterRun` accepts it. +- **When a tool batch ends**, with every result. A background call's result is + its pending acknowledgment; the real result is saved when the run collects it. +- **When steering is accepted.** Steering that arrives during a batch is saved + after the batch's results, because results must follow their calls. + +Each save appends to the last one. The saved history stays the full record: +error notes, rejected answers, and in-run compaction never reach it. + +A run that fails with a terminal error saves what it recorded, closes +unfinished calls with `cancelled` results, and adds a note on the failure; see +`error-handling.md`. The input is saved with the first model call, so a caller +that calls the handler again with the same input after a failure starts a new +turn after the failed one. To retry the same request, call the handler with an +idempotency key. + +A killed invocation (`restate invocations kill`) runs no more code, so its +batch's calls can stay without results. The next run gives each such call a +`cancelled` result saying that it may have taken effect, and saves it with its +first save. + When a run starts and the conversation compaction fails, the run logs a warning and continues with the history it loaded; the next run tries again. An `AgentTerminalError` from the compaction model fails the run instead. diff --git a/plugins/restate-agents/skills/restate-agents/references/error-handling.md b/plugins/restate-agents/skills/restate-agents/references/error-handling.md index 18296b2..a317098 100644 --- a/plugins/restate-agents/skills/restate-agents/references/error-handling.md +++ b/plugins/restate-agents/skills/restate-agents/references/error-handling.md @@ -167,7 +167,9 @@ An `AgentTerminalError` never reaches a model. Thrown from any place, it fails the run: - The other calls running in the same batch are interrupted. -- Nothing is saved to the conversation, as for any run that throws. +- The conversation keeps the failed turn: calls that didn't finish get + `cancelled` results, followed by a note on the failure. See + [What a failed run keeps](#what-a-failed-run-keeps). - Every agent run that encloses this one fails too: a parent that called this agent through `agentTool`, the run a fork was started from, and the agent whose tool called the handler that threw it. @@ -290,7 +292,8 @@ without running `beforeStep`. Set `finalize: false`, throw A rejected input stays in the run's history and is saved when the run completes. Return `{message}` from `beforeRun` to record a redacted input -instead, or throw `AgentTerminalError` to stop before anything is recorded. See +instead, or throw `AgentTerminalError` to stop before anything is recorded: a +run that `beforeRun` stops saves nothing, not even a failure note. See `approvals-and-guardrails.md` for each hook. ## Errors across journaled steps and RPC @@ -317,10 +320,11 @@ Cancellation is never sent to the model as something to recover from: - **The invocation is cancelled.** The Restate SDK throws a `CancelledError` at the run's next `yield*`, wherever the run is: in a tool, a hook, or the model - call. The run lets it through, without a note or a tool result, and skips its - fallbacks. The other running calls are interrupted, child invocations the - run started are cancelled, nothing is saved, and the handler fails with the - `CancelledError`. + call. The run lets it through, without an error note or a tool result, and + skips its fallbacks. The other running calls are interrupted, child + invocations the run started are cancelled, and the handler fails with the + `CancelledError`. The conversation keeps the turn, with a `run-cancelled` + note. - **The run is interrupted** through `interruptSignal` or `controls.interrupt`. This isn't an error: the run ends with status `interrupted`. Calls that didn't finish are recorded with `{status: "cancelled"}`. The model that @@ -358,8 +362,27 @@ An `AgentTerminalError` and cancellation skip these fallbacks and fail the run. ## What a failed run keeps -A run that throws saves nothing to its conversation, so the next run starts -from the history before the failed one. It does not return partial messages -either. `usageOf(error)` returns the token totals up to the failure; record -them only for terminal errors, because Restate retries other errors and the -replay counts the same calls again. +When a terminal error fails a run, the run closes what it started and records +why: + +- Calls that didn't finish get `cancelled` results, with the message + `Run failed: `. +- A note follows them. Its type is `run-cancelled` for a cancelled invocation: + + ```json + {"source": "agent-runtime", "type": "run-failed", "message": "model unavailable"} + ``` + +- A run with a `conversation` saves this history before it rethrows the + original error, so the next run's model reads what happened. Error notes and + rejected answers are never saved. +- `messagesOf(error)` returns the same history, for a run with `messages` too. + Save it as the next run's `messages` to keep the failed turn. + +A retryable error saves no note: Restate retries the invocation, and the +replay continues the run from what it saved. A run that `beforeRun` stops +records nothing. + +`usageOf(error)` returns the token totals up to the failure; record them only +for terminal errors, because Restate retries other errors and the replay counts +the same calls again. From 6cd0e32218f64989fcb796c0c58829a291ac1c3e Mon Sep 17 00:00:00 2001 From: Giselle van Dongen Date: Sat, 21 Nov 2026 08:27:38 +0200 Subject: [PATCH 2/2] Only read from Restate at the beginning --- packages/core/src/conversation.ts | 50 ++++++++++-- .../tests/src/conversation-progress.test.ts | 79 +++++++++++++++++++ .../references/conversations-and-controls.md | 6 ++ 3 files changed, 127 insertions(+), 8 deletions(-) diff --git a/packages/core/src/conversation.ts b/packages/core/src/conversation.ts index f32a29a..f52d6ce 100644 --- a/packages/core/src/conversation.ts +++ b/packages/core/src/conversation.ts @@ -1,5 +1,5 @@ import {TerminalError} from "@restatedev/restate-sdk"; -import {sharedState, state, type Operation} from "@restatedev/restate-sdk-gen"; +import {contextLocal, sharedState, state, type Operation} from "@restatedev/restate-sdk-gen"; import {readable} from "./context-size.js"; import type {Message, Model} from "./types.js"; import {checkProtocol, history, modelResponse, parseMessages, type OpenCall} from "./validation.js"; @@ -44,6 +44,8 @@ export class ConversationConflictError extends TerminalError { } } const CHUNK_SIZE = 32; +/** The value of each key this invocation read or wrote, per store. */ +const known = contextLocal>>(); const restateStore: ConversationStore = { *get(key: string) { return (yield* sharedState().get(key)) ?? undefined; @@ -54,7 +56,15 @@ const restateStore: ConversationStore = { }; /** Append-only chunks in an application-owned exclusive Virtual Object handler. - * No constructor I/O and no process-local cache. Use one writer per conversation. */ + * No constructor I/O. Use one writer per conversation. + * + * Within one invocation, each key is read from the store at most once. Restate + * journals every state read with its value, so a run that loads, compacts, and + * saves would otherwise journal the same chunks and metadata several times. The + * values are kept per invocation, not per instance, so every instance on the + * same store sees the writes of the others, and a replay reads them from the + * journal again. A handler that reads within one invocation does not see + * writes from other invocations after its first read of a key. */ export class LazyStateConversation implements Conversation { private readonly store: ConversationStore; private readonly metaKey: string; @@ -73,8 +83,32 @@ export class LazyStateConversation implements Conversation { this.metaKey = options.metaKey ?? `${this.stateKey}/meta`; } + private known(): Map { + let stores = known.get(); + if (!stores) known.set((stores = new WeakMap())); + let values = stores.get(this.store); + if (!values) stores.set(this.store, (values = new Map())); + return values; + } + + /** The key's value as this invocation last read or wrote it. Callers clone + * what they change: the value is shared for the rest of the invocation. */ + private *read(key: string): Operation { + if (this.known().has(key)) return this.known().get(key) as T | undefined; + const value = yield* this.store.get(key); + // A write during the read is newer than the value read. + const values = this.known(); + if (!values.has(key)) values.set(key, value); + return values.get(key) as T | undefined; + } + + private *write(key: string, value: T): Operation { + yield* this.store.set(key, value); + this.known().set(key, value); + } + private *meta(): Operation { - return structuredClone((yield* this.store.get(this.metaKey)) ?? {count: 0}); + return structuredClone((yield* this.read(this.metaKey)) ?? {count: 0}); } private key(index: number) { @@ -85,7 +119,7 @@ export class LazyStateConversation implements Conversation { private *range(from: number, through: number): Operation { const messages: Message[] = []; for (let i = Math.floor(from / CHUNK_SIZE); i < Math.ceil(through / CHUNK_SIZE); i++) { - const chunk = yield* this.store.get(this.key(i)); + const chunk = yield* this.read(this.key(i)); if (!chunk) throw new ConversationConflictError(`Missing conversation chunk: ${i}`); const offset = i * CHUNK_SIZE; messages.push(...chunk.slice(Math.max(0, from - offset), through - offset)); @@ -121,15 +155,15 @@ export class LazyStateConversation implements Conversation { while (offset < additions.length) { const index = Math.floor(meta.count / CHUNK_SIZE); const used = meta.count % CHUNK_SIZE; - const chunk = used ? yield* this.store.get(this.key(index)) : []; + const chunk = used ? yield* this.read(this.key(index)) : []; if (!chunk || chunk.length !== used) throw new ConversationConflictError("Invalid conversation tail"); const next = additions.slice(offset, offset + CHUNK_SIZE - used); - yield* this.store.set(this.key(index), [...chunk, ...next]); + yield* this.write(this.key(index), [...chunk, ...next]); offset += next.length; meta.count += next.length; } - yield* this.store.set(this.metaKey, meta); + yield* this.write(this.metaKey, meta); return meta.count; } @@ -191,7 +225,7 @@ export class LazyStateConversation implements Conversation { if ((meta.summary?.through ?? 0) !== plan.baseThrough) return false; // Verify the retained suffix independently before committing the new boundary. history(yield* this.range(plan.through, meta.count), {unfinished: true}); - yield* this.store.set(this.metaKey, {...meta, summary: {through: plan.through, text: summary}}); + yield* this.write(this.metaKey, {...meta, summary: {through: plan.through, text: summary}}); return true; } diff --git a/packages/tests/src/conversation-progress.test.ts b/packages/tests/src/conversation-progress.test.ts index 6cc81a3..f3e1c27 100644 --- a/packages/tests/src/conversation-progress.test.ts +++ b/packages/tests/src/conversation-progress.test.ts @@ -15,7 +15,9 @@ import { tool, type CompactionPlan, type Conversation, + type ConversationStore, type Message, + type Model, type ModelResponse, type ToolResult, } from "@restatedev/agentlib"; @@ -566,3 +568,80 @@ void test("a run saves at most once per model call and tool batch", async () => assert.deepEqual(appended, [1, 1]); }); }); + +/** A memory store that counts the reads of each key. */ +function countingStore() { + const {store, values} = memoryStore(); + const reads = new Map(); + const counted: ConversationStore = { + *get(key: string) { + reads.set(key, (reads.get(key) ?? 0) + 1); + return yield* store.get(key); + }, + set: (key, value) => store.set(key, value), + }; + return {store: counted, values, reads}; +} + +void test("a run reads each conversation key once, through compaction and every save", async () => { + const {store, reads} = countingStore(); + const summarize: Model = function* () { + return {type: "final", text: "twenty earlier questions"}; + }; + const conversation = new LazyStateConversation({ + store, + compaction: {model: summarize, policy: {maxMessages: 8, retainMessages: 4}}, + }); + // Two chunks of history, written by an earlier invocation. + await execute(function* () { + yield* new LazyStateConversation({store}).append(pairs(20), 0); + }); + reads.clear(); + await execute(function* () { + const a = agent({ + tools: { + lookup: tool({ + ...base, + *execute() { + return "found"; + }, + }), + }, + *model(request) { + if (request.step !== 1) return done; + assert.match(JSON.stringify(request.messages[0]), /twenty earlier questions/); + return batch("lookup"); + }, + }); + assert.equal((yield* a.run("go", {conversation})).status, "completed"); + }); + // The load, the compaction, the load after it, and three saves read each key once. + assert.deepEqual(Object.fromEntries(reads), { + "conversation/meta": 1, + "conversation/0": 1, + "conversation/1": 1, + }); + await execute(function* () { + assert.deepEqual((yield* conversation.page(40)).messages, [ + user("go"), + calls("lookup"), + success("lookup", "found"), + answer("done"), + ]); + }); +}); + +void test("the next invocation reads the store again, with the same instance", async () => { + const {store, reads} = countingStore(); + const conversation = new LazyStateConversation({store}); + await execute(function* () { + yield* conversation.append([user("one")], 0); + }); + await execute(function* () { + yield* new LazyStateConversation({store}).append([user("two")], 1); + }); + await execute(function* () { + assert.deepEqual((yield* conversation.load()).messages, [user("one"), user("two")]); + }); + assert.equal(reads.get("conversation/meta"), 3); +}); diff --git a/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md b/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md index fe11068..7306219 100644 --- a/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md +++ b/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md @@ -62,6 +62,12 @@ handler shows the turn that is running: Each save appends to the last one. The saved history stays the full record: error notes, rejected answers, and in-run compaction never reach it. +Restate journals every state read with its value. Within one invocation, +`LazyStateConversation` reads each state key at most once and keeps what it +read and wrote, so the load, compaction, and saves of a run don't journal the +same messages again. A shared handler that calls `page()` twice in one +invocation gets the values from its first read; call it once per invocation. + A run that fails with a terminal error saves what it recorded, closes unfinished calls with `cancelled` results, and adds a note on the failure; see `error-handling.md`. The input is saved with the first model call, so a caller